Rollup merge of #147355 - sayantn:masked-loads, r=RalfJung,bjorn3

Add alignment parameter to `simd_masked_{load,store}`

This PR adds an alignment parameter in `simd_masked_load` and `simd_masked_store`, in the form of a const-generic enum `core::intrinsics::simd::SimdAlign`. This represents the alignment of the `ptr` argument in these intrinsics as follows

 - `SimdAlign::Unaligned` - `ptr` is unaligned/1-byte aligned
 - `SimdAlign::Element` - `ptr` is aligned to the element type of the SIMD vector (default behavior in the old signature)
 - `SimdAlign::Vector` - `ptr` is aligned to the SIMD vector type

The main motive for this is stdarch - most vector loads are either fully aligned (to the vector size) or unaligned (byte-aligned), so the previous signature doesn't cut it.

Now, stdarch will mostly use `SimdAlign::Unaligned` and `SimdAlign::Vector`, whereas portable-simd will use `SimdAlign::Element`.

 - [x] `cg_llvm`
 - [x] `cg_clif`
 - [x] `miri`/`const_eval`

## Alternatives

Using a const-generic/"const" `u32` parameter as alignment (and we error during codegen if this argument is not a power of two). This, although more flexible than this, has a few drawbacks

 - If we use an const-generic argument, then portable-simd somehow needs to pass `align_of::<T>()` as the alignment, which isn't possible without GCE
 - "const" function parameters are just an ugly hack, and a pain to deal with in non-LLVM backends

We can remedy the problem with the const-generic `u32` parameter by adding a special rule for the element alignment case (e.g. `0` can mean "use the alignment of the element type), but I feel like this is not as expressive as the enum approach, although I am open to suggestions

cc `@workingjubilee` `@RalfJung` `@BoxyUwU`

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -2,6 +2,7 @@
 
 use cranelift_codegen::ir::immediates::Offset32;
 use rustc_abi::Endian;
+use rustc_middle::ty::SimdAlign;
 
 use super::*;
 use crate::prelude::*;
@@ -960,6 +961,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap();
             let ptr_val = ptr.load_scalar(fx);
 
+            let alignment = generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0]
+                .unwrap_leaf()
+                .to_simd_alignment();
+
+            let memflags = match alignment {
+                SimdAlign::Unaligned => MemFlags::new().with_notrap(),
+                _ => MemFlags::trusted(),
+            };
+
             for lane_idx in 0..val_lane_count {
                 let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
                 let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
@@ -972,7 +982,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
                 fx.bcx.switch_to_block(if_enabled);
                 let offset = lane_idx as i32 * lane_clif_ty.bytes() as i32;
-                fx.bcx.ins().store(MemFlags::trusted(), val_lane, ptr_val, Offset32::new(offset));
+                fx.bcx.ins().store(memflags, val_lane, ptr_val, Offset32::new(offset));
                 fx.bcx.ins().jump(next, &[]);
 
                 fx.bcx.seal_block(next);
@@ -996,6 +1006,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap();
             let ret_lane_layout = fx.layout_of(ret_lane_ty);
 
+            let alignment = generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0]
+                .unwrap_leaf()
+                .to_simd_alignment();
+
+            let memflags = match alignment {
+                SimdAlign::Unaligned => MemFlags::new().with_notrap(),
+                _ => MemFlags::trusted(),
+            };
+
             for lane_idx in 0..ptr_lane_count {
                 let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
                 let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx);
@@ -1011,7 +1030,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 fx.bcx.seal_block(if_disabled);
 
                 fx.bcx.switch_to_block(if_enabled);
-                let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), ptr_lane, 0);
+                let res = fx.bcx.ins().load(lane_clif_ty, memflags, ptr_lane, 0);
                 fx.bcx.ins().jump(next, &[res.into()]);
 
                 fx.bcx.switch_to_block(if_disabled);

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -13,7 +13,7 @@ use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_hir::{self as hir};
 use rustc_middle::mir::BinOp;
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
-use rustc_middle::ty::{self, GenericArgsRef, Instance, Ty, TyCtxt, TypingEnv};
+use rustc_middle::ty::{self, GenericArgsRef, Instance, SimdAlign, Ty, TyCtxt, TypingEnv};
 use rustc_middle::{bug, span_bug};
 use rustc_span::{Span, Symbol, sym};
 use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate};
@@ -1840,8 +1840,21 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         return Ok(call);
     }
 
+    fn llvm_alignment<'ll, 'tcx>(
+        bx: &mut Builder<'_, 'll, 'tcx>,
+        alignment: SimdAlign,
+        vector_ty: Ty<'tcx>,
+        element_ty: Ty<'tcx>,
+    ) -> u64 {
+        match alignment {
+            SimdAlign::Unaligned => 1,
+            SimdAlign::Element => bx.align_of(element_ty).bytes(),
+            SimdAlign::Vector => bx.align_of(vector_ty).bytes(),
+        }
+    }
+
     if name == sym::simd_masked_load {
-        // simd_masked_load(mask: <N x i{M}>, pointer: *_ T, values: <N x T>) -> <N x T>
+        // simd_masked_load<_, _, _, const ALIGN: SimdAlign>(mask: <N x i{M}>, pointer: *_ T, values: <N x T>) -> <N x T>
         // * N: number of elements in the input vectors
         // * T: type of the element to load
         // * M: any integer width is supported, will be truncated to i1
@@ -1849,6 +1862,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         // those lanes whose `mask` bit is enabled.
         // The memory addresses corresponding to the “off” lanes are not accessed.
 
+        let alignment = fn_args[3].expect_const().to_value().valtree.unwrap_branch()[0]
+            .unwrap_leaf()
+            .to_simd_alignment();
+
         // The element type of the "mask" argument must be a signed integer type of any width
         let mask_ty = in_ty;
         let (mask_len, mask_elem) = (in_len, in_elem);
@@ -1905,7 +1922,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
 
         // Alignment of T, must be a constant integer value:
-        let alignment = bx.align_of(values_elem).bytes();
+        let alignment = llvm_alignment(bx, alignment, values_ty, values_elem);
 
         let llvm_pointer = bx.type_ptr();
 
@@ -1932,7 +1949,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_masked_store {
-        // simd_masked_store(mask: <N x i{M}>, pointer: *mut T, values: <N x T>) -> ()
+        // simd_masked_store<_, _, _, const ALIGN: SimdAlign>(mask: <N x i{M}>, pointer: *mut T, values: <N x T>) -> ()
         // * N: number of elements in the input vectors
         // * T: type of the element to load
         // * M: any integer width is supported, will be truncated to i1
@@ -1940,6 +1957,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         // those lanes whose `mask` bit is enabled.
         // The memory addresses corresponding to the “off” lanes are not accessed.
 
+        let alignment = fn_args[3].expect_const().to_value().valtree.unwrap_branch()[0]
+            .unwrap_leaf()
+            .to_simd_alignment();
+
         // The element type of the "mask" argument must be a signed integer type of any width
         let mask_ty = in_ty;
         let (mask_len, mask_elem) = (in_len, in_elem);
@@ -1990,7 +2011,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
 
         // Alignment of T, must be a constant integer value:
-        let alignment = bx.align_of(values_elem).bytes();
+        let alignment = llvm_alignment(bx, alignment, values_ty, values_elem);
 
         let llvm_pointer = bx.type_ptr();
 

compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs

@@ -1,16 +1,16 @@
 use either::Either;
-use rustc_abi::Endian;
+use rustc_abi::{BackendRepr, Endian};
 use rustc_apfloat::ieee::{Double, Half, Quad, Single};
 use rustc_apfloat::{Float, Round};
-use rustc_middle::mir::interpret::{InterpErrorKind, UndefinedBehaviorInfo};
-use rustc_middle::ty::FloatTy;
+use rustc_middle::mir::interpret::{InterpErrorKind, Pointer, UndefinedBehaviorInfo};
+use rustc_middle::ty::{FloatTy, SimdAlign};
 use rustc_middle::{bug, err_ub_format, mir, span_bug, throw_unsup_format, ty};
 use rustc_span::{Symbol, sym};
 use tracing::trace;
 
 use super::{
     ImmTy, InterpCx, InterpResult, Machine, MinMax, MulAddType, OpTy, PlaceTy, Provenance, Scalar,
-    Size, interp_ok, throw_ub_format,
+    Size, TyAndLayout, assert_matches, interp_ok, throw_ub_format,
 };
 use crate::interpret::Writeable;
 
@@ -644,6 +644,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 }
             }
             sym::simd_masked_load => {
+                let dest_layout = dest.layout;
+
                 let (mask, mask_len) = self.project_to_simd(&args[0])?;
                 let ptr = self.read_pointer(&args[1])?;
                 let (default, default_len) = self.project_to_simd(&args[2])?;
@@ -652,6 +654,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 assert_eq!(dest_len, mask_len);
                 assert_eq!(dest_len, default_len);
 
+                self.check_simd_ptr_alignment(
+                    ptr,
+                    dest_layout,
+                    generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0]
+                        .unwrap_leaf()
+                        .to_simd_alignment(),
+                )?;
+
                 for i in 0..dest_len {
                     let mask = self.read_immediate(&self.project_index(&mask, i)?)?;
                     let default = self.read_immediate(&self.project_index(&default, i)?)?;
@@ -660,7 +670,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     let val = if simd_element_to_bool(mask)? {
                         // Size * u64 is implemented as always checked
                         let ptr = ptr.wrapping_offset(dest.layout.size * i, self);
-                        let place = self.ptr_to_mplace(ptr, dest.layout);
+                        // we have already checked the alignment requirements
+                        let place = self.ptr_to_mplace_unaligned(ptr, dest.layout);
                         self.read_immediate(&place)?
                     } else {
                         default
@@ -675,6 +686,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
 
                 assert_eq!(mask_len, vals_len);
 
+                self.check_simd_ptr_alignment(
+                    ptr,
+                    args[2].layout,
+                    generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0]
+                        .unwrap_leaf()
+                        .to_simd_alignment(),
+                )?;
+
                 for i in 0..vals_len {
                     let mask = self.read_immediate(&self.project_index(&mask, i)?)?;
                     let val = self.read_immediate(&self.project_index(&vals, i)?)?;
@@ -682,7 +701,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     if simd_element_to_bool(mask)? {
                         // Size * u64 is implemented as always checked
                         let ptr = ptr.wrapping_offset(val.layout.size * i, self);
-                        let place = self.ptr_to_mplace(ptr, val.layout);
+                        // we have already checked the alignment requirements
+                        let place = self.ptr_to_mplace_unaligned(ptr, val.layout);
                         self.write_immediate(*val, &place)?
                     };
                 }
@@ -753,6 +773,30 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             FloatTy::F128 => self.float_minmax::<Quad>(left, right, op)?,
         })
     }
+
+    fn check_simd_ptr_alignment(
+        &self,
+        ptr: Pointer<Option<M::Provenance>>,
+        vector_layout: TyAndLayout<'tcx>,
+        alignment: SimdAlign,
+    ) -> InterpResult<'tcx> {
+        assert_matches!(vector_layout.backend_repr, BackendRepr::SimdVector { .. });
+
+        let align = match alignment {
+            ty::SimdAlign::Unaligned => {
+                // The pointer is supposed to be unaligned, so no check is required.
+                return interp_ok(());
+            }
+            ty::SimdAlign::Element => {
+                // Take the alignment of the only field, which is an array and therefore has the same
+                // alignment as the element type.
+                vector_layout.field(self, 0).align.abi
+            }
+            ty::SimdAlign::Vector => vector_layout.align.abi,
+        };
+
+        self.check_ptr_align(ptr, align)
+    }
 }
 
 fn simd_bitmask_index(idx: u32, vec_len: u32, endianness: Endian) -> u32 {

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -695,8 +695,8 @@ pub(crate) fn check_intrinsic_type(
             (1, 0, vec![param(0), param(0), param(0)], param(0))
         }
         sym::simd_gather => (3, 0, vec![param(0), param(1), param(2)], param(0)),
-        sym::simd_masked_load => (3, 0, vec![param(0), param(1), param(2)], param(2)),
-        sym::simd_masked_store => (3, 0, vec![param(0), param(1), param(2)], tcx.types.unit),
+        sym::simd_masked_load => (3, 1, vec![param(0), param(1), param(2)], param(2)),
+        sym::simd_masked_store => (3, 1, vec![param(0), param(1), param(2)], tcx.types.unit),
         sym::simd_scatter => (3, 0, vec![param(0), param(1), param(2)], tcx.types.unit),
         sym::simd_insert | sym::simd_insert_dyn => {
             (2, 0, vec![param(0), tcx.types.u32, param(1)], param(0))

compiler/rustc_middle/src/ty/consts/int.rs

@@ -39,6 +39,15 @@ pub enum AtomicOrdering {
     SeqCst = 4,
 }
 
+/// An enum to represent the compiler-side view of `intrinsics::simd::SimdAlign`.
+#[derive(Debug, Copy, Clone)]
+pub enum SimdAlign {
+    // These values must match `intrinsics::simd::SimdAlign`!
+    Unaligned = 0,
+    Element = 1,
+    Vector = 2,
+}
+
 impl std::fmt::Debug for ConstInt {
     fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let Self { int, signed, is_ptr_sized_integral } = *self;
@@ -350,6 +359,21 @@ impl ScalarInt {
         }
     }
 
+    #[inline]
+    pub fn to_simd_alignment(self) -> SimdAlign {
+        use SimdAlign::*;
+        let val = self.to_u32();
+        if val == Unaligned as u32 {
+            Unaligned
+        } else if val == Element as u32 {
+            Element
+        } else if val == Vector as u32 {
+            Vector
+        } else {
+            panic!("not a valid simd alignment")
+        }
+    }
+
     /// Converts the `ScalarInt` to `bool`.
     /// Panics if the `size` of the `ScalarInt` is not equal to 1 byte.
     /// Errors if it is not a valid `bool`.

compiler/rustc_middle/src/ty/mod.rs

@@ -74,7 +74,7 @@ pub use self::closure::{
 };
 pub use self::consts::{
     AnonConstKind, AtomicOrdering, Const, ConstInt, ConstKind, ConstToValTreeResult, Expr,
-    ExprKind, ScalarInt, UnevaluatedConst, ValTree, ValTreeKind, Value,
+    ExprKind, ScalarInt, SimdAlign, UnevaluatedConst, ValTree, ValTreeKind, Value,
 };
 pub use self::context::{
     CtxtInterners, CurrentGcx, Feed, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt, TyCtxtFeed, tls,

library/core/src/intrinsics/simd.rs

@@ -2,6 +2,8 @@
 //!
 //! In this module, a "vector" is any `repr(simd)` type.
 
+use crate::marker::ConstParamTy;
+
 /// Inserts an element into a vector, returning the updated vector.
 ///
 /// `T` must be a vector with element type `U`, and `idx` must be `const`.
@@ -377,6 +379,19 @@ pub unsafe fn simd_gather<T, U, V>(val: T, ptr: U, mask: V) -> T;
 #[rustc_nounwind]
 pub unsafe fn simd_scatter<T, U, V>(val: T, ptr: U, mask: V);
 
+/// A type for alignment options for SIMD masked load/store intrinsics.
+#[derive(Debug, ConstParamTy, PartialEq, Eq)]
+pub enum SimdAlign {
+    // These values must match the compiler's `SimdAlign` defined in
+    // `rustc_middle/src/ty/consts/int.rs`!
+    /// No alignment requirements on the pointer
+    Unaligned = 0,
+    /// The pointer must be aligned to the element type of the SIMD vector
+    Element = 1,
+    /// The pointer must be aligned to the SIMD vector type
+    Vector = 2,
+}
+
 /// Reads a vector of pointers.
 ///
 /// `T` must be a vector.
@@ -392,13 +407,12 @@ pub unsafe fn simd_scatter<T, U, V>(val: T, ptr: U, mask: V);
 /// `val`.
 ///
 /// # Safety
-/// Unmasked values in `T` must be readable as if by `<ptr>::read` (e.g. aligned to the element
-/// type).
+/// `ptr` must be aligned according to the `ALIGN` parameter, see [`SimdAlign`] for details.
 ///
 /// `mask` must only contain `0` or `!0` values.
 #[rustc_intrinsic]
 #[rustc_nounwind]
-pub unsafe fn simd_masked_load<V, U, T>(mask: V, ptr: U, val: T) -> T;
+pub unsafe fn simd_masked_load<V, U, T, const ALIGN: SimdAlign>(mask: V, ptr: U, val: T) -> T;
 
 /// Writes to a vector of pointers.
 ///
@@ -414,13 +428,12 @@ pub unsafe fn simd_masked_load<V, U, T>(mask: V, ptr: U, val: T) -> T;
 /// Otherwise if the corresponding value in `mask` is `0`, do nothing.
 ///
 /// # Safety
-/// Unmasked values in `T` must be writeable as if by `<ptr>::write` (e.g. aligned to the element
-/// type).
+/// `ptr` must be aligned according to the `ALIGN` parameter, see [`SimdAlign`] for details.
 ///
 /// `mask` must only contain `0` or `!0` values.
 #[rustc_intrinsic]
 #[rustc_nounwind]
-pub unsafe fn simd_masked_store<V, U, T>(mask: V, ptr: U, val: T);
+pub unsafe fn simd_masked_store<V, U, T, const ALIGN: SimdAlign>(mask: V, ptr: U, val: T);
 
 /// Adds two simd vectors elementwise, with saturation.
 ///

library/portable-simd/crates/core_simd/src/vector.rs

@@ -474,7 +474,14 @@ where
         or: Self,
     ) -> Self {
         // SAFETY: The safety of reading elements through `ptr` is ensured by the caller.
-        unsafe { core::intrinsics::simd::simd_masked_load(enable.to_int(), ptr, or) }
+        unsafe {
+            core::intrinsics::simd::simd_masked_load::<
+                _,
+                _,
+                _,
+                { core::intrinsics::simd::SimdAlign::Element },
+            >(enable.to_int(), ptr, or)
+        }
     }
 
     /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
@@ -723,7 +730,14 @@ where
     #[inline]
     pub unsafe fn store_select_ptr(self, ptr: *mut T, enable: Mask<<T as SimdElement>::Mask, N>) {
         // SAFETY: The safety of writing elements through `ptr` is ensured by the caller.
-        unsafe { core::intrinsics::simd::simd_masked_store(enable.to_int(), ptr, self) }
+        unsafe {
+            core::intrinsics::simd::simd_masked_store::<
+                _,
+                _,
+                _,
+                { core::intrinsics::simd::SimdAlign::Element },
+            >(enable.to_int(), ptr, self)
+        }
     }
 
     /// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`.

src/tools/miri/tests/fail/intrinsics/simd_masked_load_element_misaligned.rs

@@ -0,0 +1,17 @@
+#![feature(core_intrinsics, portable_simd)]
+
+use std::intrinsics::simd::*;
+use std::simd::*;
+
+fn main() {
+    unsafe {
+        let buf = [0u32; 5];
+        //~v ERROR: accessing memory with alignment 
+        simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+            i32x4::splat(-1),
+            // This is not i32-aligned
+            buf.as_ptr().byte_offset(1),
+            i32x4::splat(0),
+        );
+    }
+}

src/tools/miri/tests/fail/intrinsics/simd_masked_load_element_misaligned.stderr

@@ -0,0 +1,18 @@
+error: Undefined Behavior: accessing memory with alignment ALIGN, but alignment ALIGN is required
+  --> tests/fail/intrinsics/simd_masked_load_element_misaligned.rs:LL:CC
+   |
+LL | /         simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+LL | |             i32x4::splat(-1),
+LL | |             // This is not i32-aligned
+LL | |             buf.as_ptr().byte_offset(1),
+LL | |             i32x4::splat(0),
+LL | |         );
+   | |_________^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+

src/tools/miri/tests/fail/intrinsics/simd_masked_load_vector_misaligned.rs

@@ -0,0 +1,17 @@
+#![feature(core_intrinsics, portable_simd)]
+
+use std::intrinsics::simd::*;
+use std::simd::*;
+
+fn main() {
+    unsafe {
+        let buf = Simd::<i32, 8>::splat(0);
+        //~v ERROR: accessing memory with alignment
+        simd_masked_load::<_, _, _, { SimdAlign::Vector }>(
+            i32x4::splat(-1),
+            // This is i32-aligned but not i32x4-aligned.
+            buf.as_array()[1..].as_ptr(),
+            i32x4::splat(0),
+        );
+    }
+}

src/tools/miri/tests/fail/intrinsics/simd_masked_load_vector_misaligned.stderr

@@ -0,0 +1,18 @@
+error: Undefined Behavior: accessing memory with alignment ALIGN, but alignment ALIGN is required
+  --> tests/fail/intrinsics/simd_masked_load_vector_misaligned.rs:LL:CC
+   |
+LL | /         simd_masked_load::<_, _, _, { SimdAlign::Vector }>(
+LL | |             i32x4::splat(-1),
+LL | |             // This is i32-aligned but not i32x4-aligned.
+LL | |             buf.as_array()[1..].as_ptr(),
+LL | |             i32x4::splat(0),
+LL | |         );
+   | |_________^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+

src/tools/miri/tests/fail/intrinsics/simd_masked_store_element_misaligned.rs

@@ -0,0 +1,17 @@
+#![feature(core_intrinsics, portable_simd)]
+
+use std::intrinsics::simd::*;
+use std::simd::*;
+
+fn main() {
+    unsafe {
+        let mut buf = [0u32; 5];
+        //~v ERROR: accessing memory with alignment
+        simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+            i32x4::splat(-1),
+            // This is not i32-aligned
+            buf.as_mut_ptr().byte_offset(1),
+            i32x4::splat(0),
+        );
+    }
+}

src/tools/miri/tests/fail/intrinsics/simd_masked_store_element_misaligned.stderr

@@ -0,0 +1,18 @@
+error: Undefined Behavior: accessing memory with alignment ALIGN, but alignment ALIGN is required
+  --> tests/fail/intrinsics/simd_masked_store_element_misaligned.rs:LL:CC
+   |
+LL | /         simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+LL | |             i32x4::splat(-1),
+LL | |             // This is not i32-aligned
+LL | |             buf.as_mut_ptr().byte_offset(1),
+LL | |             i32x4::splat(0),
+LL | |         );
+   | |_________^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+

src/tools/miri/tests/fail/intrinsics/simd_masked_store_vector_misaligned.rs

@@ -0,0 +1,17 @@
+#![feature(core_intrinsics, portable_simd)]
+
+use std::intrinsics::simd::*;
+use std::simd::*;
+
+fn main() {
+    unsafe {
+        let mut buf = Simd::<i32, 8>::splat(0);
+        //~v ERROR: accessing memory with alignment
+        simd_masked_store::<_, _, _, { SimdAlign::Vector }>(
+            i32x4::splat(-1),
+            // This is i32-aligned but not i32x4-aligned.
+            buf.as_mut_array()[1..].as_mut_ptr(),
+            i32x4::splat(0),
+        );
+    }
+}

src/tools/miri/tests/fail/intrinsics/simd_masked_store_vector_misaligned.stderr

@@ -0,0 +1,18 @@
+error: Undefined Behavior: accessing memory with alignment ALIGN, but alignment ALIGN is required
+  --> tests/fail/intrinsics/simd_masked_store_vector_misaligned.rs:LL:CC
+   |
+LL | /         simd_masked_store::<_, _, _, { SimdAlign::Vector }>(
+LL | |             i32x4::splat(-1),
+LL | |             // This is i32-aligned but not i32x4-aligned.
+LL | |             buf.as_mut_array()[1..].as_mut_ptr(),
+LL | |             i32x4::splat(0),
+LL | |         );
+   | |_________^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+

src/tools/miri/tests/pass/intrinsics/portable-simd.rs

@@ -936,26 +936,93 @@ fn simd_float_intrinsics() {
 }
 
 fn simd_masked_loadstore() {
+    use intrinsics::*;
+
     // The buffer is deliberarely too short, so reading the last element would be UB.
     let buf = [3i32; 3];
     let default = i32x4::splat(0);
     let mask = i32x4::from_array([!0, !0, !0, 0]);
-    let vals = unsafe { intrinsics::simd_masked_load(mask, buf.as_ptr(), default) };
+    let vals =
+        unsafe { simd_masked_load::<_, _, _, { SimdAlign::Element }>(mask, buf.as_ptr(), default) };
     assert_eq!(vals, i32x4::from_array([3, 3, 3, 0]));
     // Also read in a way that the *first* element is OOB.
     let mask2 = i32x4::from_array([0, !0, !0, !0]);
-    let vals =
-        unsafe { intrinsics::simd_masked_load(mask2, buf.as_ptr().wrapping_sub(1), default) };
+    let vals = unsafe {
+        simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+            mask2,
+            buf.as_ptr().wrapping_sub(1),
+            default,
+        )
+    };
     assert_eq!(vals, i32x4::from_array([0, 3, 3, 3]));
 
     // The buffer is deliberarely too short, so writing the last element would be UB.
     let mut buf = [42i32; 3];
     let vals = i32x4::from_array([1, 2, 3, 4]);
-    unsafe { intrinsics::simd_masked_store(mask, buf.as_mut_ptr(), vals) };
+    unsafe { simd_masked_store::<_, _, _, { SimdAlign::Element }>(mask, buf.as_mut_ptr(), vals) };
     assert_eq!(buf, [1, 2, 3]);
     // Also write in a way that the *first* element is OOB.
-    unsafe { intrinsics::simd_masked_store(mask2, buf.as_mut_ptr().wrapping_sub(1), vals) };
+    unsafe {
+        simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+            mask2,
+            buf.as_mut_ptr().wrapping_sub(1),
+            vals,
+        )
+    };
     assert_eq!(buf, [2, 3, 4]);
+
+    // we use a purposely misaliged buffer to make sure Miri doesn't error in this case
+    let buf = [0x03030303_i32; 5];
+    let default = i32x4::splat(0);
+    let mask = i32x4::splat(!0);
+    let vals = unsafe {
+        simd_masked_load::<_, _, _, { SimdAlign::Unaligned }>(
+            mask,
+            buf.as_ptr().byte_offset(1), // this is guaranteed to be unaligned
+            default,
+        )
+    };
+    assert_eq!(vals, i32x4::splat(0x03030303));
+
+    let mut buf = [0i32; 5];
+    let mask = i32x4::splat(!0);
+    unsafe {
+        simd_masked_store::<_, _, _, { SimdAlign::Unaligned }>(
+            mask,
+            buf.as_mut_ptr().byte_offset(1), // this is guaranteed to be unaligned
+            vals,
+        )
+    };
+    assert_eq!(
+        buf,
+        [
+            i32::from_ne_bytes([0, 3, 3, 3]),
+            0x03030303,
+            0x03030303,
+            0x03030303,
+            i32::from_ne_bytes([3, 0, 0, 0])
+        ]
+    );
+
+    // `repr(simd)` types like `Simd<T, N>` have the correct alignment for vectors
+    let buf = i32x4::splat(3);
+    let default = i32x4::splat(0);
+    let mask = i32x4::splat(!0);
+    let vals = unsafe {
+        simd_masked_load::<_, _, _, { SimdAlign::Vector }>(
+            mask,
+            &raw const buf as *const i32,
+            default,
+        )
+    };
+    assert_eq!(vals, buf);
+
+    let mut buf = i32x4::splat(0);
+    let mask = i32x4::splat(!0);
+    unsafe {
+        simd_masked_store::<_, _, _, { SimdAlign::Vector }>(mask, &raw mut buf as *mut i32, vals)
+    };
+    assert_eq!(buf, vals);
 }
 
 fn simd_ops_non_pow2() {

tests/assembly-llvm/simd-intrinsic-mask-load.rs

@@ -9,7 +9,7 @@
 //@ assembly-output: emit-asm
 //@ compile-flags: --crate-type=lib -Copt-level=3 -C panic=abort
 
-#![feature(no_core, lang_items, repr_simd, intrinsics)]
+#![feature(no_core, lang_items, repr_simd, intrinsics, adt_const_params)]
 #![no_core]
 #![allow(non_camel_case_types)]
 
@@ -35,7 +35,7 @@ pub struct f64x4([f64; 4]);
 pub struct m64x4([i64; 4]);
 
 #[rustc_intrinsic]
-unsafe fn simd_masked_load<M, P, T>(mask: M, pointer: P, values: T) -> T;
+unsafe fn simd_masked_load<M, P, T, const ALIGN: SimdAlign>(mask: M, pointer: P, values: T) -> T;
 
 // CHECK-LABEL: load_i8x16
 #[no_mangle]
@@ -56,7 +56,11 @@ pub unsafe extern "C" fn load_i8x16(mask: m8x16, pointer: *const i8) -> i8x16 {
     // x86-avx512-NOT: vpsllw
     // x86-avx512: vpmovb2m k1, xmm0
     // x86-avx512-NEXT: vmovdqu8 xmm0 {k1} {z}, xmmword ptr [rdi]
-    simd_masked_load(mask, pointer, i8x16([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]))
+    simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+        mask,
+        pointer,
+        i8x16([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+    )
 }
 
 // CHECK-LABEL: load_f32x8
@@ -68,7 +72,29 @@ pub unsafe extern "C" fn load_f32x8(mask: m32x8, pointer: *const f32) -> f32x8 {
     // x86-avx512-NOT: vpslld
     // x86-avx512: vpmovd2m k1, ymm0
     // x86-avx512-NEXT: vmovups ymm0 {k1} {z}, ymmword ptr [rdi]
-    simd_masked_load(mask, pointer, f32x8([0_f32, 0_f32, 0_f32, 0_f32, 0_f32, 0_f32, 0_f32, 0_f32]))
+    simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+        mask,
+        pointer,
+        f32x8([0_f32, 0_f32, 0_f32, 0_f32, 0_f32, 0_f32, 0_f32, 0_f32]),
+    )
+}
+
+// CHECK-LABEL: load_f32x8_aligned
+#[no_mangle]
+pub unsafe extern "C" fn load_f32x8_aligned(mask: m32x8, pointer: *const f32) -> f32x8 {
+    // x86-avx2-NOT: vpslld
+    // x86-avx2: vmaskmovps ymm0, ymm0, ymmword ptr [rdi]
+    //
+    // x86-avx512-NOT: vpslld
+    // x86-avx512: vpmovd2m k1, ymm0
+    // x86-avx512-NEXT: vmovaps ymm0 {k1} {z}, ymmword ptr [rdi]
+    //
+    // this aligned version should generate `movaps` instead of `movups`
+    simd_masked_load::<_, _, _, { SimdAlign::Vector }>(
+        mask,
+        pointer,
+        f32x8([0_f32, 0_f32, 0_f32, 0_f32, 0_f32, 0_f32, 0_f32, 0_f32]),
+    )
 }
 
 // CHECK-LABEL: load_f64x4
@@ -79,5 +105,9 @@ pub unsafe extern "C" fn load_f64x4(mask: m64x4, pointer: *const f64) -> f64x4 {
     //
     // x86-avx512-NOT: vpsllq
     // x86-avx512: vpmovq2m k1, ymm0
-    simd_masked_load(mask, pointer, f64x4([0_f64, 0_f64, 0_f64, 0_f64]))
+    simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+        mask,
+        pointer,
+        f64x4([0_f64, 0_f64, 0_f64, 0_f64]),
+    )
 }

tests/assembly-llvm/simd-intrinsic-mask-store.rs

@@ -9,7 +9,7 @@
 //@ assembly-output: emit-asm
 //@ compile-flags: --crate-type=lib -Copt-level=3 -C panic=abort
 
-#![feature(no_core, lang_items, repr_simd, intrinsics)]
+#![feature(no_core, lang_items, repr_simd, intrinsics, adt_const_params)]
 #![no_core]
 #![allow(non_camel_case_types)]
 
@@ -35,7 +35,7 @@ pub struct f64x4([f64; 4]);
 pub struct m64x4([i64; 4]);
 
 #[rustc_intrinsic]
-unsafe fn simd_masked_store<M, P, T>(mask: M, pointer: P, values: T);
+unsafe fn simd_masked_store<M, P, T, const ALIGN: SimdAlign>(mask: M, pointer: P, values: T);
 
 // CHECK-LABEL: store_i8x16
 #[no_mangle]
@@ -54,7 +54,7 @@ pub unsafe extern "C" fn store_i8x16(mask: m8x16, pointer: *mut i8, value: i8x16
     // x86-avx512-NOT: vpsllw
     // x86-avx512: vpmovb2m k1, xmm0
     // x86-avx512-NEXT: vmovdqu8 xmmword ptr [rdi] {k1}, xmm1
-    simd_masked_store(mask, pointer, value)
+    simd_masked_store::<_, _, _, { SimdAlign::Element }>(mask, pointer, value)
 }
 
 // CHECK-LABEL: store_f32x8
@@ -66,7 +66,21 @@ pub unsafe extern "C" fn store_f32x8(mask: m32x8, pointer: *mut f32, value: f32x
     // x86-avx512-NOT: vpslld
     // x86-avx512: vpmovd2m k1, ymm0
     // x86-avx512-NEXT: vmovups ymmword ptr [rdi] {k1}, ymm1
-    simd_masked_store(mask, pointer, value)
+    simd_masked_store::<_, _, _, { SimdAlign::Element }>(mask, pointer, value)
+}
+
+// CHECK-LABEL: store_f32x8_aligned
+#[no_mangle]
+pub unsafe extern "C" fn store_f32x8_aligned(mask: m32x8, pointer: *mut f32, value: f32x8) {
+    // x86-avx2-NOT: vpslld
+    // x86-avx2: vmaskmovps ymmword ptr [rdi], ymm0, ymm1
+    //
+    // x86-avx512-NOT: vpslld
+    // x86-avx512: vpmovd2m k1, ymm0
+    // x86-avx512-NEXT: vmovaps ymmword ptr [rdi] {k1}, ymm1
+    //
+    // this aligned version should generate `movaps` instead of `movups`
+    simd_masked_store::<_, _, _, { SimdAlign::Vector }>(mask, pointer, value)
 }
 
 // CHECK-LABEL: store_f64x4
@@ -78,5 +92,5 @@ pub unsafe extern "C" fn store_f64x4(mask: m64x4, pointer: *mut f64, value: f64x
     // x86-avx512-NOT: vpsllq
     // x86-avx512: vpmovq2m k1, ymm0
     // x86-avx512-NEXT: vmovupd ymmword ptr [rdi] {k1}, ymm1
-    simd_masked_store(mask, pointer, value)
+    simd_masked_store::<_, _, _, { SimdAlign::Element }>(mask, pointer, value)
 }

tests/auxiliary/minicore.rs

@@ -261,3 +261,17 @@ pub enum c_void {
     __variant1,
     __variant2,
 }
+
+#[lang = "const_param_ty"]
+#[diagnostic::on_unimplemented(message = "`{Self}` can't be used as a const parameter type")]
+pub trait ConstParamTy_ {}
+
+pub enum SimdAlign {
+    // These values must match the compiler's `SimdAlign` defined in
+    // `rustc_middle/src/ty/consts/int.rs`!
+    Unaligned = 0,
+    Element = 1,
+    Vector = 2,
+}
+
+impl ConstParamTy_ for SimdAlign {}

tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-load.rs

@@ -12,7 +12,7 @@
 mod minisimd;
 use minisimd::*;
 
-use std::intrinsics::simd::simd_masked_load;
+use std::intrinsics::simd::{SimdAlign, simd_masked_load};
 
 pub type Vec2<T> = Simd<T, 2>;
 pub type Vec4<T> = Simd<T, 4>;
@@ -23,8 +23,45 @@ pub unsafe fn load_f32x2(mask: Vec2<i32>, pointer: *const f32, values: Vec2<f32>
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
     // LLVM21: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 4, <2 x i1> [[B]], <2 x float> {{.*}})
+    //                                                                 ^^^^^
     // LLVM22: call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 4 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
-    simd_masked_load(mask, pointer, values)
+    //                                                         ^^^^^^^
+    // the align parameter should be equal to the alignment of the element type (assumed to be 4)
+    simd_masked_load::<_, _, _, { SimdAlign::Element }>(mask, pointer, values)
+}
+
+// CHECK-LABEL: @load_f32x2_aligned
+#[no_mangle]
+pub unsafe fn load_f32x2_aligned(
+    mask: Vec2<i32>,
+    pointer: *const f32,
+    values: Vec2<f32>,
+) -> Vec2<f32> {
+    // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
+    // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
+    // LLVM21: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 8, <2 x i1> [[B]], <2 x float> {{.*}})
+    //                                                                 ^^^^^
+    // LLVM22: call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 8 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
+    //                                                         ^^^^^^^
+    // the align parameter should be equal to the size of the vector
+    simd_masked_load::<_, _, _, { SimdAlign::Vector }>(mask, pointer, values)
+}
+
+// CHECK-LABEL: @load_f32x2_unaligned
+#[no_mangle]
+pub unsafe fn load_f32x2_unaligned(
+    mask: Vec2<i32>,
+    pointer: *const f32,
+    values: Vec2<f32>,
+) -> Vec2<f32> {
+    // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
+    // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
+    // LLVM21: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 1, <2 x i1> [[B]], <2 x float> {{.*}})
+    //                                                                 ^^^^^
+    // LLVM22: call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 1 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
+    //                                                         ^^^^^^^
+    // the align parameter should be 1
+    simd_masked_load::<_, _, _, { SimdAlign::Unaligned }>(mask, pointer, values)
 }
 
 // CHECK-LABEL: @load_f32x2_unsigned
@@ -38,7 +75,7 @@ pub unsafe fn load_f32x2_unsigned(
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
     // LLVM21: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 4, <2 x i1> [[B]], <2 x float> {{.*}})
     // LLVM22: call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 4 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
-    simd_masked_load(mask, pointer, values)
+    simd_masked_load::<_, _, _, { SimdAlign::Element }>(mask, pointer, values)
 }
 
 // CHECK-LABEL: @load_pf32x4
@@ -52,5 +89,5 @@ pub unsafe fn load_pf32x4(
     // CHECK: [[B:%[0-9]+]] = trunc <4 x i32> [[A]] to <4 x i1>
     // LLVM21: call <4 x ptr> @llvm.masked.load.v4p0.p0(ptr {{.*}}, i32 {{.*}}, <4 x i1> [[B]], <4 x ptr> {{.*}})
     // LLVM22: call <4 x ptr> @llvm.masked.load.v4p0.p0(ptr align {{.*}} {{.*}}, <4 x i1> [[B]], <4 x ptr> {{.*}})
-    simd_masked_load(mask, pointer, values)
+    simd_masked_load::<_, _, _, { SimdAlign::Element }>(mask, pointer, values)
 }

tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-store.rs

@@ -12,7 +12,7 @@
 mod minisimd;
 use minisimd::*;
 
-use std::intrinsics::simd::simd_masked_store;
+use std::intrinsics::simd::{SimdAlign, simd_masked_store};
 
 pub type Vec2<T> = Simd<T, 2>;
 pub type Vec4<T> = Simd<T, 4>;
@@ -23,8 +23,37 @@ pub unsafe fn store_f32x2(mask: Vec2<i32>, pointer: *mut f32, values: Vec2<f32>)
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
     // LLVM21: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 4, <2 x i1> [[B]])
+    //                                                                               ^^^^^
     // LLVM22: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr align 4 {{.*}}, <2 x i1> [[B]])
-    simd_masked_store(mask, pointer, values)
+    //                                                                       ^^^^^^^
+    // the align parameter should be equal to the alignment of the element type (assumed to be 4)
+    simd_masked_store::<_, _, _, { SimdAlign::Element }>(mask, pointer, values)
+}
+
+// CHECK-LABEL: @store_f32x2_aligned
+#[no_mangle]
+pub unsafe fn store_f32x2_aligned(mask: Vec2<i32>, pointer: *mut f32, values: Vec2<f32>) {
+    // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
+    // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
+    // LLVM21: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 8, <2 x i1> [[B]])
+    //                                                                               ^^^^^
+    // LLVM22: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr align 8 {{.*}}, <2 x i1> [[B]])
+    //                                                                       ^^^^^^^
+    // the align parameter should be equal to the size of the vector
+    simd_masked_store::<_, _, _, { SimdAlign::Vector }>(mask, pointer, values)
+}
+
+// CHECK-LABEL: @store_f32x2_unaligned
+#[no_mangle]
+pub unsafe fn store_f32x2_unaligned(mask: Vec2<i32>, pointer: *mut f32, values: Vec2<f32>) {
+    // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
+    // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
+    // LLVM21: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 1, <2 x i1> [[B]])
+    //                                                                               ^^^^^
+    // LLVM22: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr align 1 {{.*}}, <2 x i1> [[B]])
+    //                                                                       ^^^^^^^
+    // the align parameter should be 1
+    simd_masked_store::<_, _, _, { SimdAlign::Unaligned }>(mask, pointer, values)
 }
 
 // CHECK-LABEL: @store_f32x2_unsigned
@@ -34,7 +63,7 @@ pub unsafe fn store_f32x2_unsigned(mask: Vec2<u32>, pointer: *mut f32, values: V
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
     // LLVM21: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 4, <2 x i1> [[B]])
     // LLVM22: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr align 4 {{.*}}, <2 x i1> [[B]])
-    simd_masked_store(mask, pointer, values)
+    simd_masked_store::<_, _, _, { SimdAlign::Element }>(mask, pointer, values)
 }
 
 // CHECK-LABEL: @store_pf32x4
@@ -44,5 +73,5 @@ pub unsafe fn store_pf32x4(mask: Vec4<i32>, pointer: *mut *const f32, values: Ve
     // CHECK: [[B:%[0-9]+]] = trunc <4 x i32> [[A]] to <4 x i1>
     // LLVM21: call void @llvm.masked.store.v4p0.p0(<4 x ptr> {{.*}}, ptr {{.*}}, i32 {{.*}}, <4 x i1> [[B]])
     // LLVM22: call void @llvm.masked.store.v4p0.p0(<4 x ptr> {{.*}}, ptr align {{.*}} {{.*}}, <4 x i1> [[B]])
-    simd_masked_store(mask, pointer, values)
+    simd_masked_store::<_, _, _, { SimdAlign::Element }>(mask, pointer, values)
 }

tests/ui/simd/masked-load-store-build-fail.rs

@@ -2,7 +2,7 @@
 //@ ignore-backends: gcc
 #![feature(repr_simd, core_intrinsics)]
 
-use std::intrinsics::simd::{simd_masked_load, simd_masked_store};
+use std::intrinsics::simd::{SimdAlign, simd_masked_load, simd_masked_store};
 
 #[derive(Copy, Clone)]
 #[repr(simd)]
@@ -13,28 +13,60 @@ fn main() {
         let mut arr = [4u8, 5, 6, 7];
         let default = Simd::<u8, 4>([9; 4]);
 
-        simd_masked_load(Simd::<i8, 8>([-1, 0, -1, -1, 0, 0, 0, 0]), arr.as_ptr(), default);
-        //~^ ERROR expected third argument with length 8 (same as input type `Simd<i8, 8>`), found `Simd<u8, 4>` with length 4
+        //~v ERROR expected third argument with length 8 (same as input type `Simd<i8, 8>`), found `Simd<u8, 4>` with length 4
+        simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+            Simd::<i8, 8>([-1, 0, -1, -1, 0, 0, 0, 0]),
+            arr.as_ptr(),
+            default,
+        );
 
-        simd_masked_load(Simd::<i8, 4>([-1, 0, -1, -1]), arr.as_ptr() as *const i8, default);
-        //~^ ERROR expected element type `u8` of second argument `*const i8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*_ u8`
+        //~v ERROR expected element type `u8` of second argument `*const i8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*_ u8`
+        simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+            Simd::<i8, 4>([-1, 0, -1, -1]),
+            arr.as_ptr() as *const i8,
+            default,
+        );
 
-        simd_masked_load(Simd::<i8, 4>([-1, 0, -1, -1]), arr.as_ptr(), Simd::<u32, 4>([9; 4]));
-        //~^ ERROR expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*_ u32`
+        //~v ERROR expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*_ u32`
+        simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+            Simd::<i8, 4>([-1, 0, -1, -1]),
+            arr.as_ptr(),
+            Simd::<u32, 4>([9; 4]),
+        );
 
-        simd_masked_load(Simd::<f32, 4>([1.0, 0.0, 1.0, 1.0]), arr.as_ptr(), default);
-        //~^ ERROR expected mask element type to be an integer, found `f32`
+        //~v ERROR expected mask element type to be an integer, found `f32`
+        simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+            Simd::<f32, 4>([1.0, 0.0, 1.0, 1.0]),
+            arr.as_ptr(),
+            default,
+        );
 
-        simd_masked_store(Simd([-1i8; 4]), arr.as_ptr(), Simd([5u32; 4]));
-        //~^ ERROR expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*mut u32`
+        //~v ERROR expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*mut u32`
+        simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+            Simd([-1i8; 4]),
+            arr.as_ptr(),
+            Simd([5u32; 4]),
+        );
 
-        simd_masked_store(Simd([-1i8; 4]), arr.as_ptr(), Simd([5u8; 4]));
-        //~^ ERROR expected element type `u8` of second argument `*const u8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*mut u8`
+        //~v ERROR expected element type `u8` of second argument `*const u8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*mut u8`
+        simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+            Simd([-1i8; 4]),
+            arr.as_ptr(),
+            Simd([5u8; 4]),
+        );
 
-        simd_masked_store(Simd([-1i8; 4]), arr.as_mut_ptr(), Simd([5u8; 2]));
-        //~^ ERROR expected third argument with length 4 (same as input type `Simd<i8, 4>`), found `Simd<u8, 2>` with length 2
+        //~v ERROR expected third argument with length 4 (same as input type `Simd<i8, 4>`), found `Simd<u8, 2>` with length 2
+        simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+            Simd([-1i8; 4]),
+            arr.as_mut_ptr(),
+            Simd([5u8; 2]),
+        );
 
-        simd_masked_store(Simd([1f32; 4]), arr.as_mut_ptr(), Simd([5u8; 4]));
-        //~^ ERROR expected mask element type to be an integer, found `f32`
+        //~v ERROR expected mask element type to be an integer, found `f32`
+        simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+            Simd([1f32; 4]),
+            arr.as_mut_ptr(),
+            Simd([5u8; 4]),
+        );
     }
 }

tests/ui/simd/masked-load-store-build-fail.stderr

@@ -1,50 +1,82 @@
 error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected third argument with length 8 (same as input type `Simd<i8, 8>`), found `Simd<u8, 4>` with length 4
-  --> $DIR/masked-load-store-build-fail.rs:16:9
+  --> $DIR/masked-load-store-build-fail.rs:17:9
    |
-LL |         simd_masked_load(Simd::<i8, 8>([-1, 0, -1, -1, 0, 0, 0, 0]), arr.as_ptr(), default);
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL | /         simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+LL | |             Simd::<i8, 8>([-1, 0, -1, -1, 0, 0, 0, 0]),
+LL | |             arr.as_ptr(),
+LL | |             default,
+LL | |         );
+   | |_________^
 
 error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected element type `u8` of second argument `*const i8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*_ u8`
-  --> $DIR/masked-load-store-build-fail.rs:19:9
+  --> $DIR/masked-load-store-build-fail.rs:24:9
    |
-LL |         simd_masked_load(Simd::<i8, 4>([-1, 0, -1, -1]), arr.as_ptr() as *const i8, default);
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL | /         simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+LL | |             Simd::<i8, 4>([-1, 0, -1, -1]),
+LL | |             arr.as_ptr() as *const i8,
+LL | |             default,
+LL | |         );
+   | |_________^
 
 error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*_ u32`
-  --> $DIR/masked-load-store-build-fail.rs:22:9
-   |
-LL |         simd_masked_load(Simd::<i8, 4>([-1, 0, -1, -1]), arr.as_ptr(), Simd::<u32, 4>([9; 4]));
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected mask element type to be an integer, found `f32`
-  --> $DIR/masked-load-store-build-fail.rs:25:9
-   |
-LL |         simd_masked_load(Simd::<f32, 4>([1.0, 0.0, 1.0, 1.0]), arr.as_ptr(), default);
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*mut u32`
-  --> $DIR/masked-load-store-build-fail.rs:28:9
-   |
-LL |         simd_masked_store(Simd([-1i8; 4]), arr.as_ptr(), Simd([5u32; 4]));
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u8` of second argument `*const u8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*mut u8`
   --> $DIR/masked-load-store-build-fail.rs:31:9
    |
-LL |         simd_masked_store(Simd([-1i8; 4]), arr.as_ptr(), Simd([5u8; 4]));
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL | /         simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+LL | |             Simd::<i8, 4>([-1, 0, -1, -1]),
+LL | |             arr.as_ptr(),
+LL | |             Simd::<u32, 4>([9; 4]),
+LL | |         );
+   | |_________^
+
+error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected mask element type to be an integer, found `f32`
+  --> $DIR/masked-load-store-build-fail.rs:38:9
+   |
+LL | /         simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+LL | |             Simd::<f32, 4>([1.0, 0.0, 1.0, 1.0]),
+LL | |             arr.as_ptr(),
+LL | |             default,
+LL | |         );
+   | |_________^
+
+error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*mut u32`
+  --> $DIR/masked-load-store-build-fail.rs:45:9
+   |
+LL | /         simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+LL | |             Simd([-1i8; 4]),
+LL | |             arr.as_ptr(),
+LL | |             Simd([5u32; 4]),
+LL | |         );
+   | |_________^
+
+error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u8` of second argument `*const u8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*mut u8`
+  --> $DIR/masked-load-store-build-fail.rs:52:9
+   |
+LL | /         simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+LL | |             Simd([-1i8; 4]),
+LL | |             arr.as_ptr(),
+LL | |             Simd([5u8; 4]),
+LL | |         );
+   | |_________^
 
 error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected third argument with length 4 (same as input type `Simd<i8, 4>`), found `Simd<u8, 2>` with length 2
-  --> $DIR/masked-load-store-build-fail.rs:34:9
+  --> $DIR/masked-load-store-build-fail.rs:59:9
    |
-LL |         simd_masked_store(Simd([-1i8; 4]), arr.as_mut_ptr(), Simd([5u8; 2]));
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL | /         simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+LL | |             Simd([-1i8; 4]),
+LL | |             arr.as_mut_ptr(),
+LL | |             Simd([5u8; 2]),
+LL | |         );
+   | |_________^
 
 error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected mask element type to be an integer, found `f32`
-  --> $DIR/masked-load-store-build-fail.rs:37:9
+  --> $DIR/masked-load-store-build-fail.rs:66:9
    |
-LL |         simd_masked_store(Simd([1f32; 4]), arr.as_mut_ptr(), Simd([5u8; 4]));
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL | /         simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+LL | |             Simd([1f32; 4]),
+LL | |             arr.as_mut_ptr(),
+LL | |             Simd([5u8; 4]),
+LL | |         );
+   | |_________^
 
 error: aborting due to 8 previous errors
 

tests/ui/simd/masked-load-store-check-fail.rs

@@ -1,7 +1,7 @@
 //@ check-fail
 #![feature(repr_simd, core_intrinsics)]
 
-use std::intrinsics::simd::{simd_masked_load, simd_masked_store};
+use std::intrinsics::simd::{SimdAlign, simd_masked_load, simd_masked_store};
 
 #[derive(Copy, Clone)]
 #[repr(simd)]
@@ -12,11 +12,18 @@ fn main() {
         let mut arr = [4u8, 5, 6, 7];
         let default = Simd::<u8, 4>([9; 4]);
 
-        let _x: Simd<u8, 2> =
-            simd_masked_load(Simd::<i8, 4>([-1, 0, -1, -1]), arr.as_ptr(), Simd::<u8, 4>([9; 4]));
-        //~^ ERROR mismatched types
+        let _x: Simd<u8, 2> = simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+            Simd::<i8, 4>([-1, 0, -1, -1]),
+            arr.as_ptr(),
+            Simd::<u8, 4>([9; 4]),
+        );
+        //~^^ ERROR mismatched types
 
-        let _x: Simd<u32, 4> = simd_masked_load(Simd::<u8, 4>([1, 0, 1, 1]), arr.as_ptr(), default);
-        //~^ ERROR mismatched types
+        let _x: Simd<u32, 4> = simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+            Simd::<u8, 4>([1, 0, 1, 1]),
+            arr.as_ptr(),
+            default,
+        );
+        //~^^ ERROR mismatched types
     }
 }

tests/ui/simd/masked-load-store-check-fail.stderr

@@ -1,36 +1,50 @@
 error[E0308]: mismatched types
-  --> $DIR/masked-load-store-check-fail.rs:16:76
+  --> $DIR/masked-load-store-check-fail.rs:18:13
    |
-LL |             simd_masked_load(Simd::<i8, 4>([-1, 0, -1, -1]), arr.as_ptr(), Simd::<u8, 4>([9; 4]));
-   |             ---------------- arguments to this function are incorrect      ^^^^^^^^^^^^^^^^^^^^^ expected `2`, found `4`
+LL |         let _x: Simd<u8, 2> = simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+   |                               --------------------------------------------------- arguments to this function are incorrect
+...
+LL |             Simd::<u8, 4>([9; 4]),
+   |             ^^^^^^^^^^^^^^^^^^^^^ expected `2`, found `4`
    |
    = note: expected struct `Simd<_, 2>`
               found struct `Simd<_, 4>`
 help: the return type of this call is `Simd<u8, 4>` due to the type of the argument passed
-  --> $DIR/masked-load-store-check-fail.rs:16:13
+  --> $DIR/masked-load-store-check-fail.rs:15:31
    |
-LL |             simd_masked_load(Simd::<i8, 4>([-1, 0, -1, -1]), arr.as_ptr(), Simd::<u8, 4>([9; 4]));
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^---------------------^
-   |                                                                            |
-   |                                                                            this argument influences the return type of `simd_masked_load`
+LL |           let _x: Simd<u8, 2> = simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+   |  _______________________________^
+LL | |             Simd::<i8, 4>([-1, 0, -1, -1]),
+LL | |             arr.as_ptr(),
+LL | |             Simd::<u8, 4>([9; 4]),
+   | |             --------------------- this argument influences the return type of `simd_masked_load`
+LL | |         );
+   | |_________^
 note: function defined here
   --> $SRC_DIR/core/src/intrinsics/simd.rs:LL:COL
 
 error[E0308]: mismatched types
-  --> $DIR/masked-load-store-check-fail.rs:19:92
+  --> $DIR/masked-load-store-check-fail.rs:25:13
    |
-LL |         let _x: Simd<u32, 4> = simd_masked_load(Simd::<u8, 4>([1, 0, 1, 1]), arr.as_ptr(), default);
-   |                                ---------------- arguments to this function are incorrect   ^^^^^^^ expected `Simd<u32, 4>`, found `Simd<u8, 4>`
+LL |         let _x: Simd<u32, 4> = simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+   |                                --------------------------------------------------- arguments to this function are incorrect
+...
+LL |             default,
+   |             ^^^^^^^ expected `Simd<u32, 4>`, found `Simd<u8, 4>`
    |
    = note: expected struct `Simd<u32, _>`
               found struct `Simd<u8, _>`
 help: the return type of this call is `Simd<u8, 4>` due to the type of the argument passed
-  --> $DIR/masked-load-store-check-fail.rs:19:32
+  --> $DIR/masked-load-store-check-fail.rs:22:32
    |
-LL |         let _x: Simd<u32, 4> = simd_masked_load(Simd::<u8, 4>([1, 0, 1, 1]), arr.as_ptr(), default);
-   |                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-------^
-   |                                                                                            |
-   |                                                                                            this argument influences the return type of `simd_masked_load`
+LL |           let _x: Simd<u32, 4> = simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+   |  ________________________________^
+LL | |             Simd::<u8, 4>([1, 0, 1, 1]),
+LL | |             arr.as_ptr(),
+LL | |             default,
+   | |             ------- this argument influences the return type of `simd_masked_load`
+LL | |         );
+   | |_________^
 note: function defined here
   --> $SRC_DIR/core/src/intrinsics/simd.rs:LL:COL
 

tests/ui/simd/masked-load-store.rs

@@ -6,23 +6,34 @@
 mod minisimd;
 use minisimd::*;
 
-use std::intrinsics::simd::{simd_masked_load, simd_masked_store};
+use std::intrinsics::simd::{SimdAlign, simd_masked_load, simd_masked_store};
 
 fn main() {
     unsafe {
         let a = Simd::<u8, 4>([0, 1, 2, 3]);
         let b_src = [4u8, 5, 6, 7];
         let b_default = Simd::<u8, 4>([9; 4]);
-        let b: Simd<u8, 4> =
-            simd_masked_load(Simd::<i8, 4>([-1, 0, -1, -1]), b_src.as_ptr(), b_default);
+        let b: Simd<u8, 4> = simd_masked_load::<_, _, _, { SimdAlign::Element }>(
+            Simd::<i8, 4>([-1, 0, -1, -1]),
+            b_src.as_ptr(),
+            b_default,
+        );
 
         assert_eq!(b.as_array(), &[4, 9, 6, 7]);
 
         let mut output = [u8::MAX; 5];
 
-        simd_masked_store(Simd::<i8, 4>([-1, -1, -1, 0]), output.as_mut_ptr(), a);
+        simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+            Simd::<i8, 4>([-1, -1, -1, 0]),
+            output.as_mut_ptr(),
+            a,
+        );
         assert_eq!(&output, &[0, 1, 2, u8::MAX, u8::MAX]);
-        simd_masked_store(Simd::<i8, 4>([0, -1, -1, 0]), output[1..].as_mut_ptr(), b);
+        simd_masked_store::<_, _, _, { SimdAlign::Element }>(
+            Simd::<i8, 4>([0, -1, -1, 0]),
+            output[1..].as_mut_ptr(),
+            b,
+        );
         assert_eq!(&output, &[0, 1, 9, 6, u8::MAX]);
     }
 }