diff --git a/crates/core_simd/src/comparisons.rs b/crates/core_simd/src/comparisons.rs index 988ff857eab5..e8d11406c097 100644 --- a/crates/core_simd/src/comparisons.rs +++ b/crates/core_simd/src/comparisons.rs @@ -1,19 +1,19 @@ use crate::LanesAtMost32; macro_rules! implement_mask_ops { - { $($vector:ident => $mask:ident ($inner_mask_ty:ident, $inner_ty:ident),)* } => { + { $($vector:ident => $mask:ident ($inner_ty:ident),)* } => { $( impl crate::$vector where crate::$vector: LanesAtMost32, crate::$inner_ty: LanesAtMost32, + crate::$mask: crate::Mask, { /// Test if each lane is equal to the corresponding lane in `other`. #[inline] pub fn lanes_eq(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_eq(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_eq(self, other)) } } @@ -21,8 +21,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_ne(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_ne(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_ne(self, other)) } } @@ -30,8 +29,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_lt(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_lt(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_lt(self, other)) } } @@ -39,8 +37,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_gt(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_gt(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_gt(self, other)) } } @@ -48,8 +45,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_le(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_le(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_le(self, other)) } } @@ -57,8 +53,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_ge(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_ge(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_ge(self, other)) } } } @@ -67,18 +62,18 @@ macro_rules! implement_mask_ops { } implement_mask_ops! { - SimdI8 => Mask8 (SimdMask8, SimdI8), - SimdI16 => Mask16 (SimdMask16, SimdI16), - SimdI32 => Mask32 (SimdMask32, SimdI32), - SimdI64 => Mask64 (SimdMask64, SimdI64), - SimdIsize => MaskSize (SimdMaskSize, SimdIsize), + SimdI8 => Mask8 (SimdI8), + SimdI16 => Mask16 (SimdI16), + SimdI32 => Mask32 (SimdI32), + SimdI64 => Mask64 (SimdI64), + SimdIsize => MaskSize (SimdIsize), - SimdU8 => Mask8 (SimdMask8, SimdI8), - SimdU16 => Mask16 (SimdMask16, SimdI16), - SimdU32 => Mask32 (SimdMask32, SimdI32), - SimdU64 => Mask64 (SimdMask64, SimdI64), - SimdUsize => MaskSize (SimdMaskSize, SimdIsize), + SimdU8 => Mask8 (SimdI8), + SimdU16 => Mask16 (SimdI16), + SimdU32 => Mask32 (SimdI32), + SimdU64 => Mask64 (SimdI64), + SimdUsize => MaskSize (SimdIsize), - SimdF32 => Mask32 (SimdMask32, SimdI32), - SimdF64 => Mask64 (SimdMask64, SimdI64), + SimdF32 => Mask32 (SimdI32), + SimdF64 => Mask64 (SimdI64), } diff --git a/crates/core_simd/src/intrinsics.rs b/crates/core_simd/src/intrinsics.rs index 665dc1a51d74..8cbb0cbccf79 100644 --- a/crates/core_simd/src/intrinsics.rs +++ b/crates/core_simd/src/intrinsics.rs @@ -76,6 +76,12 @@ extern "platform-intrinsic" { pub(crate) fn simd_reduce_and(x: T) -> U; pub(crate) fn simd_reduce_or(x: T) -> U; pub(crate) fn simd_reduce_xor(x: T) -> U; + + // truncate integer vector to bitmask + pub(crate) fn simd_bitmask(x: T) -> U; + + // select + pub(crate) fn simd_select_bitmask(m: T, a: U, b: U) -> U; } #[cfg(feature = "std")] diff --git a/crates/core_simd/src/lanes_at_most_32.rs b/crates/core_simd/src/lanes_at_most_32.rs index 1e2f7e952c62..2d84b1306ea5 100644 --- a/crates/core_simd/src/lanes_at_most_32.rs +++ b/crates/core_simd/src/lanes_at_most_32.rs @@ -1,14 +1,38 @@ -/// Implemented for bitmask sizes that are supported by the implementation. -pub trait LanesAtMost32 {} +/// Implemented for vectors that are supported by the implementation. +pub trait LanesAtMost32: sealed::Sealed { + #[doc(hidden)] + type BitMask: Into; +} + +mod sealed { + pub trait Sealed {} +} macro_rules! impl_for { { $name:ident } => { - impl LanesAtMost32 for $name<1> {} - impl LanesAtMost32 for $name<2> {} - impl LanesAtMost32 for $name<4> {} - impl LanesAtMost32 for $name<8> {} - impl LanesAtMost32 for $name<16> {} - impl LanesAtMost32 for $name<32> {} + impl sealed::Sealed for $name + where + $name: LanesAtMost32, + {} + + impl LanesAtMost32 for $name<1> { + type BitMask = u8; + } + impl LanesAtMost32 for $name<2> { + type BitMask = u8; + } + impl LanesAtMost32 for $name<4> { + type BitMask = u8; + } + impl LanesAtMost32 for $name<8> { + type BitMask = u8; + } + impl LanesAtMost32 for $name<16> { + type BitMask = u16; + } + impl LanesAtMost32 for $name<32> { + type BitMask = u32; + } } } @@ -28,5 +52,3 @@ impl_for! { SimdIsize } impl_for! { SimdF32 } impl_for! { SimdF64 } - -impl_for! { BitMask } diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index b4d1b6d9557d..6bcb08cf9dbb 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -1,212 +1,171 @@ -use crate::LanesAtMost32; +use crate::Mask; +use core::marker::PhantomData; + +/// Helper trait for limiting int conversion types +pub trait ConvertToInt {} +impl ConvertToInt for crate::SimdI8 where Self: crate::LanesAtMost32 {} +impl ConvertToInt for crate::SimdI16 where Self: crate::LanesAtMost32 {} +impl ConvertToInt for crate::SimdI32 where Self: crate::LanesAtMost32 {} +impl ConvertToInt for crate::SimdI64 where Self: crate::LanesAtMost32 {} +impl ConvertToInt for crate::SimdIsize where Self: crate::LanesAtMost32 {} /// A mask where each lane is represented by a single bit. -#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Hash)] #[repr(transparent)] -pub struct BitMask(u64) -where - BitMask: LanesAtMost32; +pub struct BitMask(T::BitMask, PhantomData<[(); LANES]>); -impl BitMask -where - Self: LanesAtMost32, -{ - /// Construct a mask by setting all lanes to the given value. +impl Copy for BitMask {} + +impl Clone for BitMask { + fn clone(&self) -> Self { + *self + } +} + +impl PartialEq for BitMask { + fn eq(&self, other: &Self) -> bool { + self.0.as_ref() == other.0.as_ref() + } +} + +impl PartialOrd for BitMask { + fn partial_cmp(&self, other: &Self) -> Option { + self.0.as_ref().partial_cmp(other.0.as_ref()) + } +} + +impl Eq for BitMask {} + +impl Ord for BitMask { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.0.as_ref().cmp(other.0.as_ref()) + } +} + +impl BitMask { + #[inline] pub fn splat(value: bool) -> Self { + let mut mask = T::BitMask::default(); if value { - Self(u64::MAX >> (64 - LANES)) + mask.as_mut().fill(u8::MAX) } else { - Self(u64::MIN) + mask.as_mut().fill(u8::MIN) + } + if LANES % 8 > 0 { + *mask.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - LANES % 8); + } + Self(mask, PhantomData) + } + + #[inline] + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + (self.0.as_ref()[lane / 8] >> lane % 8) & 0x1 > 0 + } + + #[inline] + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + self.0.as_mut()[lane / 8] ^= ((value ^ self.test_unchecked(lane)) as u8) << (lane % 8) + } + + #[inline] + pub fn to_int(self) -> V + where + V: ConvertToInt + Default + core::ops::Not, + { + unsafe { + let mask: T::IntBitMask = core::mem::transmute_copy(&self); + crate::intrinsics::simd_select_bitmask(mask, !V::default(), V::default()) } } - /// Tests the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] - pub fn test(&self, lane: usize) -> bool { - assert!(lane < LANES, "lane index out of range"); - (self.0 >> lane) & 0x1 > 0 + pub unsafe fn from_int_unchecked(value: V) -> Self + where + V: crate::LanesAtMost32, + { + // TODO remove the transmute when rustc is more flexible + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + let mask: T::IntBitMask = crate::intrinsics::simd_bitmask(value); + Self(core::mem::transmute_copy(&mask), PhantomData) } - /// Sets the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] - pub fn set(&mut self, lane: usize, value: bool) { - assert!(lane < LANES, "lane index out of range"); - self.0 ^= ((value ^ self.test(lane)) as u64) << lane + pub fn to_bitmask(self) -> U::BitMask { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + unsafe { core::mem::transmute_copy(&self.0) } + } + + #[inline] + pub fn any(self) -> bool { + self != Self::splat(false) + } + + #[inline] + pub fn all(self) -> bool { + self == Self::splat(true) } } -impl core::ops::BitAnd for BitMask +impl core::ops::BitAnd for BitMask where - Self: LanesAtMost32, + T::BitMask: Default + AsRef<[u8]> + AsMut<[u8]>, { type Output = Self; #[inline] - fn bitand(self, rhs: Self) -> Self { - Self(self.0 & rhs.0) + fn bitand(mut self, rhs: Self) -> Self { + for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { + *l &= r; + } + self } } -impl core::ops::BitAnd for BitMask +impl core::ops::BitOr for BitMask where - Self: LanesAtMost32, + T::BitMask: Default + AsRef<[u8]> + AsMut<[u8]>, { type Output = Self; #[inline] - fn bitand(self, rhs: bool) -> Self { - self & Self::splat(rhs) + fn bitor(mut self, rhs: Self) -> Self { + for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { + *l |= r; + } + self } } -impl core::ops::BitAnd> for bool -where - BitMask: LanesAtMost32, -{ - type Output = BitMask; - #[inline] - fn bitand(self, rhs: BitMask) -> BitMask { - BitMask::::splat(self) & rhs - } -} - -impl core::ops::BitOr for BitMask -where - Self: LanesAtMost32, -{ +impl core::ops::BitXor for BitMask { type Output = Self; #[inline] - fn bitor(self, rhs: Self) -> Self { - Self(self.0 | rhs.0) + fn bitxor(mut self, rhs: Self) -> Self::Output { + for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { + *l ^= r; + } + self } } -impl core::ops::BitOr for BitMask -where - Self: LanesAtMost32, -{ +impl core::ops::Not for BitMask { type Output = Self; #[inline] - fn bitor(self, rhs: bool) -> Self { - self | Self::splat(rhs) + fn not(mut self) -> Self::Output { + for x in self.0.as_mut() { + *x = !*x; + } + if LANES % 8 > 0 { + *self.0.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - LANES % 8); + } + self } } -impl core::ops::BitOr> for bool -where - BitMask: LanesAtMost32, -{ - type Output = BitMask; - #[inline] - fn bitor(self, rhs: BitMask) -> BitMask { - BitMask::::splat(self) | rhs - } -} - -impl core::ops::BitXor for BitMask -where - Self: LanesAtMost32, -{ - type Output = Self; - #[inline] - fn bitxor(self, rhs: Self) -> Self::Output { - Self(self.0 ^ rhs.0) - } -} - -impl core::ops::BitXor for BitMask -where - Self: LanesAtMost32, -{ - type Output = Self; - #[inline] - fn bitxor(self, rhs: bool) -> Self::Output { - self ^ Self::splat(rhs) - } -} - -impl core::ops::BitXor> for bool -where - BitMask: LanesAtMost32, -{ - type Output = BitMask; - #[inline] - fn bitxor(self, rhs: BitMask) -> Self::Output { - BitMask::::splat(self) ^ rhs - } -} - -impl core::ops::Not for BitMask -where - Self: LanesAtMost32, -{ - type Output = BitMask; - #[inline] - fn not(self) -> Self::Output { - Self(!self.0) - } -} - -impl core::ops::BitAndAssign for BitMask -where - Self: LanesAtMost32, -{ - #[inline] - fn bitand_assign(&mut self, rhs: Self) { - self.0 &= rhs.0; - } -} - -impl core::ops::BitAndAssign for BitMask -where - Self: LanesAtMost32, -{ - #[inline] - fn bitand_assign(&mut self, rhs: bool) { - *self &= Self::splat(rhs); - } -} - -impl core::ops::BitOrAssign for BitMask -where - Self: LanesAtMost32, -{ - #[inline] - fn bitor_assign(&mut self, rhs: Self) { - self.0 |= rhs.0; - } -} - -impl core::ops::BitOrAssign for BitMask -where - Self: LanesAtMost32, -{ - #[inline] - fn bitor_assign(&mut self, rhs: bool) { - *self |= Self::splat(rhs); - } -} - -impl core::ops::BitXorAssign for BitMask -where - Self: LanesAtMost32, -{ - #[inline] - fn bitxor_assign(&mut self, rhs: Self) { - self.0 ^= rhs.0; - } -} - -impl core::ops::BitXorAssign for BitMask -where - Self: LanesAtMost32, -{ - #[inline] - fn bitxor_assign(&mut self, rhs: bool) { - *self ^= Self::splat(rhs); - } -} +pub type Mask8 = BitMask; +pub type Mask16 = BitMask; +pub type Mask32 = BitMask; +pub type Mask64 = BitMask; +pub type MaskSize = BitMask; diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index 60a6cb5fdbe8..f89bbefba631 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -1,17 +1,7 @@ //! Masks that take up full SIMD vector registers. -/// The error type returned when converting an integer to a mask fails. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct TryFromMaskError(()); - -impl core::fmt::Display for TryFromMaskError { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - write!( - f, - "mask vector must have all bits set or unset in each lane" - ) - } -} +use crate::Mask; +use core::marker::PhantomData; macro_rules! define_mask { { @@ -21,18 +11,19 @@ macro_rules! define_mask { ); } => { $(#[$attr])* - #[derive(Default, PartialEq, PartialOrd, Eq, Ord, Hash)] #[repr(transparent)] - pub struct $name(crate::$type<$lanes2>) + pub struct $name(crate::$type<$lanes2>, PhantomData) where crate::$type: crate::LanesAtMost32; - impl Copy for $name + impl_full_mask_reductions! { $name, $type } + + impl Copy for $name where crate::$type: crate::LanesAtMost32, {} - impl Clone for $name + impl Clone for $name where crate::$type: crate::LanesAtMost32, { @@ -42,38 +33,62 @@ macro_rules! define_mask { } } - impl $name + impl PartialEq for $name + where + crate::$type: crate::LanesAtMost32, + { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + + impl PartialOrd for $name + where + crate::$type: crate::LanesAtMost32, + { + fn partial_cmp(&self, other: &Self) -> Option { + self.0.partial_cmp(&other.0) + } + } + + impl Eq for $name + where + crate::$type: crate::LanesAtMost32, + {} + + impl Ord for $name + where + crate::$type: crate::LanesAtMost32, + { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.0.cmp(&other.0) + } + } + + impl $name where crate::$type: crate::LanesAtMost32, { - /// Construct a mask by setting all lanes to the given value. pub fn splat(value: bool) -> Self { - Self(>::splat( - if value { - -1 - } else { - 0 - } - )) + Self( + >::splat( + if value { + -1 + } else { + 0 + } + ), + PhantomData, + ) } - /// Tests the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] - pub fn test(&self, lane: usize) -> bool { - assert!(lane < LANES, "lane index out of range"); + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { self.0[lane] == -1 } - /// Sets the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] - pub fn set(&mut self, lane: usize, value: bool) { - assert!(lane < LANES, "lane index out of range"); + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { self.0[lane] = if value { -1 } else { @@ -81,344 +96,119 @@ macro_rules! define_mask { } } - /// Converts the mask to the equivalent integer representation, where -1 represents - /// "set" and 0 represents "unset". #[inline] pub fn to_int(self) -> crate::$type { self.0 } - /// Creates a mask from the equivalent integer representation, where -1 represents - /// "set" and 0 represents "unset". - /// - /// Each provided lane must be either 0 or -1. #[inline] pub unsafe fn from_int_unchecked(value: crate::$type) -> Self { - Self(value) + Self(value, PhantomData) } - /// Creates a mask from the equivalent integer representation, where -1 represents - /// "set" and 0 represents "unset". - /// - /// # Panics - /// Panics if any lane is not 0 or -1. #[inline] - pub fn from_int(value: crate::$type) -> Self { - use core::convert::TryInto; - value.try_into().unwrap() - } - } + pub fn to_bitmask(self) -> U::BitMask { + unsafe { + // TODO remove the transmute when rustc is more flexible + assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + let mask: U::IntBitMask = crate::intrinsics::simd_bitmask(self.0); + let mut bitmask: U::BitMask = core::mem::transmute_copy(&mask); - impl core::convert::From for $name - where - crate::$type: crate::LanesAtMost32, - { - fn from(value: bool) -> Self { - Self::splat(value) - } - } + // There is a bug where LLVM appears to implement this operation with the wrong + // bit order. + // TODO fix this in a better way + if cfg!(any(target_arch = "mips", target_arch = "mips64")) { + for x in bitmask.as_mut() { + *x = x.reverse_bits(); + } + } - impl core::convert::TryFrom> for $name - where - crate::$type: crate::LanesAtMost32, - { - type Error = TryFromMaskError; - fn try_from(value: crate::$type) -> Result { - let valid = (value.lanes_eq(crate::$type::::splat(0)) | value.lanes_eq(crate::$type::::splat(-1))).all(); - if valid { - Ok(Self(value)) - } else { - Err(TryFromMaskError(())) + bitmask } } } - impl core::convert::From<$name> for crate::$type + impl core::convert::From<$name> for crate::$type where crate::$type: crate::LanesAtMost32, { - fn from(value: $name) -> Self { + fn from(value: $name) -> Self { value.0 } } - impl core::convert::From> for $name - where - crate::$type: crate::LanesAtMost32, - crate::BitMask: crate::LanesAtMost32, - { - fn from(value: crate::BitMask) -> Self { - // TODO use an intrinsic to do this efficiently (with LLVM's sext instruction) - let mut mask = Self::splat(false); - for lane in 0..LANES { - mask.set(lane, value.test(lane)); - } - mask - } - } - - impl core::convert::From<$name> for crate::BitMask - where - crate::$type: crate::LanesAtMost32, - crate::BitMask: crate::LanesAtMost32, - { - fn from(value: $name<$lanes>) -> Self { - // TODO use an intrinsic to do this efficiently (with LLVM's trunc instruction) - let mut mask = Self::splat(false); - for lane in 0..LANES { - mask.set(lane, value.test(lane)); - } - mask - } - } - - impl core::fmt::Debug for $name - where - crate::$type: crate::LanesAtMost32, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - f.debug_list() - .entries((0..LANES).map(|lane| self.test(lane))) - .finish() - } - } - - impl core::fmt::Binary for $name - where - crate::$type: crate::LanesAtMost32, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - core::fmt::Binary::fmt(&self.0, f) - } - } - - impl core::fmt::Octal for $name - where - crate::$type: crate::LanesAtMost32, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - core::fmt::Octal::fmt(&self.0, f) - } - } - - impl core::fmt::LowerHex for $name - where - crate::$type: crate::LanesAtMost32, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - core::fmt::LowerHex::fmt(&self.0, f) - } - } - - impl core::fmt::UpperHex for $name - where - crate::$type: crate::LanesAtMost32, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - core::fmt::UpperHex::fmt(&self.0, f) - } - } - - impl core::ops::BitAnd for $name + impl core::ops::BitAnd for $name where crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] fn bitand(self, rhs: Self) -> Self { - Self(self.0 & rhs.0) + Self(self.0 & rhs.0, PhantomData) } } - impl core::ops::BitAnd for $name - where - crate::$type: crate::LanesAtMost32, - { - type Output = Self; - #[inline] - fn bitand(self, rhs: bool) -> Self { - self & Self::splat(rhs) - } - } - - impl core::ops::BitAnd<$name> for bool - where - crate::$type: crate::LanesAtMost32, - { - type Output = $name; - #[inline] - fn bitand(self, rhs: $name) -> $name { - $name::::splat(self) & rhs - } - } - - impl core::ops::BitOr for $name + impl core::ops::BitOr for $name where crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] fn bitor(self, rhs: Self) -> Self { - Self(self.0 | rhs.0) + Self(self.0 | rhs.0, PhantomData) } } - impl core::ops::BitOr for $name - where - crate::$type: crate::LanesAtMost32, - { - type Output = Self; - #[inline] - fn bitor(self, rhs: bool) -> Self { - self | Self::splat(rhs) - } - } - - impl core::ops::BitOr<$name> for bool - where - crate::$type: crate::LanesAtMost32, - { - type Output = $name; - #[inline] - fn bitor(self, rhs: $name) -> $name { - $name::::splat(self) | rhs - } - } - - impl core::ops::BitXor for $name + impl core::ops::BitXor for $name where crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] fn bitxor(self, rhs: Self) -> Self::Output { - Self(self.0 ^ rhs.0) + Self(self.0 ^ rhs.0, PhantomData) } } - impl core::ops::BitXor for $name + impl core::ops::Not for $name where crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] - fn bitxor(self, rhs: bool) -> Self::Output { - self ^ Self::splat(rhs) - } - } - - impl core::ops::BitXor<$name> for bool - where - crate::$type: crate::LanesAtMost32, - { - type Output = $name; - #[inline] - fn bitxor(self, rhs: $name) -> Self::Output { - $name::::splat(self) ^ rhs - } - } - - impl core::ops::Not for $name - where - crate::$type: crate::LanesAtMost32, - { - type Output = $name; - #[inline] fn not(self) -> Self::Output { - Self(!self.0) + Self(!self.0, PhantomData) } } - - impl core::ops::BitAndAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitand_assign(&mut self, rhs: Self) { - self.0 &= rhs.0; - } - } - - impl core::ops::BitAndAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitand_assign(&mut self, rhs: bool) { - *self &= Self::splat(rhs); - } - } - - impl core::ops::BitOrAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitor_assign(&mut self, rhs: Self) { - self.0 |= rhs.0; - } - } - - impl core::ops::BitOrAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitor_assign(&mut self, rhs: bool) { - *self |= Self::splat(rhs); - } - } - - impl core::ops::BitXorAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitxor_assign(&mut self, rhs: Self) { - self.0 ^= rhs.0; - } - } - - impl core::ops::BitXorAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitxor_assign(&mut self, rhs: bool) { - *self ^= Self::splat(rhs); - } - } - - impl_full_mask_reductions! { $name, $type } } } define_mask! { /// A mask equivalent to [SimdI8](crate::SimdI8), where all bits in the lane must be either set /// or unset. - struct SimdMask8(crate::SimdI8); + struct Mask8(crate::SimdI8); } define_mask! { /// A mask equivalent to [SimdI16](crate::SimdI16), where all bits in the lane must be either set /// or unset. - struct SimdMask16(crate::SimdI16); + struct Mask16(crate::SimdI16); } define_mask! { /// A mask equivalent to [SimdI32](crate::SimdI32), where all bits in the lane must be either set /// or unset. - struct SimdMask32(crate::SimdI32); + struct Mask32(crate::SimdI32); } define_mask! { /// A mask equivalent to [SimdI64](crate::SimdI64), where all bits in the lane must be either set /// or unset. - struct SimdMask64(crate::SimdI64); + struct Mask64(crate::SimdI64); } define_mask! { /// A mask equivalent to [SimdIsize](crate::SimdIsize), where all bits in the lane must be either set /// or unset. - struct SimdMaskSize(crate::SimdIsize); + struct MaskSize(crate::SimdIsize); } diff --git a/crates/core_simd/src/masks/mod.rs b/crates/core_simd/src/masks/mod.rs index c394c7003a35..deaf2be5dca4 100644 --- a/crates/core_simd/src/masks/mod.rs +++ b/crates/core_simd/src/masks/mod.rs @@ -1,33 +1,86 @@ //! Types and traits associated with masking lanes of vectors. +//! Types representing #![allow(non_camel_case_types)] -mod full_masks; -pub use full_masks::*; - -mod bitmask; -pub use bitmask::*; +#[cfg_attr( + not(all(target_arch = "x86_64", target_feature = "avx512f")), + path = "full_masks.rs" +)] +#[cfg_attr( + all(target_arch = "x86_64", target_feature = "avx512f"), + path = "bitmask.rs" +)] +mod mask_impl; use crate::{LanesAtMost32, SimdI16, SimdI32, SimdI64, SimdI8, SimdIsize}; +mod sealed { + pub trait Sealed {} +} + +/// Helper trait for mask types. +pub trait Mask: sealed::Sealed { + /// The bitmask representation of a mask. + type BitMask: Copy + Default + AsRef<[u8]> + AsMut<[u8]>; + + // TODO remove this when rustc intrinsics are more flexible + #[doc(hidden)] + type IntBitMask; +} + macro_rules! define_opaque_mask { { $(#[$attr:meta])* - struct $name:ident($inner_ty:ident<$lanes2:ident>); + struct $name:ident($inner_ty:ty); @bits $bits_ty:ident } => { $(#[$attr])* #[allow(non_camel_case_types)] - pub struct $name($inner_ty) where $bits_ty: LanesAtMost32; + pub struct $name($inner_ty) + where + $bits_ty: LanesAtMost32, + Self: Mask; - impl_opaque_mask_reductions! { $name, $inner_ty, $bits_ty } + impl sealed::Sealed for $name + where + $bits_ty: LanesAtMost32, + Self: Mask, + {} + impl Mask for $name<1> { + type BitMask = [u8; 1]; + type IntBitMask = u8; + } + impl Mask for $name<2> { + type BitMask = [u8; 1]; + type IntBitMask = u8; + } + impl Mask for $name<4> { + type BitMask = [u8; 1]; + type IntBitMask = u8; + } + impl Mask for $name<8> { + type BitMask = [u8; 1]; + type IntBitMask = u8; + } + impl Mask for $name<16> { + type BitMask = [u8; 2]; + type IntBitMask = u16; + } + impl Mask for $name<32> { + type BitMask = [u8; 4]; + type IntBitMask = u32; + } + + impl_opaque_mask_reductions! { $name, $bits_ty } impl $name where - $bits_ty: LanesAtMost32 + $bits_ty: LanesAtMost32, + Self: Mask, { /// Construct a mask by setting all lanes to the given value. pub fn splat(value: bool) -> Self { - Self(<$inner_ty>::splat(value)) + Self(<$inner_ty>::splat(value)) } /// Converts an array to a SIMD vector. @@ -52,13 +105,63 @@ macro_rules! define_opaque_mask { array } + /// Converts a vector of integers to a mask, where 0 represents `false` and -1 + /// represents `true`. + /// + /// # Safety + /// All lanes must be either 0 or -1. + #[inline] + pub unsafe fn from_int_unchecked(value: $bits_ty) -> Self { + Self(<$inner_ty>::from_int_unchecked(value)) + } + + /// Converts a vector of integers to a mask, where 0 represents `false` and -1 + /// represents `true`. + /// + /// # Panics + /// Panics if any lane is not 0 or -1. + #[inline] + pub fn from_int(value: $bits_ty) -> Self { + assert!( + (value.lanes_eq($bits_ty::splat(0)) | value.lanes_eq($bits_ty::splat(-1))).all(), + "all values must be either 0 or -1", + ); + unsafe { Self::from_int_unchecked(value) } + } + + /// Converts the mask to a vector of integers, where 0 represents `false` and -1 + /// represents `true`. + #[inline] + pub fn to_int(self) -> $bits_ty { + self.0.to_int() + } + + /// Tests the value of the specified lane. + /// + /// # Safety + /// `lane` must be less than `LANES`. + #[inline] + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + self.0.test_unchecked(lane) + } + /// Tests the value of the specified lane. /// /// # Panics /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] pub fn test(&self, lane: usize) -> bool { - self.0.test(lane) + assert!(lane < LANES, "lane index out of range"); + unsafe { self.test_unchecked(lane) } + } + + /// Sets the value of the specified lane. + /// + /// # Safety + /// `lane` must be less than `LANES`. + #[inline] + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + self.0.set_unchecked(lane, value); } /// Sets the value of the specified lane. @@ -67,52 +170,21 @@ macro_rules! define_opaque_mask { /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] pub fn set(&mut self, lane: usize, value: bool) { - self.0.set(lane, value); + assert!(lane < LANES, "lane index out of range"); + unsafe { self.set_unchecked(lane, value); } } - } - impl From> for $name - where - $bits_ty: LanesAtMost32, - BitMask: LanesAtMost32, - { - fn from(value: BitMask) -> Self { - Self(value.into()) - } - } - - impl From<$name> for crate::BitMask - where - $bits_ty: LanesAtMost32, - BitMask: LanesAtMost32, - { - fn from(value: $name) -> Self { - value.0.into() - } - } - - impl From<$inner_ty> for $name - where - $bits_ty: LanesAtMost32, - { - fn from(value: $inner_ty) -> Self { - Self(value) - } - } - - impl From<$name> for $inner_ty - where - $bits_ty: LanesAtMost32, - { - fn from(value: $name) -> Self { - value.0 + /// Convert this mask to a bitmask, with one bit set per lane. + pub fn to_bitmask(self) -> ::BitMask { + self.0.to_bitmask::() } } // vector/array conversion impl From<[bool; LANES]> for $name where - $bits_ty: crate::LanesAtMost32 + $bits_ty: crate::LanesAtMost32, + Self: Mask, { fn from(array: [bool; LANES]) -> Self { Self::from_array(array) @@ -121,7 +193,8 @@ macro_rules! define_opaque_mask { impl From<$name> for [bool; LANES] where - $bits_ty: crate::LanesAtMost32 + $bits_ty: crate::LanesAtMost32, + $name: Mask, { fn from(vector: $name) -> Self { vector.to_array() @@ -130,13 +203,14 @@ macro_rules! define_opaque_mask { impl Copy for $name where - $inner_ty: Copy, $bits_ty: LanesAtMost32, + Self: Mask, {} impl Clone for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn clone(&self) -> Self { @@ -147,6 +221,7 @@ macro_rules! define_opaque_mask { impl Default for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn default() -> Self { @@ -157,6 +232,7 @@ macro_rules! define_opaque_mask { impl PartialEq for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn eq(&self, other: &Self) -> bool { @@ -167,6 +243,7 @@ macro_rules! define_opaque_mask { impl PartialOrd for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn partial_cmp(&self, other: &Self) -> Option { @@ -176,16 +253,20 @@ macro_rules! define_opaque_mask { impl core::fmt::Debug for $name where - $bits_ty: LanesAtMost32, + $bits_ty: crate::LanesAtMost32, + Self: Mask, { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - core::fmt::Debug::fmt(&self.0, f) + f.debug_list() + .entries((0..LANES).map(|lane| self.test(lane))) + .finish() } } impl core::ops::BitAnd for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -197,6 +278,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitAnd for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -208,6 +290,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitAnd<$name> for bool where $bits_ty: LanesAtMost32, + $name: Mask, { type Output = $name; #[inline] @@ -219,6 +302,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOr for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -230,6 +314,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOr for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -241,6 +326,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOr<$name> for bool where $bits_ty: LanesAtMost32, + $name: Mask, { type Output = $name; #[inline] @@ -252,6 +338,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXor for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -263,6 +350,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXor for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -274,6 +362,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXor<$name> for bool where $bits_ty: LanesAtMost32, + $name: Mask, { type Output = $name; #[inline] @@ -285,6 +374,7 @@ macro_rules! define_opaque_mask { impl core::ops::Not for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = $name; #[inline] @@ -296,16 +386,18 @@ macro_rules! define_opaque_mask { impl core::ops::BitAndAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitand_assign(&mut self, rhs: Self) { - self.0 &= rhs.0; + self.0 = self.0 & rhs.0; } } impl core::ops::BitAndAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitand_assign(&mut self, rhs: bool) { @@ -316,16 +408,18 @@ macro_rules! define_opaque_mask { impl core::ops::BitOrAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitor_assign(&mut self, rhs: Self) { - self.0 |= rhs.0; + self.0 = self.0 | rhs.0; } } impl core::ops::BitOrAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitor_assign(&mut self, rhs: bool) { @@ -336,16 +430,18 @@ macro_rules! define_opaque_mask { impl core::ops::BitXorAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitxor_assign(&mut self, rhs: Self) { - self.0 ^= rhs.0; + self.0 = self.0 ^ rhs.0; } } impl core::ops::BitXorAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitxor_assign(&mut self, rhs: bool) { @@ -359,7 +455,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 8-bit elements. /// /// The layout of this type is unspecified. - struct Mask8(SimdMask8); + struct Mask8(mask_impl::Mask8); @bits SimdI8 } @@ -367,7 +463,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 16-bit elements. /// /// The layout of this type is unspecified. - struct Mask16(SimdMask16); + struct Mask16(mask_impl::Mask16); @bits SimdI16 } @@ -375,7 +471,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 32-bit elements. /// /// The layout of this type is unspecified. - struct Mask32(SimdMask32); + struct Mask32(mask_impl::Mask32); @bits SimdI32 } @@ -383,7 +479,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 64-bit elements. /// /// The layout of this type is unspecified. - struct Mask64(SimdMask64); + struct Mask64(mask_impl::Mask64); @bits SimdI64 } @@ -391,7 +487,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` pointer-width elements. /// /// The layout of this type is unspecified. - struct MaskSize(SimdMaskSize); + struct MaskSize(mask_impl::MaskSize); @bits SimdIsize } diff --git a/crates/core_simd/src/reduction.rs b/crates/core_simd/src/reduction.rs index 382d366dd3d1..8687d1af5167 100644 --- a/crates/core_simd/src/reduction.rs +++ b/crates/core_simd/src/reduction.rs @@ -103,18 +103,16 @@ macro_rules! impl_float_reductions { } macro_rules! impl_full_mask_reductions { - { $name:ident, $inner:ident } => { - impl crate::$name + { $name:ident, $bits_ty:ident } => { + impl $name where - crate::$inner: crate::LanesAtMost32 + crate::$bits_ty: crate::LanesAtMost32 { - /// Returns true if any lane is set, or false otherwise. #[inline] pub fn any(self) -> bool { unsafe { crate::intrinsics::simd_reduce_any(self.to_int()) } } - /// Returns true if all lanes are set, or false otherwise. #[inline] pub fn all(self) -> bool { unsafe { crate::intrinsics::simd_reduce_all(self.to_int()) } @@ -124,10 +122,11 @@ macro_rules! impl_full_mask_reductions { } macro_rules! impl_opaque_mask_reductions { - { $name:ident, $inner:ident, $bits_ty:ident } => { + { $name:ident, $bits_ty:ident } => { impl $name where - $bits_ty: crate::LanesAtMost32 + crate::$bits_ty: crate::LanesAtMost32, + $name: crate::Mask, { /// Returns true if any lane is set, or false otherwise. #[inline] @@ -143,20 +142,3 @@ macro_rules! impl_opaque_mask_reductions { } } } - -impl crate::BitMask -where - crate::BitMask: crate::LanesAtMost32, -{ - /// Returns true if any lane is set, or false otherwise. - #[inline] - pub fn any(self) -> bool { - self != Self::splat(false) - } - - /// Returns true if all lanes are set, or false otherwise. - #[inline] - pub fn all(self) -> bool { - self == Self::splat(true) - } -} diff --git a/crates/core_simd/src/vector/float.rs b/crates/core_simd/src/vector/float.rs index 47013053ae15..6371f88a40a2 100644 --- a/crates/core_simd/src/vector/float.rs +++ b/crates/core_simd/src/vector/float.rs @@ -42,6 +42,7 @@ macro_rules! impl_float_vector { Self: crate::LanesAtMost32, crate::$bits_ty: crate::LanesAtMost32, crate::$mask_impl_ty: crate::LanesAtMost32, + crate::$mask_ty: crate::Mask, { /// Returns true for each lane if it has a positive sign, including /// `+0.0`, `NaN`s with positive sign bit and positive infinity. diff --git a/crates/core_simd/src/vector/int.rs b/crates/core_simd/src/vector/int.rs index 30b09a229e9b..a535fad7bc1d 100644 --- a/crates/core_simd/src/vector/int.rs +++ b/crates/core_simd/src/vector/int.rs @@ -30,6 +30,7 @@ macro_rules! impl_integer_vector { where Self: crate::LanesAtMost32, crate::$mask_impl_ty: crate::LanesAtMost32, + crate::$mask_ty: crate::Mask, { /// Returns true for each positive lane and false if it is zero or negative. pub fn is_positive(self) -> crate::$mask_ty { diff --git a/crates/core_simd/src/vector/uint.rs b/crates/core_simd/src/vector/uint.rs index 53e780520a79..db027b0941f2 100644 --- a/crates/core_simd/src/vector/uint.rs +++ b/crates/core_simd/src/vector/uint.rs @@ -1,6 +1,5 @@ #![allow(non_camel_case_types)] - /// Implements additional integer traits (Eq, Ord, Hash) on the specified vector `$name`, holding multiple `$lanes` of `$type`. macro_rules! impl_unsigned_vector { { $name:ident, $type:ty } => { diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index 59da77de622b..7021d58aa543 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -1,30 +1,9 @@ -use core::convert::TryFrom; -use core_simd::{BitMask, Mask8, SimdI8, SimdMask8}; - #[cfg(target_arch = "wasm32")] use wasm_bindgen_test::*; #[cfg(target_arch = "wasm32")] wasm_bindgen_test_configure!(run_in_browser); -#[test] -#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] -fn mask_format_round_trip() { - let ints = SimdI8::from_array([-1, 0, 0, -1]); - - let simd_mask = SimdMask8::try_from(ints).unwrap(); - - let bitmask = BitMask::from(simd_mask); - - let opaque_mask = Mask8::from(bitmask); - - let simd_mask_returned = SimdMask8::from(opaque_mask); - - let ints_returned = SimdI8::from(simd_mask_returned); - - assert_eq!(ints_returned, ints); -} - macro_rules! test_mask_api { { $name:ident } => { #[allow(non_snake_case)] @@ -77,12 +56,29 @@ macro_rules! test_mask_api { v.set(2, true); assert!(!v.all()); } + + #[test] + fn roundtrip_int_conversion() { + let values = [true, false, false, true, false, false, true, false]; + let mask = core_simd::$name::<8>::from_array(values); + let int = mask.to_int(); + assert_eq!(int.to_array(), [-1, 0, 0, -1, 0, 0, -1, 0]); + assert_eq!(core_simd::$name::<8>::from_int(int), mask); + } + + #[test] + fn to_bitmask() { + let values = [ + true, false, false, true, false, false, true, false, + true, true, false, false, false, false, false, true, + ]; + let mask = core_simd::$name::<16>::from_array(values); + assert_eq!(mask.to_bitmask(), [0b01001001, 0b10000011]); + } } } } mod mask_api { test_mask_api! { Mask8 } - test_mask_api! { SimdMask8 } - test_mask_api! { BitMask } } diff --git a/crates/test_helpers/src/array.rs b/crates/test_helpers/src/array.rs index c64bfee4f2d1..5ffc92269769 100644 --- a/crates/test_helpers/src/array.rs +++ b/crates/test_helpers/src/array.rs @@ -3,14 +3,11 @@ // Adapted from proptest's array code // Copyright 2017 Jason Lingle +use core::{marker::PhantomData, mem::MaybeUninit}; use proptest::{ strategy::{NewTree, Strategy, ValueTree}, test_runner::TestRunner, }; -use core::{ - marker::PhantomData, - mem::MaybeUninit, -}; #[must_use = "strategies do nothing unless used"] #[derive(Clone, Copy, Debug)] diff --git a/crates/test_helpers/src/lib.rs b/crates/test_helpers/src/lib.rs index 9e8790842b49..fffd088f4da3 100644 --- a/crates/test_helpers/src/lib.rs +++ b/crates/test_helpers/src/lib.rs @@ -281,7 +281,11 @@ macro_rules! test_lanes { core_simd::SimdIsize<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF32<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF64<$lanes>: core_simd::LanesAtMost32, - core_simd::BitMask<$lanes>: core_simd::LanesAtMost32, + core_simd::Mask8<$lanes>: core_simd::Mask, + core_simd::Mask16<$lanes>: core_simd::Mask, + core_simd::Mask32<$lanes>: core_simd::Mask, + core_simd::Mask64<$lanes>: core_simd::Mask, + core_simd::MaskSize<$lanes>: core_simd::Mask, $body #[cfg(target_arch = "wasm32")] @@ -351,7 +355,11 @@ macro_rules! test_lanes_panic { core_simd::SimdIsize<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF32<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF64<$lanes>: core_simd::LanesAtMost32, - core_simd::BitMask<$lanes>: core_simd::LanesAtMost32, + core_simd::Mask8<$lanes>: core_simd::Mask, + core_simd::Mask16<$lanes>: core_simd::Mask, + core_simd::Mask32<$lanes>: core_simd::Mask, + core_simd::Mask64<$lanes>: core_simd::Mask, + core_simd::MaskSize<$lanes>: core_simd::Mask, $body #[test]