diff --git a/crates/core_simd/src/comparisons.rs b/crates/core_simd/src/comparisons.rs index 988ff857eab5..f3a1954fda22 100644 --- a/crates/core_simd/src/comparisons.rs +++ b/crates/core_simd/src/comparisons.rs @@ -1,7 +1,7 @@ use crate::LanesAtMost32; macro_rules! implement_mask_ops { - { $($vector:ident => $mask:ident ($inner_mask_ty:ident, $inner_ty:ident),)* } => { + { $($vector:ident => $mask:ident ($inner_ty:ident),)* } => { $( impl crate::$vector where @@ -12,8 +12,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_eq(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_eq(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_eq(self, other)) } } @@ -21,8 +20,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_ne(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_ne(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_ne(self, other)) } } @@ -30,8 +28,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_lt(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_lt(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_lt(self, other)) } } @@ -39,8 +36,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_gt(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_gt(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_gt(self, other)) } } @@ -48,8 +44,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_le(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_le(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_le(self, other)) } } @@ -57,8 +52,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_ge(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_ge(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_ge(self, other)) } } } @@ -67,18 +61,18 @@ macro_rules! implement_mask_ops { } implement_mask_ops! { - SimdI8 => Mask8 (SimdMask8, SimdI8), - SimdI16 => Mask16 (SimdMask16, SimdI16), - SimdI32 => Mask32 (SimdMask32, SimdI32), - SimdI64 => Mask64 (SimdMask64, SimdI64), - SimdIsize => MaskSize (SimdMaskSize, SimdIsize), + SimdI8 => Mask8 (SimdI8), + SimdI16 => Mask16 (SimdI16), + SimdI32 => Mask32 (SimdI32), + SimdI64 => Mask64 (SimdI64), + SimdIsize => MaskSize (SimdIsize), - SimdU8 => Mask8 (SimdMask8, SimdI8), - SimdU16 => Mask16 (SimdMask16, SimdI16), - SimdU32 => Mask32 (SimdMask32, SimdI32), - SimdU64 => Mask64 (SimdMask64, SimdI64), - SimdUsize => MaskSize (SimdMaskSize, SimdIsize), + SimdU8 => Mask8 (SimdI8), + SimdU16 => Mask16 (SimdI16), + SimdU32 => Mask32 (SimdI32), + SimdU64 => Mask64 (SimdI64), + SimdUsize => MaskSize (SimdIsize), - SimdF32 => Mask32 (SimdMask32, SimdI32), - SimdF64 => Mask64 (SimdMask64, SimdI64), + SimdF32 => Mask32 (SimdI32), + SimdF64 => Mask64 (SimdI64), } diff --git a/crates/core_simd/src/lanes_at_most_32.rs b/crates/core_simd/src/lanes_at_most_32.rs index 1e2f7e952c62..2fee9ca91891 100644 --- a/crates/core_simd/src/lanes_at_most_32.rs +++ b/crates/core_simd/src/lanes_at_most_32.rs @@ -1,4 +1,4 @@ -/// Implemented for bitmask sizes that are supported by the implementation. +/// Implemented for vectors that are supported by the implementation. pub trait LanesAtMost32 {} macro_rules! impl_for { @@ -28,5 +28,3 @@ impl_for! { SimdIsize } impl_for! { SimdF32 } impl_for! { SimdF64 } - -impl_for! { BitMask } diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index b4d1b6d9557d..32e2ffb86153 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -4,14 +4,12 @@ use crate::LanesAtMost32; #[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Hash)] #[repr(transparent)] pub struct BitMask(u64) -where - BitMask: LanesAtMost32; impl BitMask where Self: LanesAtMost32, { - /// Construct a mask by setting all lanes to the given value. + #[inline] pub fn splat(value: bool) -> Self { if value { Self(u64::MAX >> (64 - LANES)) @@ -20,23 +18,13 @@ where } } - /// Tests the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] - pub fn test(&self, lane: usize) -> bool { - assert!(lane < LANES, "lane index out of range"); + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { (self.0 >> lane) & 0x1 > 0 } - /// Sets the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] - pub fn set(&mut self, lane: usize, value: bool) { - assert!(lane < LANES, "lane index out of range"); + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { self.0 ^= ((value ^ self.test(lane)) as u64) << lane } } diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index 60a6cb5fdbe8..6972a4216b68 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -1,18 +1,5 @@ //! Masks that take up full SIMD vector registers. -/// The error type returned when converting an integer to a mask fails. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct TryFromMaskError(()); - -impl core::fmt::Display for TryFromMaskError { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - write!( - f, - "mask vector must have all bits set or unset in each lane" - ) - } -} - macro_rules! define_mask { { $(#[$attr:meta])* @@ -27,6 +14,8 @@ macro_rules! define_mask { where crate::$type: crate::LanesAtMost32; + impl_full_mask_reductions! { $name, $type } + impl Copy for $name where crate::$type: crate::LanesAtMost32, @@ -46,7 +35,6 @@ macro_rules! define_mask { where crate::$type: crate::LanesAtMost32, { - /// Construct a mask by setting all lanes to the given value. pub fn splat(value: bool) -> Self { Self(>::splat( if value { @@ -57,20 +45,12 @@ macro_rules! define_mask { )) } - /// Tests the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] pub fn test(&self, lane: usize) -> bool { assert!(lane < LANES, "lane index out of range"); self.0[lane] == -1 } - /// Sets the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] pub fn set(&mut self, lane: usize, value: bool) { assert!(lane < LANES, "lane index out of range"); @@ -81,56 +61,15 @@ macro_rules! define_mask { } } - /// Converts the mask to the equivalent integer representation, where -1 represents - /// "set" and 0 represents "unset". #[inline] pub fn to_int(self) -> crate::$type { self.0 } - /// Creates a mask from the equivalent integer representation, where -1 represents - /// "set" and 0 represents "unset". - /// - /// Each provided lane must be either 0 or -1. #[inline] pub unsafe fn from_int_unchecked(value: crate::$type) -> Self { Self(value) } - - /// Creates a mask from the equivalent integer representation, where -1 represents - /// "set" and 0 represents "unset". - /// - /// # Panics - /// Panics if any lane is not 0 or -1. - #[inline] - pub fn from_int(value: crate::$type) -> Self { - use core::convert::TryInto; - value.try_into().unwrap() - } - } - - impl core::convert::From for $name - where - crate::$type: crate::LanesAtMost32, - { - fn from(value: bool) -> Self { - Self::splat(value) - } - } - - impl core::convert::TryFrom> for $name - where - crate::$type: crate::LanesAtMost32, - { - type Error = TryFromMaskError; - fn try_from(value: crate::$type) -> Result { - let valid = (value.lanes_eq(crate::$type::::splat(0)) | value.lanes_eq(crate::$type::::splat(-1))).all(); - if valid { - Ok(Self(value)) - } else { - Err(TryFromMaskError(())) - } - } } impl core::convert::From<$name> for crate::$type @@ -142,36 +81,6 @@ macro_rules! define_mask { } } - impl core::convert::From> for $name - where - crate::$type: crate::LanesAtMost32, - crate::BitMask: crate::LanesAtMost32, - { - fn from(value: crate::BitMask) -> Self { - // TODO use an intrinsic to do this efficiently (with LLVM's sext instruction) - let mut mask = Self::splat(false); - for lane in 0..LANES { - mask.set(lane, value.test(lane)); - } - mask - } - } - - impl core::convert::From<$name> for crate::BitMask - where - crate::$type: crate::LanesAtMost32, - crate::BitMask: crate::LanesAtMost32, - { - fn from(value: $name<$lanes>) -> Self { - // TODO use an intrinsic to do this efficiently (with LLVM's trunc instruction) - let mut mask = Self::splat(false); - for lane in 0..LANES { - mask.set(lane, value.test(lane)); - } - mask - } - } - impl core::fmt::Debug for $name where crate::$type: crate::LanesAtMost32, @@ -230,28 +139,6 @@ macro_rules! define_mask { } } - impl core::ops::BitAnd for $name - where - crate::$type: crate::LanesAtMost32, - { - type Output = Self; - #[inline] - fn bitand(self, rhs: bool) -> Self { - self & Self::splat(rhs) - } - } - - impl core::ops::BitAnd<$name> for bool - where - crate::$type: crate::LanesAtMost32, - { - type Output = $name; - #[inline] - fn bitand(self, rhs: $name) -> $name { - $name::::splat(self) & rhs - } - } - impl core::ops::BitOr for $name where crate::$type: crate::LanesAtMost32, @@ -263,28 +150,6 @@ macro_rules! define_mask { } } - impl core::ops::BitOr for $name - where - crate::$type: crate::LanesAtMost32, - { - type Output = Self; - #[inline] - fn bitor(self, rhs: bool) -> Self { - self | Self::splat(rhs) - } - } - - impl core::ops::BitOr<$name> for bool - where - crate::$type: crate::LanesAtMost32, - { - type Output = $name; - #[inline] - fn bitor(self, rhs: $name) -> $name { - $name::::splat(self) | rhs - } - } - impl core::ops::BitXor for $name where crate::$type: crate::LanesAtMost32, @@ -296,28 +161,6 @@ macro_rules! define_mask { } } - impl core::ops::BitXor for $name - where - crate::$type: crate::LanesAtMost32, - { - type Output = Self; - #[inline] - fn bitxor(self, rhs: bool) -> Self::Output { - self ^ Self::splat(rhs) - } - } - - impl core::ops::BitXor<$name> for bool - where - crate::$type: crate::LanesAtMost32, - { - type Output = $name; - #[inline] - fn bitxor(self, rhs: $name) -> Self::Output { - $name::::splat(self) ^ rhs - } - } - impl core::ops::Not for $name where crate::$type: crate::LanesAtMost32, @@ -339,16 +182,6 @@ macro_rules! define_mask { } } - impl core::ops::BitAndAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitand_assign(&mut self, rhs: bool) { - *self &= Self::splat(rhs); - } - } - impl core::ops::BitOrAssign for $name where crate::$type: crate::LanesAtMost32, @@ -359,16 +192,6 @@ macro_rules! define_mask { } } - impl core::ops::BitOrAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitor_assign(&mut self, rhs: bool) { - *self |= Self::splat(rhs); - } - } - impl core::ops::BitXorAssign for $name where crate::$type: crate::LanesAtMost32, @@ -378,47 +201,35 @@ macro_rules! define_mask { self.0 ^= rhs.0; } } - - impl core::ops::BitXorAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitxor_assign(&mut self, rhs: bool) { - *self ^= Self::splat(rhs); - } - } - - impl_full_mask_reductions! { $name, $type } } } define_mask! { /// A mask equivalent to [SimdI8](crate::SimdI8), where all bits in the lane must be either set /// or unset. - struct SimdMask8(crate::SimdI8); + struct Mask8(crate::SimdI8); } define_mask! { /// A mask equivalent to [SimdI16](crate::SimdI16), where all bits in the lane must be either set /// or unset. - struct SimdMask16(crate::SimdI16); + struct Mask16(crate::SimdI16); } define_mask! { /// A mask equivalent to [SimdI32](crate::SimdI32), where all bits in the lane must be either set /// or unset. - struct SimdMask32(crate::SimdI32); + struct Mask32(crate::SimdI32); } define_mask! { /// A mask equivalent to [SimdI64](crate::SimdI64), where all bits in the lane must be either set /// or unset. - struct SimdMask64(crate::SimdI64); + struct Mask64(crate::SimdI64); } define_mask! { /// A mask equivalent to [SimdIsize](crate::SimdIsize), where all bits in the lane must be either set /// or unset. - struct SimdMaskSize(crate::SimdIsize); + struct MaskSize(crate::SimdIsize); } diff --git a/crates/core_simd/src/masks/mod.rs b/crates/core_simd/src/masks/mod.rs index c394c7003a35..fbb934b96424 100644 --- a/crates/core_simd/src/masks/mod.rs +++ b/crates/core_simd/src/masks/mod.rs @@ -1,25 +1,24 @@ //! Types and traits associated with masking lanes of vectors. +//! Types representing #![allow(non_camel_case_types)] -mod full_masks; -pub use full_masks::*; - -mod bitmask; -pub use bitmask::*; +#[cfg_attr(not(all(target_arch = "x86_64", target_feature = "avx512f")), path = "full_masks.rs")] +#[cfg_attr(all(target_arch = "x86_64", target_feature = "avx512f"), path = "bitmask.rs")] +mod mask_impl; use crate::{LanesAtMost32, SimdI16, SimdI32, SimdI64, SimdI8, SimdIsize}; macro_rules! define_opaque_mask { { $(#[$attr:meta])* - struct $name:ident($inner_ty:ident<$lanes2:ident>); + struct $name:ident($inner_ty:ty); @bits $bits_ty:ident } => { $(#[$attr])* #[allow(non_camel_case_types)] - pub struct $name($inner_ty) where $bits_ty: LanesAtMost32; + pub struct $name($inner_ty) where $bits_ty: LanesAtMost32; - impl_opaque_mask_reductions! { $name, $inner_ty, $bits_ty } + impl_opaque_mask_reductions! { $name, $bits_ty } impl $name where @@ -27,7 +26,7 @@ macro_rules! define_opaque_mask { { /// Construct a mask by setting all lanes to the given value. pub fn splat(value: bool) -> Self { - Self(<$inner_ty>::splat(value)) + Self(<$inner_ty>::splat(value)) } /// Converts an array to a SIMD vector. @@ -52,6 +51,16 @@ macro_rules! define_opaque_mask { array } + /// Converts a vector of integers to a mask, where 0 represents `false` and -1 + /// represents `true`. + /// + /// # Safety + /// All lanes must be either 0 or -1. + #[inline] + pub unsafe fn from_int_unchecked(value: $bits_ty) -> Self { + Self(<$inner_ty>::from_int_unchecked(value)) + } + /// Tests the value of the specified lane. /// /// # Panics @@ -71,44 +80,6 @@ macro_rules! define_opaque_mask { } } - impl From> for $name - where - $bits_ty: LanesAtMost32, - BitMask: LanesAtMost32, - { - fn from(value: BitMask) -> Self { - Self(value.into()) - } - } - - impl From<$name> for crate::BitMask - where - $bits_ty: LanesAtMost32, - BitMask: LanesAtMost32, - { - fn from(value: $name) -> Self { - value.0.into() - } - } - - impl From<$inner_ty> for $name - where - $bits_ty: LanesAtMost32, - { - fn from(value: $inner_ty) -> Self { - Self(value) - } - } - - impl From<$name> for $inner_ty - where - $bits_ty: LanesAtMost32, - { - fn from(value: $name) -> Self { - value.0 - } - } - // vector/array conversion impl From<[bool; LANES]> for $name where @@ -130,7 +101,7 @@ macro_rules! define_opaque_mask { impl Copy for $name where - $inner_ty: Copy, + $inner_ty: Copy, $bits_ty: LanesAtMost32, {} @@ -359,7 +330,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 8-bit elements. /// /// The layout of this type is unspecified. - struct Mask8(SimdMask8); + struct Mask8(mask_impl::Mask8); @bits SimdI8 } @@ -367,7 +338,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 16-bit elements. /// /// The layout of this type is unspecified. - struct Mask16(SimdMask16); + struct Mask16(mask_impl::Mask16); @bits SimdI16 } @@ -375,7 +346,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 32-bit elements. /// /// The layout of this type is unspecified. - struct Mask32(SimdMask32); + struct Mask32(mask_impl::Mask32); @bits SimdI32 } @@ -383,7 +354,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 64-bit elements. /// /// The layout of this type is unspecified. - struct Mask64(SimdMask64); + struct Mask64(mask_impl::Mask64); @bits SimdI64 } @@ -391,7 +362,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` pointer-width elements. /// /// The layout of this type is unspecified. - struct MaskSize(SimdMaskSize); + struct MaskSize(mask_impl::MaskSize); @bits SimdIsize } diff --git a/crates/core_simd/src/reduction.rs b/crates/core_simd/src/reduction.rs index 382d366dd3d1..2d4f1bca2647 100644 --- a/crates/core_simd/src/reduction.rs +++ b/crates/core_simd/src/reduction.rs @@ -103,18 +103,16 @@ macro_rules! impl_float_reductions { } macro_rules! impl_full_mask_reductions { - { $name:ident, $inner:ident } => { - impl crate::$name + { $name:ident, $bits_ty:ident } => { + impl $name where - crate::$inner: crate::LanesAtMost32 + crate::$bits_ty: crate::LanesAtMost32 { - /// Returns true if any lane is set, or false otherwise. #[inline] pub fn any(self) -> bool { unsafe { crate::intrinsics::simd_reduce_any(self.to_int()) } } - /// Returns true if all lanes are set, or false otherwise. #[inline] pub fn all(self) -> bool { unsafe { crate::intrinsics::simd_reduce_all(self.to_int()) } @@ -124,10 +122,10 @@ macro_rules! impl_full_mask_reductions { } macro_rules! impl_opaque_mask_reductions { - { $name:ident, $inner:ident, $bits_ty:ident } => { + { $name:ident, $bits_ty:ident } => { impl $name where - $bits_ty: crate::LanesAtMost32 + crate::$bits_ty: crate::LanesAtMost32 { /// Returns true if any lane is set, or false otherwise. #[inline] @@ -143,20 +141,3 @@ macro_rules! impl_opaque_mask_reductions { } } } - -impl crate::BitMask -where - crate::BitMask: crate::LanesAtMost32, -{ - /// Returns true if any lane is set, or false otherwise. - #[inline] - pub fn any(self) -> bool { - self != Self::splat(false) - } - - /// Returns true if all lanes are set, or false otherwise. - #[inline] - pub fn all(self) -> bool { - self == Self::splat(true) - } -} diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index 59da77de622b..6c3993e39a93 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -1,30 +1,9 @@ -use core::convert::TryFrom; -use core_simd::{BitMask, Mask8, SimdI8, SimdMask8}; - #[cfg(target_arch = "wasm32")] use wasm_bindgen_test::*; #[cfg(target_arch = "wasm32")] wasm_bindgen_test_configure!(run_in_browser); -#[test] -#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] -fn mask_format_round_trip() { - let ints = SimdI8::from_array([-1, 0, 0, -1]); - - let simd_mask = SimdMask8::try_from(ints).unwrap(); - - let bitmask = BitMask::from(simd_mask); - - let opaque_mask = Mask8::from(bitmask); - - let simd_mask_returned = SimdMask8::from(opaque_mask); - - let ints_returned = SimdI8::from(simd_mask_returned); - - assert_eq!(ints_returned, ints); -} - macro_rules! test_mask_api { { $name:ident } => { #[allow(non_snake_case)] @@ -83,6 +62,4 @@ macro_rules! test_mask_api { mod mask_api { test_mask_api! { Mask8 } - test_mask_api! { SimdMask8 } - test_mask_api! { BitMask } } diff --git a/crates/test_helpers/src/lib.rs b/crates/test_helpers/src/lib.rs index 9e8790842b49..8b56877967cb 100644 --- a/crates/test_helpers/src/lib.rs +++ b/crates/test_helpers/src/lib.rs @@ -281,7 +281,6 @@ macro_rules! test_lanes { core_simd::SimdIsize<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF32<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF64<$lanes>: core_simd::LanesAtMost32, - core_simd::BitMask<$lanes>: core_simd::LanesAtMost32, $body #[cfg(target_arch = "wasm32")] @@ -351,7 +350,6 @@ macro_rules! test_lanes_panic { core_simd::SimdIsize<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF32<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF64<$lanes>: core_simd::LanesAtMost32, - core_simd::BitMask<$lanes>: core_simd::LanesAtMost32, $body #[test]