diff --git a/crates/core_simd/src/fmt.rs b/crates/core_simd/src/fmt.rs index c634e0546bc1..07332c1ccc8f 100644 --- a/crates/core_simd/src/fmt.rs +++ b/crates/core_simd/src/fmt.rs @@ -74,10 +74,10 @@ macro_rules! impl_fmt_trait { impl_fmt_trait! { integers: - crate::u8x8, crate::u8x16, crate::u8x32, crate::u8x64, - crate::i8x8, crate::i8x16, crate::i8x32, crate::i8x64, - crate::u16x4, crate::u16x8, crate::u16x16, crate::u16x32, - crate::i16x4, crate::i16x8, crate::i16x16, crate::i16x32, + crate::u8x8, crate::u8x16, crate::u8x32, crate::u8x64, + crate::i8x8, crate::i8x16, crate::i8x32, crate::i8x64, + crate::u16x4, crate::u16x8, crate::u16x16, crate::u16x32, + crate::i16x4, crate::i16x8, crate::i16x16, crate::i16x32, crate::u32x2, crate::u32x4, crate::u32x8, crate::u32x16, crate::i32x2, crate::i32x4, crate::i32x8, crate::i32x16, crate::u64x2, crate::u64x4, crate::u64x8, @@ -96,10 +96,10 @@ impl_fmt_trait! { impl_fmt_trait! { masks: - crate::mask8x8, crate::mask8x16, crate::mask8x32, crate::mask8x64, - crate::mask16x4, crate::mask16x8, crate::mask16x16, crate::mask16x32, - crate::mask32x2, crate::mask32x4, crate::mask32x8, crate::mask32x16, - crate::mask64x2, crate::mask64x4, crate::mask64x8, - crate::mask128x2, crate::mask128x4, - crate::masksizex2, crate::masksizex4, crate::masksizex8, + crate::masks::wide::m8x8, crate::masks::wide::m8x16, crate::masks::wide::m8x32, crate::masks::wide::m8x64, + crate::masks::wide::m16x4, crate::masks::wide::m16x8, crate::masks::wide::m16x16, crate::masks::wide::m16x32, + crate::masks::wide::m32x2, crate::masks::wide::m32x4, crate::masks::wide::m32x8, crate::masks::wide::m32x16, + crate::masks::wide::m64x2, crate::masks::wide::m64x4, crate::masks::wide::m64x8, + crate::masks::wide::m128x2, crate::masks::wide::m128x4, + crate::masks::wide::msizex2, crate::masks::wide::msizex4, crate::masks::wide::msizex8, } diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index cea39e6f3f3d..fd4f9dd16fd5 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -10,8 +10,7 @@ mod fmt; mod intrinsics; mod ops; -mod masks; -pub use masks::*; +pub mod masks; mod vectors_u8; pub use vectors_u8::*; @@ -44,17 +43,4 @@ pub use vectors_f32::*; mod vectors_f64; pub use vectors_f64::*; -mod vectors_mask8; -pub use vectors_mask8::*; -mod vectors_mask16; -pub use vectors_mask16::*; -mod vectors_mask32; -pub use vectors_mask32::*; -mod vectors_mask64; -pub use vectors_mask64::*; -mod vectors_mask128; -pub use vectors_mask128::*; -mod vectors_masksize; -pub use vectors_masksize::*; - mod round; diff --git a/crates/core_simd/src/macros.rs b/crates/core_simd/src/macros.rs index f37d13c3ca32..b8324ffdb929 100644 --- a/crates/core_simd/src/macros.rs +++ b/crates/core_simd/src/macros.rs @@ -314,7 +314,6 @@ macro_rules! define_float_vector { } } - /// Defines an integer vector `$name` containing multiple `$lanes` of integer `$type`. macro_rules! define_integer_vector { { $(#[$attr:meta])* struct $name:ident([$type:ty; $lanes:tt]); } => { @@ -336,6 +335,7 @@ macro_rules! define_mask_vector { impl $name { call_repeat! { $lanes => define_mask_vector [$impl_type] splat $type | } call_counting_args! { $lanes => define_mask_vector => new $type | } + call_counting_args! { $lanes => define_mask_vector => new_from_bool $type | } } base_vector_traits! { $name => [$type; $lanes] } @@ -361,5 +361,14 @@ macro_rules! define_mask_vector { pub const fn new($($var: $type),*) -> Self { Self($($var.0),*) } + }; + { new_from_bool $type:ty | $($var:ident)* } => { + /// Used internally (since we can't use the Into trait in `const fn`s) + #[allow(clippy::too_many_arguments)] + #[allow(unused)] + #[inline] + pub(crate) const fn new_from_bool($($var: bool),*) -> Self { + Self($(<$type>::new($var).0),*) + } } } diff --git a/crates/core_simd/src/masks/mod.rs b/crates/core_simd/src/masks/mod.rs new file mode 100644 index 000000000000..e138a1b4dd80 --- /dev/null +++ b/crates/core_simd/src/masks/mod.rs @@ -0,0 +1,215 @@ +//! Types and traits associated with masking lanes of vectors. + +pub mod wide; + +trait MaskImpl { + type Mask; +} + +impl MaskImpl for [u8; 8] { + type Mask = wide::m8x8; +} + +impl MaskImpl for [u8; 16] { + type Mask = wide::m8x16; +} + +impl MaskImpl for [u8; 32] { + type Mask = wide::m8x32; +} + +impl MaskImpl for [u8; 64] { + type Mask = wide::m8x64; +} + +impl MaskImpl for [u16; 4] { + type Mask = wide::m16x4; +} + +impl MaskImpl for [u16; 8] { + type Mask = wide::m16x8; +} + +impl MaskImpl for [u16; 16] { + type Mask = wide::m16x16; +} + +impl MaskImpl for [u16; 32] { + type Mask = wide::m16x32; +} + +impl MaskImpl for [u32; 2] { + type Mask = wide::m32x2; +} + +impl MaskImpl for [u32; 4] { + type Mask = wide::m32x4; +} + +impl MaskImpl for [u32; 8] { + type Mask = wide::m32x8; +} + +impl MaskImpl for [u32; 16] { + type Mask = wide::m32x16; +} + +impl MaskImpl for [u64; 2] { + type Mask = wide::m64x2; +} + +impl MaskImpl for [u64; 4] { + type Mask = wide::m64x4; +} + +impl MaskImpl for [u64; 8] { + type Mask = wide::m64x8; +} + +impl MaskImpl for [u128; 2] { + type Mask = wide::m128x2; +} + +impl MaskImpl for [u128; 4] { + type Mask = wide::m128x4; +} + +impl MaskImpl for [usize; 2] { + type Mask = wide::msizex2; +} + +impl MaskImpl for [usize; 4] { + type Mask = wide::msizex4; +} + +impl MaskImpl for [usize; 8] { + type Mask = wide::msizex8; +} + +macro_rules! define_opaque_mask { + { + $(#[$attr:meta])* + struct $name:ident([$width:ty; $lanes:tt]); + } => { + $(#[$attr])* + #[allow(non_camel_case_types)] + pub struct $name(<[$width; $lanes] as MaskImpl>::Mask); + + impl $name { + /// Construct a mask by setting all lanes to the given value. + pub fn splat(value: bool) -> Self { + Self(<[$width; $lanes] as MaskImpl>::Mask::splat(value.into())) + } + + call_counting_args! { $lanes => define_opaque_mask => new [$width; $lanes] } + } + }; + { new [$width:ty; $lanes:tt] $($var:ident)* } => { + /// Construct a vector by setting each lane to the given values. + #[allow(clippy::too_many_arguments)] + #[inline] + pub const fn new($($var: bool),*) -> Self { + Self(<[$width; $lanes] as MaskImpl>::Mask::new_from_bool($($var),*)) + } + } +} + +define_opaque_mask! { + /// Mask for 8 8-bit lanes + struct mask8x8([u8; 8]); +} + +define_opaque_mask! { + /// Mask for 16 8-bit lanes + struct mask8x16([u8; 16]); +} + +define_opaque_mask! { + /// Mask for 32 8-bit lanes + struct mask8x32([u8; 32]); +} + +define_opaque_mask! { + /// Mask for 64 8-bit lanes + struct mask8x64([u8; 64]); +} + +define_opaque_mask! { + /// Mask for 4 16-bit lanes + struct mask16x4([u16; 4]); +} + +define_opaque_mask! { + /// Mask for 8 16-bit lanes + struct mask16x8([u16; 8]); +} + +define_opaque_mask! { + /// Mask for 16 16-bit lanes + struct mask16x16([u16; 16]); +} + +define_opaque_mask! { + /// Mask for 32 16-bit lanes + struct mask16x32([u16; 32]); +} + +define_opaque_mask! { + /// Mask for 2 32-bit lanes + struct mask32x2([u32; 2]); +} + +define_opaque_mask! { + /// Mask for 4 32-bit lanes + struct mask32x4([u32; 4]); +} + +define_opaque_mask! { + /// Mask for 8 32-bit lanes + struct mask32x8([u32; 8]); +} + +define_opaque_mask! { + /// Mask for 16 32-bit lanes + struct mask32x16([u32; 16]); +} + +define_opaque_mask! { + /// Mask for 2 64-bit lanes + struct mask64x2([u64; 2]); +} + +define_opaque_mask! { + /// Mask for 4 64-bit lanes + struct mask64x4([u64; 4]); +} + +define_opaque_mask! { + /// Mask for 8 64-bit lanes + struct mask64x8([u64; 8]); +} + +define_opaque_mask! { + /// Mask for 2 128-bit lanes + struct mask128x2([u128; 2]); +} + +define_opaque_mask! { + /// Mask for 4 128-bit lanes + struct mask128x4([u128; 4]); +} + +define_opaque_mask! { + /// Mask for 2 `isize`-wide lanes + struct masksizex2([usize; 2]); +} + +define_opaque_mask! { + /// Mask for 4 `isize`-wide lanes + struct masksizex4([usize; 4]); +} + +define_opaque_mask! { + /// Mask for 8 `isize`-wide lanes + struct masksizex8([usize; 8]); +} diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks/wide/mod.rs similarity index 87% rename from crates/core_simd/src/masks.rs rename to crates/core_simd/src/masks/wide/mod.rs index cba76b6a2a35..1462992197dc 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks/wide/mod.rs @@ -1,3 +1,18 @@ +//! Masks that take up full vector registers. + +mod vectors_m8; +pub use vectors_m8::*; +mod vectors_m16; +pub use vectors_m16::*; +mod vectors_m32; +pub use vectors_m32::*; +mod vectors_m64; +pub use vectors_m64::*; +mod vectors_m128; +pub use vectors_m128::*; +mod vectors_msize; +pub use vectors_msize::*; + /// The error type returned when converting an integer to a mask fails. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct TryFromMaskError(()); @@ -95,30 +110,30 @@ macro_rules! define_mask { define_mask! { /// 8-bit mask - struct mask8(i8); + struct m8(i8); } define_mask! { /// 16-bit mask - struct mask16(i16); + struct m16(i16); } define_mask! { /// 32-bit mask - struct mask32(i32); + struct m32(i32); } define_mask! { /// 64-bit mask - struct mask64(i64); + struct m64(i64); } define_mask! { /// 128-bit mask - struct mask128(i128); + struct m128(i128); } define_mask! { /// `isize`-wide mask - struct masksize(isize); + struct msize(isize); } diff --git a/crates/core_simd/src/masks/wide/vectors_m128.rs b/crates/core_simd/src/masks/wide/vectors_m128.rs new file mode 100644 index 000000000000..fddddac5fc4f --- /dev/null +++ b/crates/core_simd/src/masks/wide/vectors_m128.rs @@ -0,0 +1,11 @@ +use super::m128; + +define_mask_vector! { + /// Vector of two `m128` values + struct m128x2([i128 as m128; 2]); +} + +define_mask_vector! { + /// Vector of four `m128` values + struct m128x4([i128 as m128; 4]); +} diff --git a/crates/core_simd/src/masks/wide/vectors_m16.rs b/crates/core_simd/src/masks/wide/vectors_m16.rs new file mode 100644 index 000000000000..3b05e83f673d --- /dev/null +++ b/crates/core_simd/src/masks/wide/vectors_m16.rs @@ -0,0 +1,21 @@ +use super::m16; + +define_mask_vector! { + /// Vector of four `m16` values + struct m16x4([i16 as m16; 4]); +} + +define_mask_vector! { + /// Vector of eight `m16` values + struct m16x8([i16 as m16; 8]); +} + +define_mask_vector! { + /// Vector of 16 `m16` values + struct m16x16([i16 as m16; 16]); +} + +define_mask_vector! { + /// Vector of 32 `m16` values + struct m16x32([i16 as m16; 32]); +} diff --git a/crates/core_simd/src/masks/wide/vectors_m32.rs b/crates/core_simd/src/masks/wide/vectors_m32.rs new file mode 100644 index 000000000000..de5745fb2833 --- /dev/null +++ b/crates/core_simd/src/masks/wide/vectors_m32.rs @@ -0,0 +1,21 @@ +use super::m32; + +define_mask_vector! { + /// Vector of two `m32` values + struct m32x2([i32 as m32; 2]); +} + +define_mask_vector! { + /// Vector of four `m32` values + struct m32x4([i32 as m32; 4]); +} + +define_mask_vector! { + /// Vector of eight `m32` values + struct m32x8([i32 as m32; 8]); +} + +define_mask_vector! { + /// Vector of 16 `m32` values + struct m32x16([i32 as m32; 16]); +} diff --git a/crates/core_simd/src/masks/wide/vectors_m64.rs b/crates/core_simd/src/masks/wide/vectors_m64.rs new file mode 100644 index 000000000000..55c8687fcfc4 --- /dev/null +++ b/crates/core_simd/src/masks/wide/vectors_m64.rs @@ -0,0 +1,16 @@ +use super::m64; + +define_mask_vector! { + /// Vector of two `m64` values + struct m64x2([i64 as m64; 2]); +} + +define_mask_vector! { + /// Vector of four `m64` values + struct m64x4([i64 as m64; 4]); +} + +define_mask_vector! { + /// Vector of eight `m64` values + struct m64x8([i64 as m64; 8]); +} diff --git a/crates/core_simd/src/masks/wide/vectors_m8.rs b/crates/core_simd/src/masks/wide/vectors_m8.rs new file mode 100644 index 000000000000..149e138739dc --- /dev/null +++ b/crates/core_simd/src/masks/wide/vectors_m8.rs @@ -0,0 +1,21 @@ +use super::m8; + +define_mask_vector! { + /// Vector of eight `m8` values + struct m8x8([i8 as m8; 8]); +} + +define_mask_vector! { + /// Vector of 16 `m8` values + struct m8x16([i8 as m8; 16]); +} + +define_mask_vector! { + /// Vector of 32 `m8` values + struct m8x32([i8 as m8; 32]); +} + +define_mask_vector! { + /// Vector of 64 `m8` values + struct m8x64([i8 as m8; 64]); +} diff --git a/crates/core_simd/src/masks/wide/vectors_msize.rs b/crates/core_simd/src/masks/wide/vectors_msize.rs new file mode 100644 index 000000000000..497aba8ddbbf --- /dev/null +++ b/crates/core_simd/src/masks/wide/vectors_msize.rs @@ -0,0 +1,16 @@ +use super::msize; + +define_mask_vector! { + /// Vector of two `msize` values + struct msizex2([isize as msize; 2]); +} + +define_mask_vector! { + /// Vector of four `msize` values + struct msizex4([isize as msize; 4]); +} + +define_mask_vector! { + /// Vector of eight `msize` values + struct msizex8([isize as msize; 8]); +} diff --git a/crates/core_simd/src/ops.rs b/crates/core_simd/src/ops.rs index 5a186649821b..ac89feca9d66 100644 --- a/crates/core_simd/src/ops.rs +++ b/crates/core_simd/src/ops.rs @@ -96,7 +96,7 @@ macro_rules! impl_ref_ops { } /// Implements op traits for masks -macro_rules! impl_mask_ops { +macro_rules! impl_mask_element_ops { { $($mask:ty),* } => { $( impl_ref_ops! { @@ -161,7 +161,15 @@ macro_rules! impl_mask_ops { )* } } -impl_mask_ops! { crate::mask8, crate::mask16, crate::mask32, crate::mask64, crate::mask128, crate::masksize } + +impl_mask_element_ops! { + crate::masks::wide::m8, + crate::masks::wide::m16, + crate::masks::wide::m32, + crate::masks::wide::m64, + crate::masks::wide::m128, + crate::masks::wide::msize +} /// Automatically implements operators over vectors and scalars for a particular vector. macro_rules! impl_op { @@ -632,10 +640,10 @@ impl_float_ops! { } impl_mask_ops! { - crate::mask8 => crate::mask8x8, crate::mask8x16, crate::mask8x32, crate::mask8x64; - crate::mask16 => crate::mask16x4, crate::mask16x8, crate::mask16x16, crate::mask16x32; - crate::mask32 => crate::mask32x2, crate::mask32x4, crate::mask32x8, crate::mask32x16; - crate::mask64 => crate::mask64x2, crate::mask64x4, crate::mask64x8; - crate::mask128 => crate::mask128x2, crate::mask128x4; - crate::masksize => crate::masksizex2, crate::masksizex4, crate::masksizex8; + crate::masks::wide::m8 => crate::masks::wide::m8x8, crate::masks::wide::m8x16, crate::masks::wide::m8x32, crate::masks::wide::m8x64; + crate::masks::wide::m16 => crate::masks::wide::m16x4, crate::masks::wide::m16x8, crate::masks::wide::m16x16, crate::masks::wide::m16x32; + crate::masks::wide::m32 => crate::masks::wide::m32x2, crate::masks::wide::m32x4, crate::masks::wide::m32x8, crate::masks::wide::m32x16; + crate::masks::wide::m64 => crate::masks::wide::m64x2, crate::masks::wide::m64x4, crate::masks::wide::m64x8; + crate::masks::wide::m128 => crate::masks::wide::m128x2, crate::masks::wide::m128x4; + crate::masks::wide::msize => crate::masks::wide::msizex2, crate::masks::wide::msizex4, crate::masks::wide::msizex8; } diff --git a/crates/core_simd/src/vectors_mask128.rs b/crates/core_simd/src/vectors_mask128.rs deleted file mode 100644 index adf56a3684b3..000000000000 --- a/crates/core_simd/src/vectors_mask128.rs +++ /dev/null @@ -1,11 +0,0 @@ -use crate::mask128; - -define_mask_vector! { - /// Vector of two `mask128` values - struct mask128x2([i128 as mask128; 2]); -} - -define_mask_vector! { - /// Vector of four `mask128` values - struct mask128x4([i128 as mask128; 4]); -} diff --git a/crates/core_simd/src/vectors_mask16.rs b/crates/core_simd/src/vectors_mask16.rs deleted file mode 100644 index 406d7255a11e..000000000000 --- a/crates/core_simd/src/vectors_mask16.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::mask16; - -define_mask_vector! { - /// Vector of four `mask16` values - struct mask16x4([i16 as mask16; 4]); -} - -define_mask_vector! { - /// Vector of eight `mask16` values - struct mask16x8([i16 as mask16; 8]); -} - -define_mask_vector! { - /// Vector of 16 `mask16` values - struct mask16x16([i16 as mask16; 16]); -} - -define_mask_vector! { - /// Vector of 32 `mask16` values - struct mask16x32([i16 as mask16; 32]); -} diff --git a/crates/core_simd/src/vectors_mask32.rs b/crates/core_simd/src/vectors_mask32.rs deleted file mode 100644 index fad191421f38..000000000000 --- a/crates/core_simd/src/vectors_mask32.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::mask32; - -define_mask_vector! { - /// Vector of two `mask32` values - struct mask32x2([i32 as mask32; 2]); -} - -define_mask_vector! { - /// Vector of four `mask32` values - struct mask32x4([i32 as mask32; 4]); -} - -define_mask_vector! { - /// Vector of eight `mask32` values - struct mask32x8([i32 as mask32; 8]); -} - -define_mask_vector! { - /// Vector of 16 `mask32` values - struct mask32x16([i32 as mask32; 16]); -} diff --git a/crates/core_simd/src/vectors_mask64.rs b/crates/core_simd/src/vectors_mask64.rs deleted file mode 100644 index 554e731ccf24..000000000000 --- a/crates/core_simd/src/vectors_mask64.rs +++ /dev/null @@ -1,16 +0,0 @@ -use crate::mask64; - -define_mask_vector! { - /// Vector of two `mask64` values - struct mask64x2([i64 as mask64; 2]); -} - -define_mask_vector! { - /// Vector of four `mask64` values - struct mask64x4([i64 as mask64; 4]); -} - -define_mask_vector! { - /// Vector of eight `mask64` values - struct mask64x8([i64 as mask64; 8]); -} diff --git a/crates/core_simd/src/vectors_mask8.rs b/crates/core_simd/src/vectors_mask8.rs deleted file mode 100644 index d038b3361044..000000000000 --- a/crates/core_simd/src/vectors_mask8.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::mask8; - -define_mask_vector! { - /// Vector of eight `mask8` values - struct mask8x8([i8 as mask8; 8]); -} - -define_mask_vector! { - /// Vector of 16 `mask8` values - struct mask8x16([i8 as mask8; 16]); -} - -define_mask_vector! { - /// Vector of 32 `mask8` values - struct mask8x32([i8 as mask8; 32]); -} - -define_mask_vector! { - /// Vector of 64 `mask8` values - struct mask8x64([i8 as mask8; 64]); -} diff --git a/crates/core_simd/src/vectors_masksize.rs b/crates/core_simd/src/vectors_masksize.rs deleted file mode 100644 index a838aee51985..000000000000 --- a/crates/core_simd/src/vectors_masksize.rs +++ /dev/null @@ -1,16 +0,0 @@ -use crate::masksize; - -define_mask_vector! { - /// Vector of two `masksize` values - struct masksizex2([isize as masksize; 2]); -} - -define_mask_vector! { - /// Vector of four `masksize` values - struct masksizex4([isize as masksize; 4]); -} - -define_mask_vector! { - /// Vector of eight `masksize` values - struct masksizex8([isize as masksize; 8]); -}