From da42aa5403659a6f1f6f4bc4b65d177f13fb6536 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 18 Apr 2021 19:26:07 +0000 Subject: [PATCH 1/4] Begin reducing mask API --- crates/core_simd/src/comparisons.rs | 44 +++-- crates/core_simd/src/lanes_at_most_32.rs | 4 +- crates/core_simd/src/masks/bitmask.rs | 18 +- crates/core_simd/src/masks/full_masks.rs | 203 +---------------------- crates/core_simd/src/masks/mod.rs | 77 +++------ crates/core_simd/src/reduction.rs | 29 +--- crates/core_simd/tests/masks.rs | 23 --- crates/test_helpers/src/lib.rs | 2 - 8 files changed, 59 insertions(+), 341 deletions(-) diff --git a/crates/core_simd/src/comparisons.rs b/crates/core_simd/src/comparisons.rs index 988ff857eab5..f3a1954fda22 100644 --- a/crates/core_simd/src/comparisons.rs +++ b/crates/core_simd/src/comparisons.rs @@ -1,7 +1,7 @@ use crate::LanesAtMost32; macro_rules! implement_mask_ops { - { $($vector:ident => $mask:ident ($inner_mask_ty:ident, $inner_ty:ident),)* } => { + { $($vector:ident => $mask:ident ($inner_ty:ident),)* } => { $( impl crate::$vector where @@ -12,8 +12,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_eq(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_eq(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_eq(self, other)) } } @@ -21,8 +20,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_ne(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_ne(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_ne(self, other)) } } @@ -30,8 +28,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_lt(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_lt(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_lt(self, other)) } } @@ -39,8 +36,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_gt(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_gt(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_gt(self, other)) } } @@ -48,8 +44,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_le(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_le(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_le(self, other)) } } @@ -57,8 +52,7 @@ macro_rules! implement_mask_ops { #[inline] pub fn lanes_ge(self, other: Self) -> crate::$mask { unsafe { - crate::$inner_mask_ty::from_int_unchecked(crate::intrinsics::simd_ge(self, other)) - .into() + crate::$mask::from_int_unchecked(crate::intrinsics::simd_ge(self, other)) } } } @@ -67,18 +61,18 @@ macro_rules! implement_mask_ops { } implement_mask_ops! { - SimdI8 => Mask8 (SimdMask8, SimdI8), - SimdI16 => Mask16 (SimdMask16, SimdI16), - SimdI32 => Mask32 (SimdMask32, SimdI32), - SimdI64 => Mask64 (SimdMask64, SimdI64), - SimdIsize => MaskSize (SimdMaskSize, SimdIsize), + SimdI8 => Mask8 (SimdI8), + SimdI16 => Mask16 (SimdI16), + SimdI32 => Mask32 (SimdI32), + SimdI64 => Mask64 (SimdI64), + SimdIsize => MaskSize (SimdIsize), - SimdU8 => Mask8 (SimdMask8, SimdI8), - SimdU16 => Mask16 (SimdMask16, SimdI16), - SimdU32 => Mask32 (SimdMask32, SimdI32), - SimdU64 => Mask64 (SimdMask64, SimdI64), - SimdUsize => MaskSize (SimdMaskSize, SimdIsize), + SimdU8 => Mask8 (SimdI8), + SimdU16 => Mask16 (SimdI16), + SimdU32 => Mask32 (SimdI32), + SimdU64 => Mask64 (SimdI64), + SimdUsize => MaskSize (SimdIsize), - SimdF32 => Mask32 (SimdMask32, SimdI32), - SimdF64 => Mask64 (SimdMask64, SimdI64), + SimdF32 => Mask32 (SimdI32), + SimdF64 => Mask64 (SimdI64), } diff --git a/crates/core_simd/src/lanes_at_most_32.rs b/crates/core_simd/src/lanes_at_most_32.rs index 1e2f7e952c62..2fee9ca91891 100644 --- a/crates/core_simd/src/lanes_at_most_32.rs +++ b/crates/core_simd/src/lanes_at_most_32.rs @@ -1,4 +1,4 @@ -/// Implemented for bitmask sizes that are supported by the implementation. +/// Implemented for vectors that are supported by the implementation. pub trait LanesAtMost32 {} macro_rules! impl_for { @@ -28,5 +28,3 @@ impl_for! { SimdIsize } impl_for! { SimdF32 } impl_for! { SimdF64 } - -impl_for! { BitMask } diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index b4d1b6d9557d..32e2ffb86153 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -4,14 +4,12 @@ use crate::LanesAtMost32; #[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Hash)] #[repr(transparent)] pub struct BitMask(u64) -where - BitMask: LanesAtMost32; impl BitMask where Self: LanesAtMost32, { - /// Construct a mask by setting all lanes to the given value. + #[inline] pub fn splat(value: bool) -> Self { if value { Self(u64::MAX >> (64 - LANES)) @@ -20,23 +18,13 @@ where } } - /// Tests the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] - pub fn test(&self, lane: usize) -> bool { - assert!(lane < LANES, "lane index out of range"); + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { (self.0 >> lane) & 0x1 > 0 } - /// Sets the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] - pub fn set(&mut self, lane: usize, value: bool) { - assert!(lane < LANES, "lane index out of range"); + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { self.0 ^= ((value ^ self.test(lane)) as u64) << lane } } diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index 60a6cb5fdbe8..6972a4216b68 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -1,18 +1,5 @@ //! Masks that take up full SIMD vector registers. -/// The error type returned when converting an integer to a mask fails. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct TryFromMaskError(()); - -impl core::fmt::Display for TryFromMaskError { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - write!( - f, - "mask vector must have all bits set or unset in each lane" - ) - } -} - macro_rules! define_mask { { $(#[$attr:meta])* @@ -27,6 +14,8 @@ macro_rules! define_mask { where crate::$type: crate::LanesAtMost32; + impl_full_mask_reductions! { $name, $type } + impl Copy for $name where crate::$type: crate::LanesAtMost32, @@ -46,7 +35,6 @@ macro_rules! define_mask { where crate::$type: crate::LanesAtMost32, { - /// Construct a mask by setting all lanes to the given value. pub fn splat(value: bool) -> Self { Self(>::splat( if value { @@ -57,20 +45,12 @@ macro_rules! define_mask { )) } - /// Tests the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] pub fn test(&self, lane: usize) -> bool { assert!(lane < LANES, "lane index out of range"); self.0[lane] == -1 } - /// Sets the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] pub fn set(&mut self, lane: usize, value: bool) { assert!(lane < LANES, "lane index out of range"); @@ -81,56 +61,15 @@ macro_rules! define_mask { } } - /// Converts the mask to the equivalent integer representation, where -1 represents - /// "set" and 0 represents "unset". #[inline] pub fn to_int(self) -> crate::$type { self.0 } - /// Creates a mask from the equivalent integer representation, where -1 represents - /// "set" and 0 represents "unset". - /// - /// Each provided lane must be either 0 or -1. #[inline] pub unsafe fn from_int_unchecked(value: crate::$type) -> Self { Self(value) } - - /// Creates a mask from the equivalent integer representation, where -1 represents - /// "set" and 0 represents "unset". - /// - /// # Panics - /// Panics if any lane is not 0 or -1. - #[inline] - pub fn from_int(value: crate::$type) -> Self { - use core::convert::TryInto; - value.try_into().unwrap() - } - } - - impl core::convert::From for $name - where - crate::$type: crate::LanesAtMost32, - { - fn from(value: bool) -> Self { - Self::splat(value) - } - } - - impl core::convert::TryFrom> for $name - where - crate::$type: crate::LanesAtMost32, - { - type Error = TryFromMaskError; - fn try_from(value: crate::$type) -> Result { - let valid = (value.lanes_eq(crate::$type::::splat(0)) | value.lanes_eq(crate::$type::::splat(-1))).all(); - if valid { - Ok(Self(value)) - } else { - Err(TryFromMaskError(())) - } - } } impl core::convert::From<$name> for crate::$type @@ -142,36 +81,6 @@ macro_rules! define_mask { } } - impl core::convert::From> for $name - where - crate::$type: crate::LanesAtMost32, - crate::BitMask: crate::LanesAtMost32, - { - fn from(value: crate::BitMask) -> Self { - // TODO use an intrinsic to do this efficiently (with LLVM's sext instruction) - let mut mask = Self::splat(false); - for lane in 0..LANES { - mask.set(lane, value.test(lane)); - } - mask - } - } - - impl core::convert::From<$name> for crate::BitMask - where - crate::$type: crate::LanesAtMost32, - crate::BitMask: crate::LanesAtMost32, - { - fn from(value: $name<$lanes>) -> Self { - // TODO use an intrinsic to do this efficiently (with LLVM's trunc instruction) - let mut mask = Self::splat(false); - for lane in 0..LANES { - mask.set(lane, value.test(lane)); - } - mask - } - } - impl core::fmt::Debug for $name where crate::$type: crate::LanesAtMost32, @@ -230,28 +139,6 @@ macro_rules! define_mask { } } - impl core::ops::BitAnd for $name - where - crate::$type: crate::LanesAtMost32, - { - type Output = Self; - #[inline] - fn bitand(self, rhs: bool) -> Self { - self & Self::splat(rhs) - } - } - - impl core::ops::BitAnd<$name> for bool - where - crate::$type: crate::LanesAtMost32, - { - type Output = $name; - #[inline] - fn bitand(self, rhs: $name) -> $name { - $name::::splat(self) & rhs - } - } - impl core::ops::BitOr for $name where crate::$type: crate::LanesAtMost32, @@ -263,28 +150,6 @@ macro_rules! define_mask { } } - impl core::ops::BitOr for $name - where - crate::$type: crate::LanesAtMost32, - { - type Output = Self; - #[inline] - fn bitor(self, rhs: bool) -> Self { - self | Self::splat(rhs) - } - } - - impl core::ops::BitOr<$name> for bool - where - crate::$type: crate::LanesAtMost32, - { - type Output = $name; - #[inline] - fn bitor(self, rhs: $name) -> $name { - $name::::splat(self) | rhs - } - } - impl core::ops::BitXor for $name where crate::$type: crate::LanesAtMost32, @@ -296,28 +161,6 @@ macro_rules! define_mask { } } - impl core::ops::BitXor for $name - where - crate::$type: crate::LanesAtMost32, - { - type Output = Self; - #[inline] - fn bitxor(self, rhs: bool) -> Self::Output { - self ^ Self::splat(rhs) - } - } - - impl core::ops::BitXor<$name> for bool - where - crate::$type: crate::LanesAtMost32, - { - type Output = $name; - #[inline] - fn bitxor(self, rhs: $name) -> Self::Output { - $name::::splat(self) ^ rhs - } - } - impl core::ops::Not for $name where crate::$type: crate::LanesAtMost32, @@ -339,16 +182,6 @@ macro_rules! define_mask { } } - impl core::ops::BitAndAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitand_assign(&mut self, rhs: bool) { - *self &= Self::splat(rhs); - } - } - impl core::ops::BitOrAssign for $name where crate::$type: crate::LanesAtMost32, @@ -359,16 +192,6 @@ macro_rules! define_mask { } } - impl core::ops::BitOrAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitor_assign(&mut self, rhs: bool) { - *self |= Self::splat(rhs); - } - } - impl core::ops::BitXorAssign for $name where crate::$type: crate::LanesAtMost32, @@ -378,47 +201,35 @@ macro_rules! define_mask { self.0 ^= rhs.0; } } - - impl core::ops::BitXorAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitxor_assign(&mut self, rhs: bool) { - *self ^= Self::splat(rhs); - } - } - - impl_full_mask_reductions! { $name, $type } } } define_mask! { /// A mask equivalent to [SimdI8](crate::SimdI8), where all bits in the lane must be either set /// or unset. - struct SimdMask8(crate::SimdI8); + struct Mask8(crate::SimdI8); } define_mask! { /// A mask equivalent to [SimdI16](crate::SimdI16), where all bits in the lane must be either set /// or unset. - struct SimdMask16(crate::SimdI16); + struct Mask16(crate::SimdI16); } define_mask! { /// A mask equivalent to [SimdI32](crate::SimdI32), where all bits in the lane must be either set /// or unset. - struct SimdMask32(crate::SimdI32); + struct Mask32(crate::SimdI32); } define_mask! { /// A mask equivalent to [SimdI64](crate::SimdI64), where all bits in the lane must be either set /// or unset. - struct SimdMask64(crate::SimdI64); + struct Mask64(crate::SimdI64); } define_mask! { /// A mask equivalent to [SimdIsize](crate::SimdIsize), where all bits in the lane must be either set /// or unset. - struct SimdMaskSize(crate::SimdIsize); + struct MaskSize(crate::SimdIsize); } diff --git a/crates/core_simd/src/masks/mod.rs b/crates/core_simd/src/masks/mod.rs index c394c7003a35..fbb934b96424 100644 --- a/crates/core_simd/src/masks/mod.rs +++ b/crates/core_simd/src/masks/mod.rs @@ -1,25 +1,24 @@ //! Types and traits associated with masking lanes of vectors. +//! Types representing #![allow(non_camel_case_types)] -mod full_masks; -pub use full_masks::*; - -mod bitmask; -pub use bitmask::*; +#[cfg_attr(not(all(target_arch = "x86_64", target_feature = "avx512f")), path = "full_masks.rs")] +#[cfg_attr(all(target_arch = "x86_64", target_feature = "avx512f"), path = "bitmask.rs")] +mod mask_impl; use crate::{LanesAtMost32, SimdI16, SimdI32, SimdI64, SimdI8, SimdIsize}; macro_rules! define_opaque_mask { { $(#[$attr:meta])* - struct $name:ident($inner_ty:ident<$lanes2:ident>); + struct $name:ident($inner_ty:ty); @bits $bits_ty:ident } => { $(#[$attr])* #[allow(non_camel_case_types)] - pub struct $name($inner_ty) where $bits_ty: LanesAtMost32; + pub struct $name($inner_ty) where $bits_ty: LanesAtMost32; - impl_opaque_mask_reductions! { $name, $inner_ty, $bits_ty } + impl_opaque_mask_reductions! { $name, $bits_ty } impl $name where @@ -27,7 +26,7 @@ macro_rules! define_opaque_mask { { /// Construct a mask by setting all lanes to the given value. pub fn splat(value: bool) -> Self { - Self(<$inner_ty>::splat(value)) + Self(<$inner_ty>::splat(value)) } /// Converts an array to a SIMD vector. @@ -52,6 +51,16 @@ macro_rules! define_opaque_mask { array } + /// Converts a vector of integers to a mask, where 0 represents `false` and -1 + /// represents `true`. + /// + /// # Safety + /// All lanes must be either 0 or -1. + #[inline] + pub unsafe fn from_int_unchecked(value: $bits_ty) -> Self { + Self(<$inner_ty>::from_int_unchecked(value)) + } + /// Tests the value of the specified lane. /// /// # Panics @@ -71,44 +80,6 @@ macro_rules! define_opaque_mask { } } - impl From> for $name - where - $bits_ty: LanesAtMost32, - BitMask: LanesAtMost32, - { - fn from(value: BitMask) -> Self { - Self(value.into()) - } - } - - impl From<$name> for crate::BitMask - where - $bits_ty: LanesAtMost32, - BitMask: LanesAtMost32, - { - fn from(value: $name) -> Self { - value.0.into() - } - } - - impl From<$inner_ty> for $name - where - $bits_ty: LanesAtMost32, - { - fn from(value: $inner_ty) -> Self { - Self(value) - } - } - - impl From<$name> for $inner_ty - where - $bits_ty: LanesAtMost32, - { - fn from(value: $name) -> Self { - value.0 - } - } - // vector/array conversion impl From<[bool; LANES]> for $name where @@ -130,7 +101,7 @@ macro_rules! define_opaque_mask { impl Copy for $name where - $inner_ty: Copy, + $inner_ty: Copy, $bits_ty: LanesAtMost32, {} @@ -359,7 +330,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 8-bit elements. /// /// The layout of this type is unspecified. - struct Mask8(SimdMask8); + struct Mask8(mask_impl::Mask8); @bits SimdI8 } @@ -367,7 +338,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 16-bit elements. /// /// The layout of this type is unspecified. - struct Mask16(SimdMask16); + struct Mask16(mask_impl::Mask16); @bits SimdI16 } @@ -375,7 +346,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 32-bit elements. /// /// The layout of this type is unspecified. - struct Mask32(SimdMask32); + struct Mask32(mask_impl::Mask32); @bits SimdI32 } @@ -383,7 +354,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 64-bit elements. /// /// The layout of this type is unspecified. - struct Mask64(SimdMask64); + struct Mask64(mask_impl::Mask64); @bits SimdI64 } @@ -391,7 +362,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` pointer-width elements. /// /// The layout of this type is unspecified. - struct MaskSize(SimdMaskSize); + struct MaskSize(mask_impl::MaskSize); @bits SimdIsize } diff --git a/crates/core_simd/src/reduction.rs b/crates/core_simd/src/reduction.rs index 382d366dd3d1..2d4f1bca2647 100644 --- a/crates/core_simd/src/reduction.rs +++ b/crates/core_simd/src/reduction.rs @@ -103,18 +103,16 @@ macro_rules! impl_float_reductions { } macro_rules! impl_full_mask_reductions { - { $name:ident, $inner:ident } => { - impl crate::$name + { $name:ident, $bits_ty:ident } => { + impl $name where - crate::$inner: crate::LanesAtMost32 + crate::$bits_ty: crate::LanesAtMost32 { - /// Returns true if any lane is set, or false otherwise. #[inline] pub fn any(self) -> bool { unsafe { crate::intrinsics::simd_reduce_any(self.to_int()) } } - /// Returns true if all lanes are set, or false otherwise. #[inline] pub fn all(self) -> bool { unsafe { crate::intrinsics::simd_reduce_all(self.to_int()) } @@ -124,10 +122,10 @@ macro_rules! impl_full_mask_reductions { } macro_rules! impl_opaque_mask_reductions { - { $name:ident, $inner:ident, $bits_ty:ident } => { + { $name:ident, $bits_ty:ident } => { impl $name where - $bits_ty: crate::LanesAtMost32 + crate::$bits_ty: crate::LanesAtMost32 { /// Returns true if any lane is set, or false otherwise. #[inline] @@ -143,20 +141,3 @@ macro_rules! impl_opaque_mask_reductions { } } } - -impl crate::BitMask -where - crate::BitMask: crate::LanesAtMost32, -{ - /// Returns true if any lane is set, or false otherwise. - #[inline] - pub fn any(self) -> bool { - self != Self::splat(false) - } - - /// Returns true if all lanes are set, or false otherwise. - #[inline] - pub fn all(self) -> bool { - self == Self::splat(true) - } -} diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index 59da77de622b..6c3993e39a93 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -1,30 +1,9 @@ -use core::convert::TryFrom; -use core_simd::{BitMask, Mask8, SimdI8, SimdMask8}; - #[cfg(target_arch = "wasm32")] use wasm_bindgen_test::*; #[cfg(target_arch = "wasm32")] wasm_bindgen_test_configure!(run_in_browser); -#[test] -#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] -fn mask_format_round_trip() { - let ints = SimdI8::from_array([-1, 0, 0, -1]); - - let simd_mask = SimdMask8::try_from(ints).unwrap(); - - let bitmask = BitMask::from(simd_mask); - - let opaque_mask = Mask8::from(bitmask); - - let simd_mask_returned = SimdMask8::from(opaque_mask); - - let ints_returned = SimdI8::from(simd_mask_returned); - - assert_eq!(ints_returned, ints); -} - macro_rules! test_mask_api { { $name:ident } => { #[allow(non_snake_case)] @@ -83,6 +62,4 @@ macro_rules! test_mask_api { mod mask_api { test_mask_api! { Mask8 } - test_mask_api! { SimdMask8 } - test_mask_api! { BitMask } } diff --git a/crates/test_helpers/src/lib.rs b/crates/test_helpers/src/lib.rs index 9e8790842b49..8b56877967cb 100644 --- a/crates/test_helpers/src/lib.rs +++ b/crates/test_helpers/src/lib.rs @@ -281,7 +281,6 @@ macro_rules! test_lanes { core_simd::SimdIsize<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF32<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF64<$lanes>: core_simd::LanesAtMost32, - core_simd::BitMask<$lanes>: core_simd::LanesAtMost32, $body #[cfg(target_arch = "wasm32")] @@ -351,7 +350,6 @@ macro_rules! test_lanes_panic { core_simd::SimdIsize<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF32<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF64<$lanes>: core_simd::LanesAtMost32, - core_simd::BitMask<$lanes>: core_simd::LanesAtMost32, $body #[test] From eec42808aa024d354bb40ec890612c37ba4a496c Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Mon, 19 Apr 2021 04:31:43 +0000 Subject: [PATCH 2/4] Update bitmask API --- crates/core_simd/src/intrinsics.rs | 3 + crates/core_simd/src/lanes_at_most_32.rs | 38 ++++-- crates/core_simd/src/masks/bitmask.rs | 146 ++++++++--------------- crates/core_simd/src/masks/full_masks.rs | 59 ++------- crates/core_simd/src/masks/mod.rs | 93 ++++++++++++++- crates/core_simd/tests/masks.rs | 17 +++ 6 files changed, 196 insertions(+), 160 deletions(-) diff --git a/crates/core_simd/src/intrinsics.rs b/crates/core_simd/src/intrinsics.rs index 665dc1a51d74..1812a9c624dc 100644 --- a/crates/core_simd/src/intrinsics.rs +++ b/crates/core_simd/src/intrinsics.rs @@ -76,6 +76,9 @@ extern "platform-intrinsic" { pub(crate) fn simd_reduce_and(x: T) -> U; pub(crate) fn simd_reduce_or(x: T) -> U; pub(crate) fn simd_reduce_xor(x: T) -> U; + + // truncate integer vector to bitmask + pub(crate) fn simd_bitmask(x: T) -> U; } #[cfg(feature = "std")] diff --git a/crates/core_simd/src/lanes_at_most_32.rs b/crates/core_simd/src/lanes_at_most_32.rs index 2fee9ca91891..2d84b1306ea5 100644 --- a/crates/core_simd/src/lanes_at_most_32.rs +++ b/crates/core_simd/src/lanes_at_most_32.rs @@ -1,14 +1,38 @@ /// Implemented for vectors that are supported by the implementation. -pub trait LanesAtMost32 {} +pub trait LanesAtMost32: sealed::Sealed { + #[doc(hidden)] + type BitMask: Into; +} + +mod sealed { + pub trait Sealed {} +} macro_rules! impl_for { { $name:ident } => { - impl LanesAtMost32 for $name<1> {} - impl LanesAtMost32 for $name<2> {} - impl LanesAtMost32 for $name<4> {} - impl LanesAtMost32 for $name<8> {} - impl LanesAtMost32 for $name<16> {} - impl LanesAtMost32 for $name<32> {} + impl sealed::Sealed for $name + where + $name: LanesAtMost32, + {} + + impl LanesAtMost32 for $name<1> { + type BitMask = u8; + } + impl LanesAtMost32 for $name<2> { + type BitMask = u8; + } + impl LanesAtMost32 for $name<4> { + type BitMask = u8; + } + impl LanesAtMost32 for $name<8> { + type BitMask = u8; + } + impl LanesAtMost32 for $name<16> { + type BitMask = u16; + } + impl LanesAtMost32 for $name<32> { + type BitMask = u32; + } } } diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index 32e2ffb86153..bf7c70c5a3ad 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -1,13 +1,9 @@ -use crate::LanesAtMost32; - /// A mask where each lane is represented by a single bit. #[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Hash)] #[repr(transparent)] -pub struct BitMask(u64) +pub struct BitMask(u64); impl BitMask -where - Self: LanesAtMost32, { #[inline] pub fn splat(value: bool) -> Self { @@ -25,13 +21,50 @@ where #[inline] pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { - self.0 ^= ((value ^ self.test(lane)) as u64) << lane + self.0 ^= ((value ^ self.test_unchecked(lane)) as u64) << lane + } + + #[inline] + pub fn to_int(self) -> V + where + V: Default + AsMut<[T; LANES]>, + T: From, + { + // TODO this should be an intrinsic sign-extension + let mut v = V::default(); + for i in 0..LANES { + let lane = unsafe { self.test_unchecked(i) }; + v.as_mut()[i] = (-(lane as i8)).into(); + } + v + } + + #[inline] + pub unsafe fn from_int_unchecked(value: V) -> Self + where + V: crate::LanesAtMost32, + { + let mask: V::BitMask = crate::intrinsics::simd_bitmask(value); + Self(mask.into()) + } + + #[inline] + pub fn to_bitmask(self) -> u64 { + self.0 + } + + #[inline] + pub fn any(self) -> bool { + self != Self::splat(false) + } + + #[inline] + pub fn all(self) -> bool { + self == Self::splat(true) } } impl core::ops::BitAnd for BitMask -where - Self: LanesAtMost32, { type Output = Self; #[inline] @@ -41,8 +74,6 @@ where } impl core::ops::BitAnd for BitMask -where - Self: LanesAtMost32, { type Output = Self; #[inline] @@ -52,8 +83,6 @@ where } impl core::ops::BitAnd> for bool -where - BitMask: LanesAtMost32, { type Output = BitMask; #[inline] @@ -63,8 +92,6 @@ where } impl core::ops::BitOr for BitMask -where - Self: LanesAtMost32, { type Output = Self; #[inline] @@ -73,31 +100,7 @@ where } } -impl core::ops::BitOr for BitMask -where - Self: LanesAtMost32, -{ - type Output = Self; - #[inline] - fn bitor(self, rhs: bool) -> Self { - self | Self::splat(rhs) - } -} - -impl core::ops::BitOr> for bool -where - BitMask: LanesAtMost32, -{ - type Output = BitMask; - #[inline] - fn bitor(self, rhs: BitMask) -> BitMask { - BitMask::::splat(self) | rhs - } -} - impl core::ops::BitXor for BitMask -where - Self: LanesAtMost32, { type Output = Self; #[inline] @@ -106,42 +109,16 @@ where } } -impl core::ops::BitXor for BitMask -where - Self: LanesAtMost32, -{ - type Output = Self; - #[inline] - fn bitxor(self, rhs: bool) -> Self::Output { - self ^ Self::splat(rhs) - } -} - -impl core::ops::BitXor> for bool -where - BitMask: LanesAtMost32, -{ - type Output = BitMask; - #[inline] - fn bitxor(self, rhs: BitMask) -> Self::Output { - BitMask::::splat(self) ^ rhs - } -} - impl core::ops::Not for BitMask -where - Self: LanesAtMost32, { type Output = BitMask; #[inline] fn not(self) -> Self::Output { - Self(!self.0) + Self(!self.0) & Self::splat(true) } } impl core::ops::BitAndAssign for BitMask -where - Self: LanesAtMost32, { #[inline] fn bitand_assign(&mut self, rhs: Self) { @@ -149,19 +126,7 @@ where } } -impl core::ops::BitAndAssign for BitMask -where - Self: LanesAtMost32, -{ - #[inline] - fn bitand_assign(&mut self, rhs: bool) { - *self &= Self::splat(rhs); - } -} - impl core::ops::BitOrAssign for BitMask -where - Self: LanesAtMost32, { #[inline] fn bitor_assign(&mut self, rhs: Self) { @@ -169,19 +134,7 @@ where } } -impl core::ops::BitOrAssign for BitMask -where - Self: LanesAtMost32, -{ - #[inline] - fn bitor_assign(&mut self, rhs: bool) { - *self |= Self::splat(rhs); - } -} - impl core::ops::BitXorAssign for BitMask -where - Self: LanesAtMost32, { #[inline] fn bitxor_assign(&mut self, rhs: Self) { @@ -189,12 +142,9 @@ where } } -impl core::ops::BitXorAssign for BitMask -where - Self: LanesAtMost32, -{ - #[inline] - fn bitxor_assign(&mut self, rhs: bool) { - *self ^= Self::splat(rhs); - } -} +pub type Mask8 = BitMask; +pub type Mask16 = BitMask; +pub type Mask32 = BitMask; +pub type Mask64 = BitMask; +pub type Mask128 = BitMask; +pub type MaskSize = BitMask; diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index 6972a4216b68..2d1ddd6dc305 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -46,14 +46,12 @@ macro_rules! define_mask { } #[inline] - pub fn test(&self, lane: usize) -> bool { - assert!(lane < LANES, "lane index out of range"); + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { self.0[lane] == -1 } #[inline] - pub fn set(&mut self, lane: usize, value: bool) { - assert!(lane < LANES, "lane index out of range"); + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { self.0[lane] = if value { -1 } else { @@ -70,6 +68,12 @@ macro_rules! define_mask { pub unsafe fn from_int_unchecked(value: crate::$type) -> Self { Self(value) } + + #[inline] + pub fn to_bitmask(self) -> u64 { + let mask: as crate::LanesAtMost32>::BitMask = unsafe { crate::intrinsics::simd_bitmask(self.0) }; + mask.into() + } } impl core::convert::From<$name> for crate::$type @@ -81,53 +85,6 @@ macro_rules! define_mask { } } - impl core::fmt::Debug for $name - where - crate::$type: crate::LanesAtMost32, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - f.debug_list() - .entries((0..LANES).map(|lane| self.test(lane))) - .finish() - } - } - - impl core::fmt::Binary for $name - where - crate::$type: crate::LanesAtMost32, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - core::fmt::Binary::fmt(&self.0, f) - } - } - - impl core::fmt::Octal for $name - where - crate::$type: crate::LanesAtMost32, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - core::fmt::Octal::fmt(&self.0, f) - } - } - - impl core::fmt::LowerHex for $name - where - crate::$type: crate::LanesAtMost32, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - core::fmt::LowerHex::fmt(&self.0, f) - } - } - - impl core::fmt::UpperHex for $name - where - crate::$type: crate::LanesAtMost32, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - core::fmt::UpperHex::fmt(&self.0, f) - } - } - impl core::ops::BitAnd for $name where crate::$type: crate::LanesAtMost32, diff --git a/crates/core_simd/src/masks/mod.rs b/crates/core_simd/src/masks/mod.rs index fbb934b96424..e5352ef4d1a2 100644 --- a/crates/core_simd/src/masks/mod.rs +++ b/crates/core_simd/src/masks/mod.rs @@ -8,6 +8,12 @@ mod mask_impl; use crate::{LanesAtMost32, SimdI16, SimdI32, SimdI64, SimdI8, SimdIsize}; +/// Converts masks to bitmasks, with one bit set for each lane. +pub trait ToBitMask { + /// Converts this mask to a bitmask. + fn to_bitmask(self) -> u64; +} + macro_rules! define_opaque_mask { { $(#[$attr:meta])* @@ -61,13 +67,53 @@ macro_rules! define_opaque_mask { Self(<$inner_ty>::from_int_unchecked(value)) } + /// Converts a vector of integers to a mask, where 0 represents `false` and -1 + /// represents `true`. + /// + /// # Panics + /// Panics if any lane is not 0 or -1. + #[inline] + pub fn from_int(value: $bits_ty) -> Self { + assert!( + (value.lanes_eq($bits_ty::splat(0)) | value.lanes_eq($bits_ty::splat(-1))).all(), + "all values must be either 0 or -1", + ); + unsafe { Self::from_int_unchecked(value) } + } + + /// Converts the mask to a vector of integers, where 0 represents `false` and -1 + /// represents `true`. + #[inline] + pub fn to_int(self) -> $bits_ty { + self.0.to_int() + } + + /// Tests the value of the specified lane. + /// + /// # Safety + /// `lane` must be less than `LANES`. + #[inline] + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + self.0.test_unchecked(lane) + } + /// Tests the value of the specified lane. /// /// # Panics /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] pub fn test(&self, lane: usize) -> bool { - self.0.test(lane) + assert!(lane < LANES, "lane index out of range"); + unsafe { self.test_unchecked(lane) } + } + + /// Sets the value of the specified lane. + /// + /// # Safety + /// `lane` must be less than `LANES`. + #[inline] + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + self.0.set_unchecked(lane, value); } /// Sets the value of the specified lane. @@ -76,7 +122,44 @@ macro_rules! define_opaque_mask { /// Panics if `lane` is greater than or equal to the number of lanes in the vector. #[inline] pub fn set(&mut self, lane: usize, value: bool) { - self.0.set(lane, value); + assert!(lane < LANES, "lane index out of range"); + unsafe { self.set_unchecked(lane, value); } + } + } + + impl ToBitMask for $name<1> { + fn to_bitmask(self) -> u64 { + self.0.to_bitmask() + } + } + + impl ToBitMask for $name<2> { + fn to_bitmask(self) -> u64 { + self.0.to_bitmask() + } + } + + impl ToBitMask for $name<4> { + fn to_bitmask(self) -> u64 { + self.0.to_bitmask() + } + } + + impl ToBitMask for $name<8> { + fn to_bitmask(self) -> u64 { + self.0.to_bitmask() + } + } + + impl ToBitMask for $name<16> { + fn to_bitmask(self) -> u64 { + self.0.to_bitmask() + } + } + + impl ToBitMask for $name<32> { + fn to_bitmask(self) -> u64 { + self.0.to_bitmask() } } @@ -147,10 +230,12 @@ macro_rules! define_opaque_mask { impl core::fmt::Debug for $name where - $bits_ty: LanesAtMost32, + $bits_ty: crate::LanesAtMost32, { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - core::fmt::Debug::fmt(&self.0, f) + f.debug_list() + .entries((0..LANES).map(|lane| self.test(lane))) + .finish() } } diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index 6c3993e39a93..be83f4c2ec77 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -56,6 +56,23 @@ macro_rules! test_mask_api { v.set(2, true); assert!(!v.all()); } + + #[test] + fn roundtrip_int_conversion() { + let values = [true, false, false, true, false, false, true, false]; + let mask = core_simd::$name::<8>::from_array(values); + let int = mask.to_int(); + assert_eq!(int.to_array(), [-1, 0, 0, -1, 0, 0, -1, 0]); + assert_eq!(core_simd::$name::<8>::from_int(int), mask); + } + + #[test] + fn to_bitmask() { + use core_simd::ToBitMask; + let values = [true, false, false, true, false, false, true, false]; + let mask = core_simd::$name::<8>::from_array(values); + assert_eq!(mask.to_bitmask(), 0b01001001); + } } } } From 98dad135268e0da590a162d24da6f7e2d8781648 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Thu, 29 Apr 2021 02:41:09 +0000 Subject: [PATCH 3/4] Make implementation more scalable by using a helper trait to determine bitmask size. Improve bitmask to int conversion. --- crates/core_simd/src/comparisons.rs | 1 + crates/core_simd/src/intrinsics.rs | 3 + crates/core_simd/src/masks/bitmask.rs | 191 +++++++++++++---------- crates/core_simd/src/masks/full_masks.rs | 125 ++++++++------- crates/core_simd/src/masks/mod.rs | 146 ++++++++++------- crates/core_simd/src/reduction.rs | 5 +- crates/core_simd/src/vector/float.rs | 1 + crates/core_simd/src/vector/int.rs | 1 + crates/core_simd/src/vector/uint.rs | 1 - crates/core_simd/tests/masks.rs | 10 +- crates/test_helpers/src/array.rs | 5 +- crates/test_helpers/src/lib.rs | 10 ++ 12 files changed, 293 insertions(+), 206 deletions(-) diff --git a/crates/core_simd/src/comparisons.rs b/crates/core_simd/src/comparisons.rs index f3a1954fda22..e8d11406c097 100644 --- a/crates/core_simd/src/comparisons.rs +++ b/crates/core_simd/src/comparisons.rs @@ -7,6 +7,7 @@ macro_rules! implement_mask_ops { where crate::$vector: LanesAtMost32, crate::$inner_ty: LanesAtMost32, + crate::$mask: crate::Mask, { /// Test if each lane is equal to the corresponding lane in `other`. #[inline] diff --git a/crates/core_simd/src/intrinsics.rs b/crates/core_simd/src/intrinsics.rs index 1812a9c624dc..8cbb0cbccf79 100644 --- a/crates/core_simd/src/intrinsics.rs +++ b/crates/core_simd/src/intrinsics.rs @@ -79,6 +79,9 @@ extern "platform-intrinsic" { // truncate integer vector to bitmask pub(crate) fn simd_bitmask(x: T) -> U; + + // select + pub(crate) fn simd_select_bitmask(m: T, a: U, b: U) -> U; } #[cfg(feature = "std")] diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index bf7c70c5a3ad..6bcb08cf9dbb 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -1,42 +1,80 @@ -/// A mask where each lane is represented by a single bit. -#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Hash)] -#[repr(transparent)] -pub struct BitMask(u64); +use crate::Mask; +use core::marker::PhantomData; -impl BitMask -{ +/// Helper trait for limiting int conversion types +pub trait ConvertToInt {} +impl ConvertToInt for crate::SimdI8 where Self: crate::LanesAtMost32 {} +impl ConvertToInt for crate::SimdI16 where Self: crate::LanesAtMost32 {} +impl ConvertToInt for crate::SimdI32 where Self: crate::LanesAtMost32 {} +impl ConvertToInt for crate::SimdI64 where Self: crate::LanesAtMost32 {} +impl ConvertToInt for crate::SimdIsize where Self: crate::LanesAtMost32 {} + +/// A mask where each lane is represented by a single bit. +#[repr(transparent)] +pub struct BitMask(T::BitMask, PhantomData<[(); LANES]>); + +impl Copy for BitMask {} + +impl Clone for BitMask { + fn clone(&self) -> Self { + *self + } +} + +impl PartialEq for BitMask { + fn eq(&self, other: &Self) -> bool { + self.0.as_ref() == other.0.as_ref() + } +} + +impl PartialOrd for BitMask { + fn partial_cmp(&self, other: &Self) -> Option { + self.0.as_ref().partial_cmp(other.0.as_ref()) + } +} + +impl Eq for BitMask {} + +impl Ord for BitMask { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.0.as_ref().cmp(other.0.as_ref()) + } +} + +impl BitMask { #[inline] pub fn splat(value: bool) -> Self { + let mut mask = T::BitMask::default(); if value { - Self(u64::MAX >> (64 - LANES)) + mask.as_mut().fill(u8::MAX) } else { - Self(u64::MIN) + mask.as_mut().fill(u8::MIN) } + if LANES % 8 > 0 { + *mask.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - LANES % 8); + } + Self(mask, PhantomData) } #[inline] pub unsafe fn test_unchecked(&self, lane: usize) -> bool { - (self.0 >> lane) & 0x1 > 0 + (self.0.as_ref()[lane / 8] >> lane % 8) & 0x1 > 0 } #[inline] pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { - self.0 ^= ((value ^ self.test_unchecked(lane)) as u64) << lane + self.0.as_mut()[lane / 8] ^= ((value ^ self.test_unchecked(lane)) as u8) << (lane % 8) } #[inline] - pub fn to_int(self) -> V + pub fn to_int(self) -> V where - V: Default + AsMut<[T; LANES]>, - T: From, + V: ConvertToInt + Default + core::ops::Not, { - // TODO this should be an intrinsic sign-extension - let mut v = V::default(); - for i in 0..LANES { - let lane = unsafe { self.test_unchecked(i) }; - v.as_mut()[i] = (-(lane as i8)).into(); + unsafe { + let mask: T::IntBitMask = core::mem::transmute_copy(&self); + crate::intrinsics::simd_select_bitmask(mask, !V::default(), V::default()) } - v } #[inline] @@ -44,13 +82,22 @@ impl BitMask where V: crate::LanesAtMost32, { - let mask: V::BitMask = crate::intrinsics::simd_bitmask(value); - Self(mask.into()) + // TODO remove the transmute when rustc is more flexible + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + let mask: T::IntBitMask = crate::intrinsics::simd_bitmask(value); + Self(core::mem::transmute_copy(&mask), PhantomData) } #[inline] - pub fn to_bitmask(self) -> u64 { - self.0 + pub fn to_bitmask(self) -> U::BitMask { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + unsafe { core::mem::transmute_copy(&self.0) } } #[inline] @@ -64,87 +111,61 @@ impl BitMask } } -impl core::ops::BitAnd for BitMask +impl core::ops::BitAnd for BitMask +where + T::BitMask: Default + AsRef<[u8]> + AsMut<[u8]>, { type Output = Self; #[inline] - fn bitand(self, rhs: Self) -> Self { - Self(self.0 & rhs.0) + fn bitand(mut self, rhs: Self) -> Self { + for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { + *l &= r; + } + self } } -impl core::ops::BitAnd for BitMask +impl core::ops::BitOr for BitMask +where + T::BitMask: Default + AsRef<[u8]> + AsMut<[u8]>, { type Output = Self; #[inline] - fn bitand(self, rhs: bool) -> Self { - self & Self::splat(rhs) + fn bitor(mut self, rhs: Self) -> Self { + for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { + *l |= r; + } + self } } -impl core::ops::BitAnd> for bool -{ - type Output = BitMask; - #[inline] - fn bitand(self, rhs: BitMask) -> BitMask { - BitMask::::splat(self) & rhs - } -} - -impl core::ops::BitOr for BitMask -{ +impl core::ops::BitXor for BitMask { type Output = Self; #[inline] - fn bitor(self, rhs: Self) -> Self { - Self(self.0 | rhs.0) + fn bitxor(mut self, rhs: Self) -> Self::Output { + for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { + *l ^= r; + } + self } } -impl core::ops::BitXor for BitMask -{ +impl core::ops::Not for BitMask { type Output = Self; #[inline] - fn bitxor(self, rhs: Self) -> Self::Output { - Self(self.0 ^ rhs.0) + fn not(mut self) -> Self::Output { + for x in self.0.as_mut() { + *x = !*x; + } + if LANES % 8 > 0 { + *self.0.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - LANES % 8); + } + self } } -impl core::ops::Not for BitMask -{ - type Output = BitMask; - #[inline] - fn not(self) -> Self::Output { - Self(!self.0) & Self::splat(true) - } -} - -impl core::ops::BitAndAssign for BitMask -{ - #[inline] - fn bitand_assign(&mut self, rhs: Self) { - self.0 &= rhs.0; - } -} - -impl core::ops::BitOrAssign for BitMask -{ - #[inline] - fn bitor_assign(&mut self, rhs: Self) { - self.0 |= rhs.0; - } -} - -impl core::ops::BitXorAssign for BitMask -{ - #[inline] - fn bitxor_assign(&mut self, rhs: Self) { - self.0 ^= rhs.0; - } -} - -pub type Mask8 = BitMask; -pub type Mask16 = BitMask; -pub type Mask32 = BitMask; -pub type Mask64 = BitMask; -pub type Mask128 = BitMask; -pub type MaskSize = BitMask; +pub type Mask8 = BitMask; +pub type Mask16 = BitMask; +pub type Mask32 = BitMask; +pub type Mask64 = BitMask; +pub type MaskSize = BitMask; diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index 2d1ddd6dc305..bd52a25551e2 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -1,5 +1,8 @@ //! Masks that take up full SIMD vector registers. +use crate::Mask; +use core::marker::PhantomData; + macro_rules! define_mask { { $(#[$attr:meta])* @@ -8,20 +11,19 @@ macro_rules! define_mask { ); } => { $(#[$attr])* - #[derive(Default, PartialEq, PartialOrd, Eq, Ord, Hash)] #[repr(transparent)] - pub struct $name(crate::$type<$lanes2>) + pub struct $name(crate::$type<$lanes2>, PhantomData) where crate::$type: crate::LanesAtMost32; impl_full_mask_reductions! { $name, $type } - impl Copy for $name + impl Copy for $name where crate::$type: crate::LanesAtMost32, {} - impl Clone for $name + impl Clone for $name where crate::$type: crate::LanesAtMost32, { @@ -31,18 +33,53 @@ macro_rules! define_mask { } } - impl $name + impl PartialEq for $name + where + crate::$type: crate::LanesAtMost32, + { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + + impl PartialOrd for $name + where + crate::$type: crate::LanesAtMost32, + { + fn partial_cmp(&self, other: &Self) -> Option { + self.0.partial_cmp(&other.0) + } + } + + impl Eq for $name + where + crate::$type: crate::LanesAtMost32, + {} + + impl Ord for $name + where + crate::$type: crate::LanesAtMost32, + { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.0.cmp(&other.0) + } + } + + impl $name where crate::$type: crate::LanesAtMost32, { pub fn splat(value: bool) -> Self { - Self(>::splat( - if value { - -1 - } else { - 0 - } - )) + Self( + >::splat( + if value { + -1 + } else { + 0 + } + ), + PhantomData, + ) } #[inline] @@ -66,96 +103,70 @@ macro_rules! define_mask { #[inline] pub unsafe fn from_int_unchecked(value: crate::$type) -> Self { - Self(value) + Self(value, PhantomData) } #[inline] - pub fn to_bitmask(self) -> u64 { - let mask: as crate::LanesAtMost32>::BitMask = unsafe { crate::intrinsics::simd_bitmask(self.0) }; - mask.into() + pub fn to_bitmask(self) -> U::BitMask { + unsafe { + // TODO remove the transmute when rustc is more flexible + assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + let mask: U::IntBitMask = crate::intrinsics::simd_bitmask(self.0); + core::mem::transmute_copy(&mask) + } } } - impl core::convert::From<$name> for crate::$type + impl core::convert::From<$name> for crate::$type where crate::$type: crate::LanesAtMost32, { - fn from(value: $name) -> Self { + fn from(value: $name) -> Self { value.0 } } - impl core::ops::BitAnd for $name + impl core::ops::BitAnd for $name where crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] fn bitand(self, rhs: Self) -> Self { - Self(self.0 & rhs.0) + Self(self.0 & rhs.0, PhantomData) } } - impl core::ops::BitOr for $name + impl core::ops::BitOr for $name where crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] fn bitor(self, rhs: Self) -> Self { - Self(self.0 | rhs.0) + Self(self.0 | rhs.0, PhantomData) } } - impl core::ops::BitXor for $name + impl core::ops::BitXor for $name where crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] fn bitxor(self, rhs: Self) -> Self::Output { - Self(self.0 ^ rhs.0) + Self(self.0 ^ rhs.0, PhantomData) } } - impl core::ops::Not for $name + impl core::ops::Not for $name where crate::$type: crate::LanesAtMost32, { - type Output = $name; + type Output = Self; #[inline] fn not(self) -> Self::Output { - Self(!self.0) - } - } - - impl core::ops::BitAndAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitand_assign(&mut self, rhs: Self) { - self.0 &= rhs.0; - } - } - - impl core::ops::BitOrAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitor_assign(&mut self, rhs: Self) { - self.0 |= rhs.0; - } - } - - impl core::ops::BitXorAssign for $name - where - crate::$type: crate::LanesAtMost32, - { - #[inline] - fn bitxor_assign(&mut self, rhs: Self) { - self.0 ^= rhs.0; + Self(!self.0, PhantomData) } } } diff --git a/crates/core_simd/src/masks/mod.rs b/crates/core_simd/src/masks/mod.rs index e5352ef4d1a2..deaf2be5dca4 100644 --- a/crates/core_simd/src/masks/mod.rs +++ b/crates/core_simd/src/masks/mod.rs @@ -2,16 +2,30 @@ //! Types representing #![allow(non_camel_case_types)] -#[cfg_attr(not(all(target_arch = "x86_64", target_feature = "avx512f")), path = "full_masks.rs")] -#[cfg_attr(all(target_arch = "x86_64", target_feature = "avx512f"), path = "bitmask.rs")] +#[cfg_attr( + not(all(target_arch = "x86_64", target_feature = "avx512f")), + path = "full_masks.rs" +)] +#[cfg_attr( + all(target_arch = "x86_64", target_feature = "avx512f"), + path = "bitmask.rs" +)] mod mask_impl; use crate::{LanesAtMost32, SimdI16, SimdI32, SimdI64, SimdI8, SimdIsize}; -/// Converts masks to bitmasks, with one bit set for each lane. -pub trait ToBitMask { - /// Converts this mask to a bitmask. - fn to_bitmask(self) -> u64; +mod sealed { + pub trait Sealed {} +} + +/// Helper trait for mask types. +pub trait Mask: sealed::Sealed { + /// The bitmask representation of a mask. + type BitMask: Copy + Default + AsRef<[u8]> + AsMut<[u8]>; + + // TODO remove this when rustc intrinsics are more flexible + #[doc(hidden)] + type IntBitMask; } macro_rules! define_opaque_mask { @@ -22,13 +36,47 @@ macro_rules! define_opaque_mask { } => { $(#[$attr])* #[allow(non_camel_case_types)] - pub struct $name($inner_ty) where $bits_ty: LanesAtMost32; + pub struct $name($inner_ty) + where + $bits_ty: LanesAtMost32, + Self: Mask; + + impl sealed::Sealed for $name + where + $bits_ty: LanesAtMost32, + Self: Mask, + {} + impl Mask for $name<1> { + type BitMask = [u8; 1]; + type IntBitMask = u8; + } + impl Mask for $name<2> { + type BitMask = [u8; 1]; + type IntBitMask = u8; + } + impl Mask for $name<4> { + type BitMask = [u8; 1]; + type IntBitMask = u8; + } + impl Mask for $name<8> { + type BitMask = [u8; 1]; + type IntBitMask = u8; + } + impl Mask for $name<16> { + type BitMask = [u8; 2]; + type IntBitMask = u16; + } + impl Mask for $name<32> { + type BitMask = [u8; 4]; + type IntBitMask = u32; + } impl_opaque_mask_reductions! { $name, $bits_ty } impl $name where - $bits_ty: LanesAtMost32 + $bits_ty: LanesAtMost32, + Self: Mask, { /// Construct a mask by setting all lanes to the given value. pub fn splat(value: bool) -> Self { @@ -125,48 +173,18 @@ macro_rules! define_opaque_mask { assert!(lane < LANES, "lane index out of range"); unsafe { self.set_unchecked(lane, value); } } - } - impl ToBitMask for $name<1> { - fn to_bitmask(self) -> u64 { - self.0.to_bitmask() - } - } - - impl ToBitMask for $name<2> { - fn to_bitmask(self) -> u64 { - self.0.to_bitmask() - } - } - - impl ToBitMask for $name<4> { - fn to_bitmask(self) -> u64 { - self.0.to_bitmask() - } - } - - impl ToBitMask for $name<8> { - fn to_bitmask(self) -> u64 { - self.0.to_bitmask() - } - } - - impl ToBitMask for $name<16> { - fn to_bitmask(self) -> u64 { - self.0.to_bitmask() - } - } - - impl ToBitMask for $name<32> { - fn to_bitmask(self) -> u64 { - self.0.to_bitmask() + /// Convert this mask to a bitmask, with one bit set per lane. + pub fn to_bitmask(self) -> ::BitMask { + self.0.to_bitmask::() } } // vector/array conversion impl From<[bool; LANES]> for $name where - $bits_ty: crate::LanesAtMost32 + $bits_ty: crate::LanesAtMost32, + Self: Mask, { fn from(array: [bool; LANES]) -> Self { Self::from_array(array) @@ -175,7 +193,8 @@ macro_rules! define_opaque_mask { impl From<$name> for [bool; LANES] where - $bits_ty: crate::LanesAtMost32 + $bits_ty: crate::LanesAtMost32, + $name: Mask, { fn from(vector: $name) -> Self { vector.to_array() @@ -184,13 +203,14 @@ macro_rules! define_opaque_mask { impl Copy for $name where - $inner_ty: Copy, $bits_ty: LanesAtMost32, + Self: Mask, {} impl Clone for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn clone(&self) -> Self { @@ -201,6 +221,7 @@ macro_rules! define_opaque_mask { impl Default for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn default() -> Self { @@ -211,6 +232,7 @@ macro_rules! define_opaque_mask { impl PartialEq for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn eq(&self, other: &Self) -> bool { @@ -221,6 +243,7 @@ macro_rules! define_opaque_mask { impl PartialOrd for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn partial_cmp(&self, other: &Self) -> Option { @@ -231,6 +254,7 @@ macro_rules! define_opaque_mask { impl core::fmt::Debug for $name where $bits_ty: crate::LanesAtMost32, + Self: Mask, { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { f.debug_list() @@ -242,6 +266,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitAnd for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -253,6 +278,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitAnd for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -264,6 +290,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitAnd<$name> for bool where $bits_ty: LanesAtMost32, + $name: Mask, { type Output = $name; #[inline] @@ -275,6 +302,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOr for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -286,6 +314,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOr for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -297,6 +326,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOr<$name> for bool where $bits_ty: LanesAtMost32, + $name: Mask, { type Output = $name; #[inline] @@ -308,6 +338,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXor for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -319,6 +350,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXor for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = Self; #[inline] @@ -330,6 +362,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXor<$name> for bool where $bits_ty: LanesAtMost32, + $name: Mask, { type Output = $name; #[inline] @@ -341,6 +374,7 @@ macro_rules! define_opaque_mask { impl core::ops::Not for $name where $bits_ty: LanesAtMost32, + Self: Mask, { type Output = $name; #[inline] @@ -352,16 +386,18 @@ macro_rules! define_opaque_mask { impl core::ops::BitAndAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitand_assign(&mut self, rhs: Self) { - self.0 &= rhs.0; + self.0 = self.0 & rhs.0; } } impl core::ops::BitAndAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitand_assign(&mut self, rhs: bool) { @@ -372,16 +408,18 @@ macro_rules! define_opaque_mask { impl core::ops::BitOrAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitor_assign(&mut self, rhs: Self) { - self.0 |= rhs.0; + self.0 = self.0 | rhs.0; } } impl core::ops::BitOrAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitor_assign(&mut self, rhs: bool) { @@ -392,16 +430,18 @@ macro_rules! define_opaque_mask { impl core::ops::BitXorAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitxor_assign(&mut self, rhs: Self) { - self.0 ^= rhs.0; + self.0 = self.0 ^ rhs.0; } } impl core::ops::BitXorAssign for $name where $bits_ty: LanesAtMost32, + Self: Mask, { #[inline] fn bitxor_assign(&mut self, rhs: bool) { @@ -415,7 +455,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 8-bit elements. /// /// The layout of this type is unspecified. - struct Mask8(mask_impl::Mask8); + struct Mask8(mask_impl::Mask8); @bits SimdI8 } @@ -423,7 +463,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 16-bit elements. /// /// The layout of this type is unspecified. - struct Mask16(mask_impl::Mask16); + struct Mask16(mask_impl::Mask16); @bits SimdI16 } @@ -431,7 +471,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 32-bit elements. /// /// The layout of this type is unspecified. - struct Mask32(mask_impl::Mask32); + struct Mask32(mask_impl::Mask32); @bits SimdI32 } @@ -439,7 +479,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` 64-bit elements. /// /// The layout of this type is unspecified. - struct Mask64(mask_impl::Mask64); + struct Mask64(mask_impl::Mask64); @bits SimdI64 } @@ -447,7 +487,7 @@ define_opaque_mask! { /// Mask for vectors with `LANES` pointer-width elements. /// /// The layout of this type is unspecified. - struct MaskSize(mask_impl::MaskSize); + struct MaskSize(mask_impl::MaskSize); @bits SimdIsize } diff --git a/crates/core_simd/src/reduction.rs b/crates/core_simd/src/reduction.rs index 2d4f1bca2647..8687d1af5167 100644 --- a/crates/core_simd/src/reduction.rs +++ b/crates/core_simd/src/reduction.rs @@ -104,7 +104,7 @@ macro_rules! impl_float_reductions { macro_rules! impl_full_mask_reductions { { $name:ident, $bits_ty:ident } => { - impl $name + impl $name where crate::$bits_ty: crate::LanesAtMost32 { @@ -125,7 +125,8 @@ macro_rules! impl_opaque_mask_reductions { { $name:ident, $bits_ty:ident } => { impl $name where - crate::$bits_ty: crate::LanesAtMost32 + crate::$bits_ty: crate::LanesAtMost32, + $name: crate::Mask, { /// Returns true if any lane is set, or false otherwise. #[inline] diff --git a/crates/core_simd/src/vector/float.rs b/crates/core_simd/src/vector/float.rs index 47013053ae15..6371f88a40a2 100644 --- a/crates/core_simd/src/vector/float.rs +++ b/crates/core_simd/src/vector/float.rs @@ -42,6 +42,7 @@ macro_rules! impl_float_vector { Self: crate::LanesAtMost32, crate::$bits_ty: crate::LanesAtMost32, crate::$mask_impl_ty: crate::LanesAtMost32, + crate::$mask_ty: crate::Mask, { /// Returns true for each lane if it has a positive sign, including /// `+0.0`, `NaN`s with positive sign bit and positive infinity. diff --git a/crates/core_simd/src/vector/int.rs b/crates/core_simd/src/vector/int.rs index 30b09a229e9b..a535fad7bc1d 100644 --- a/crates/core_simd/src/vector/int.rs +++ b/crates/core_simd/src/vector/int.rs @@ -30,6 +30,7 @@ macro_rules! impl_integer_vector { where Self: crate::LanesAtMost32, crate::$mask_impl_ty: crate::LanesAtMost32, + crate::$mask_ty: crate::Mask, { /// Returns true for each positive lane and false if it is zero or negative. pub fn is_positive(self) -> crate::$mask_ty { diff --git a/crates/core_simd/src/vector/uint.rs b/crates/core_simd/src/vector/uint.rs index 53e780520a79..db027b0941f2 100644 --- a/crates/core_simd/src/vector/uint.rs +++ b/crates/core_simd/src/vector/uint.rs @@ -1,6 +1,5 @@ #![allow(non_camel_case_types)] - /// Implements additional integer traits (Eq, Ord, Hash) on the specified vector `$name`, holding multiple `$lanes` of `$type`. macro_rules! impl_unsigned_vector { { $name:ident, $type:ty } => { diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index be83f4c2ec77..54427ec1ec0d 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -68,10 +68,12 @@ macro_rules! test_mask_api { #[test] fn to_bitmask() { - use core_simd::ToBitMask; - let values = [true, false, false, true, false, false, true, false]; - let mask = core_simd::$name::<8>::from_array(values); - assert_eq!(mask.to_bitmask(), 0b01001001); + let values = [ + true, false, false, true, false, false, true, false, + false, false, false, false, false, false, false, false, + ]; + let mask = core_simd::$name::<16>::from_array(values); + assert_eq!(mask.to_bitmask(), [0b01001001, 0]); } } } diff --git a/crates/test_helpers/src/array.rs b/crates/test_helpers/src/array.rs index c64bfee4f2d1..5ffc92269769 100644 --- a/crates/test_helpers/src/array.rs +++ b/crates/test_helpers/src/array.rs @@ -3,14 +3,11 @@ // Adapted from proptest's array code // Copyright 2017 Jason Lingle +use core::{marker::PhantomData, mem::MaybeUninit}; use proptest::{ strategy::{NewTree, Strategy, ValueTree}, test_runner::TestRunner, }; -use core::{ - marker::PhantomData, - mem::MaybeUninit, -}; #[must_use = "strategies do nothing unless used"] #[derive(Clone, Copy, Debug)] diff --git a/crates/test_helpers/src/lib.rs b/crates/test_helpers/src/lib.rs index 8b56877967cb..fffd088f4da3 100644 --- a/crates/test_helpers/src/lib.rs +++ b/crates/test_helpers/src/lib.rs @@ -281,6 +281,11 @@ macro_rules! test_lanes { core_simd::SimdIsize<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF32<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF64<$lanes>: core_simd::LanesAtMost32, + core_simd::Mask8<$lanes>: core_simd::Mask, + core_simd::Mask16<$lanes>: core_simd::Mask, + core_simd::Mask32<$lanes>: core_simd::Mask, + core_simd::Mask64<$lanes>: core_simd::Mask, + core_simd::MaskSize<$lanes>: core_simd::Mask, $body #[cfg(target_arch = "wasm32")] @@ -350,6 +355,11 @@ macro_rules! test_lanes_panic { core_simd::SimdIsize<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF32<$lanes>: core_simd::LanesAtMost32, core_simd::SimdF64<$lanes>: core_simd::LanesAtMost32, + core_simd::Mask8<$lanes>: core_simd::Mask, + core_simd::Mask16<$lanes>: core_simd::Mask, + core_simd::Mask32<$lanes>: core_simd::Mask, + core_simd::Mask64<$lanes>: core_simd::Mask, + core_simd::MaskSize<$lanes>: core_simd::Mask, $body #[test] From 589fce03131225f9167b6b90c6382f40ea22edb6 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Thu, 29 Apr 2021 03:34:28 +0000 Subject: [PATCH 4/4] Attempt to workaround MIPS bug --- crates/core_simd/src/masks/full_masks.rs | 13 ++++++++++++- crates/core_simd/tests/masks.rs | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index bd52a25551e2..f89bbefba631 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -112,7 +112,18 @@ macro_rules! define_mask { // TODO remove the transmute when rustc is more flexible assert_eq!(core::mem::size_of::(), core::mem::size_of::()); let mask: U::IntBitMask = crate::intrinsics::simd_bitmask(self.0); - core::mem::transmute_copy(&mask) + let mut bitmask: U::BitMask = core::mem::transmute_copy(&mask); + + // There is a bug where LLVM appears to implement this operation with the wrong + // bit order. + // TODO fix this in a better way + if cfg!(any(target_arch = "mips", target_arch = "mips64")) { + for x in bitmask.as_mut() { + *x = x.reverse_bits(); + } + } + + bitmask } } } diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index 54427ec1ec0d..7021d58aa543 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -70,10 +70,10 @@ macro_rules! test_mask_api { fn to_bitmask() { let values = [ true, false, false, true, false, false, true, false, - false, false, false, false, false, false, false, false, + true, true, false, false, false, false, false, true, ]; let mask = core_simd::$name::<16>::from_array(values); - assert_eq!(mask.to_bitmask(), [0b01001001, 0]); + assert_eq!(mask.to_bitmask(), [0b01001001, 0b10000011]); } } }