user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use core::intrinsics

Browse source
This commit is contained in:
Caleb Zulawski 2023-12-29 18:30:53 -05:00
parent 1273da658e
commit bb50fa2325
18 changed files with 106 additions and 285 deletions
Download patch file Download diff file Expand all files Collapse all files

170
crates/core_simd/src/intrinsics.rs
View file

@ -1,170 +0,0 @@
//! This module contains the LLVM intrinsics bindings that provide the functionality for this
//! crate.
//!
//! The LLVM assembly language is documented here: <https://llvm.org/docs/LangRef.html>
//!
//! A quick glossary of jargon that may appear in this module, mostly paraphrasing LLVM's LangRef:
//! - poison: "undefined behavior as a value". specifically, it is like uninit memory (such as padding bytes). it is "safe" to create poison, BUT
//! poison MUST NOT be observed from safe code, as operations on poison return poison, like NaN. unlike NaN, which has defined comparisons,
//! poison is neither true nor false, and LLVM may also convert it to undef (at which point it is both). so, it can't be conditioned on, either.
//! - undef: "a value that is every value". functionally like poison, insofar as Rust is concerned. poison may become this. note:
//! this means that division by poison or undef is like division by zero, which means it inflicts...
//! - "UB": poison and undef cover most of what people call "UB". "UB" means this operation immediately invalidates the program:
//! LLVM is allowed to lower it to `ud2` or other opcodes that may cause an illegal instruction exception, and this is the "good end".
//! The "bad end" is that LLVM may reverse time to the moment control flow diverged on a path towards undefined behavior,
//! and destroy the other branch, potentially deleting safe code and violating Rust's `unsafe` contract.
//!
//! Note that according to LLVM, vectors are not arrays, but they are equivalent when stored to and loaded from memory.
//!
//! Unless stated otherwise, all intrinsics for binary operations require SIMD vectors of equal types and lengths.
// These intrinsics aren't linked directly from LLVM and are mostly undocumented, however they are
// mostly lowered to the matching LLVM instructions by the compiler in a fairly straightforward manner.
// The associated LLVM instruction or intrinsic is documented alongside each Rust intrinsic function.
extern "platform-intrinsic" {
/// add/fadd
pub(crate) fn simd_add<T>(x: T, y: T) -> T;
/// sub/fsub
pub(crate) fn simd_sub<T>(lhs: T, rhs: T) -> T;
/// mul/fmul
pub(crate) fn simd_mul<T>(x: T, y: T) -> T;
/// udiv/sdiv/fdiv
/// ints and uints: {s,u}div incur UB if division by zero occurs.
/// ints: sdiv is UB for int::MIN / -1.
/// floats: fdiv is never UB, but may create NaNs or infinities.
pub(crate) fn simd_div<T>(lhs: T, rhs: T) -> T;
/// urem/srem/frem
/// ints and uints: {s,u}rem incur UB if division by zero occurs.
/// ints: srem is UB for int::MIN / -1.
/// floats: frem is equivalent to libm::fmod in the "default" floating point environment, sans errno.
pub(crate) fn simd_rem<T>(lhs: T, rhs: T) -> T;
/// shl
/// for (u)ints. poison if rhs >= lhs::BITS
pub(crate) fn simd_shl<T>(lhs: T, rhs: T) -> T;
/// ints: ashr
/// uints: lshr
/// poison if rhs >= lhs::BITS
pub(crate) fn simd_shr<T>(lhs: T, rhs: T) -> T;
/// and
pub(crate) fn simd_and<T>(x: T, y: T) -> T;
/// or
pub(crate) fn simd_or<T>(x: T, y: T) -> T;
/// xor
pub(crate) fn simd_xor<T>(x: T, y: T) -> T;
/// fptoui/fptosi/uitofp/sitofp
/// casting floats to integers is truncating, so it is safe to convert values like e.g. 1.5
/// but the truncated value must fit in the target type or the result is poison.
/// use `simd_as` instead for a cast that performs a saturating conversion.
pub(crate) fn simd_cast<T, U>(x: T) -> U;
/// follows Rust's `T as U` semantics, including saturating float casts
/// which amounts to the same as `simd_cast` for many cases
pub(crate) fn simd_as<T, U>(x: T) -> U;
/// neg/fneg
/// ints: ultimately becomes a call to cg_ssa's BuilderMethods::neg. cg_llvm equates this to `simd_sub(Simd::splat(0), x)`.
/// floats: LLVM's fneg, which changes the floating point sign bit. Some arches have instructions for it.
/// Rust panics for Neg::neg(int::MIN) due to overflow, but it is not UB in LLVM without `nsw`.
pub(crate) fn simd_neg<T>(x: T) -> T;
/// fabs
pub(crate) fn simd_fabs<T>(x: T) -> T;
// minnum/maxnum
pub(crate) fn simd_fmin<T>(x: T, y: T) -> T;
pub(crate) fn simd_fmax<T>(x: T, y: T) -> T;
// these return Simd<int, N> with the same BITS size as the inputs
pub(crate) fn simd_eq<T, U>(x: T, y: T) -> U;
pub(crate) fn simd_ne<T, U>(x: T, y: T) -> U;
pub(crate) fn simd_lt<T, U>(x: T, y: T) -> U;
pub(crate) fn simd_le<T, U>(x: T, y: T) -> U;
pub(crate) fn simd_gt<T, U>(x: T, y: T) -> U;
pub(crate) fn simd_ge<T, U>(x: T, y: T) -> U;
// shufflevector
// idx: LLVM calls it a "shuffle mask vector constant", a vector of i32s
pub(crate) fn simd_shuffle<T, U, V>(x: T, y: T, idx: U) -> V;
/// llvm.masked.gather
/// like a loop of pointer reads
/// val: vector of values to select if a lane is masked
/// ptr: vector of pointers to read from
/// mask: a "wide" mask of integers, selects as if simd_select(mask, read(ptr), val)
/// note, the LLVM intrinsic accepts a mask vector of `<N x i1>`
/// FIXME: review this if/when we fix up our mask story in general?
pub(crate) fn simd_gather<T, U, V>(val: T, ptr: U, mask: V) -> T;
/// llvm.masked.scatter
/// like gather, but more spicy, as it writes instead of reads
pub(crate) fn simd_scatter<T, U, V>(val: T, ptr: U, mask: V);
// {s,u}add.sat
pub(crate) fn simd_saturating_add<T>(x: T, y: T) -> T;
// {s,u}sub.sat
pub(crate) fn simd_saturating_sub<T>(lhs: T, rhs: T) -> T;
// reductions
// llvm.vector.reduce.{add,fadd}
pub(crate) fn simd_reduce_add_ordered<T, U>(x: T, y: U) -> U;
// llvm.vector.reduce.{mul,fmul}
pub(crate) fn simd_reduce_mul_ordered<T, U>(x: T, y: U) -> U;
#[allow(unused)]
pub(crate) fn simd_reduce_all<T>(x: T) -> bool;
#[allow(unused)]
pub(crate) fn simd_reduce_any<T>(x: T) -> bool;
pub(crate) fn simd_reduce_max<T, U>(x: T) -> U;
pub(crate) fn simd_reduce_min<T, U>(x: T) -> U;
pub(crate) fn simd_reduce_and<T, U>(x: T) -> U;
pub(crate) fn simd_reduce_or<T, U>(x: T) -> U;
pub(crate) fn simd_reduce_xor<T, U>(x: T) -> U;
// truncate integer vector to bitmask
// `fn simd_bitmask(vector) -> unsigned integer` takes a vector of integers and
// returns either an unsigned integer or array of `u8`.
// Every element in the vector becomes a single bit in the returned bitmask.
// The bit order of the result depends on the byte endianness. LSB-first for little
// endian and MSB-first for big endian.
// If the vector has less than 8 lanes, the mask lives in the least-significant bits
// (e.g., [true, false] becomes `0b01` on little endian and `0b10` on big endian).
//
// UB if called on a vector with values other than 0 and -1.
#[allow(unused)]
pub(crate) fn simd_bitmask<T, U>(x: T) -> U;
// select
// first argument is a vector of integers, -1 (all bits 1) is "true"
// logically equivalent to (yes & m) | (no & (m^-1),
// but you can use it on floats.
pub(crate) fn simd_select<M, T>(m: M, yes: T, no: T) -> T;
#[allow(unused)]
pub(crate) fn simd_select_bitmask<M, T>(m: M, yes: T, no: T) -> T;
/// getelementptr (without inbounds)
/// equivalent to wrapping_offset
pub(crate) fn simd_arith_offset<T, U>(ptr: T, offset: U) -> T;
/// equivalent to `T as U` semantics, specifically for pointers
pub(crate) fn simd_cast_ptr<T, U>(ptr: T) -> U;
/// expose a pointer as an address
pub(crate) fn simd_expose_addr<T, U>(ptr: T) -> U;
/// convert an exposed address back to a pointer
pub(crate) fn simd_from_exposed_addr<T, U>(addr: T) -> U;
// Integer operations
pub(crate) fn simd_bswap<T>(x: T) -> T;
pub(crate) fn simd_bitreverse<T>(x: T) -> T;
pub(crate) fn simd_ctlz<T>(x: T) -> T;
pub(crate) fn simd_cttz<T>(x: T) -> T;
}

2
crates/core_simd/src/lib.rs
View file

@ -1,5 +1,6 @@
#![no_std]
#![feature(
core_intrinsics,
const_refs_to_cell,
const_maybe_uninit_as_mut_ptr,
const_mut_refs,
@ -7,7 +8,6 @@
decl_macro,
inline_const,
intra_doc_pointers,
platform_intrinsics,
repr_simd,
simd_ffi,
staged_api,

19
crates/core_simd/src/masks.rs
View file

@ -12,9 +12,7 @@
)]
mod mask_impl;
use crate::simd::{
cmp::SimdPartialEq, intrinsics, LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount,
};
use crate::simd::{cmp::SimdPartialEq, LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount};
use core::cmp::Ordering;
use core::{fmt, mem};
@ -141,8 +139,9 @@ where
// but these are "dependently-sized" types, so copy elision it is!
unsafe {
let bytes: [u8; N] = mem::transmute_copy(&array);
let bools: Simd<i8, N> = intrinsics::simd_ne(Simd::from_array(bytes), Simd::splat(0u8));
Mask::from_int_unchecked(intrinsics::simd_cast(bools))
let bools: Simd<i8, N> =
core::intrinsics::simd::simd_ne(Simd::from_array(bytes), Simd::splat(0u8));
Mask::from_int_unchecked(core::intrinsics::simd::simd_cast(bools))
}
}
@ -160,7 +159,7 @@ where
// This would be hypothetically valid as an "in-place" transmute,
// but these are "dependently-sized" types, so copy elision it is!
unsafe {
let mut bytes: Simd<i8, N> = intrinsics::simd_cast(self.to_int());
let mut bytes: Simd<i8, N> = core::intrinsics::simd::simd_cast(self.to_int());
bytes &= Simd::splat(1i8);
mem::transmute_copy(&bytes)
}
@ -374,15 +373,17 @@ where
);
// Safety: the input and output are integer vectors
let index: Simd<T, N> = unsafe { intrinsics::simd_cast(index) };
let index: Simd<T, N> = unsafe { core::intrinsics::simd::simd_cast(index) };
let masked_index = self.select(index, Self::splat(true).to_int());
// Safety: the input and output are integer vectors
let masked_index: Simd<T::Unsigned, N> = unsafe { intrinsics::simd_cast(masked_index) };
let masked_index: Simd<T::Unsigned, N> =
unsafe { core::intrinsics::simd::simd_cast(masked_index) };
// Safety: the input is an integer vector
let min_index: T::Unsigned = unsafe { intrinsics::simd_reduce_min(masked_index) };
let min_index: T::Unsigned =
unsafe { core::intrinsics::simd::simd_reduce_min(masked_index) };
// Safety: the return value is the unsigned version of T
let min_index: T = unsafe { core::mem::transmute_copy(&min_index) };

9
crates/core_simd/src/masks/bitmask.rs
View file

@ -1,6 +1,5 @@
#![allow(unused_imports)]
use super::MaskElement;
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Simd, SupportedLaneCount};
use core::marker::PhantomData;
@ -109,14 +108,18 @@ where
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn to_int(self) -> Simd<T, N> {
unsafe {
intrinsics::simd_select_bitmask(self.0, Simd::splat(T::TRUE), Simd::splat(T::FALSE))
core::intrinsics::simd::simd_select_bitmask(
self.0,
Simd::splat(T::TRUE),
Simd::splat(T::FALSE),
)
}
}
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub unsafe fn from_int_unchecked(value: Simd<T, N>) -> Self {
unsafe { Self(intrinsics::simd_bitmask(value), PhantomData) }
unsafe { Self(core::intrinsics::simd::simd_bitmask(value), PhantomData) }
}
#[inline]

21
crates/core_simd/src/masks/full_masks.rs
View file

@ -1,6 +1,5 @@
//! Masks that take up full SIMD vector registers.
use crate::simd::intrinsics;
use crate::simd::{LaneCount, MaskElement, Simd, SupportedLaneCount};
#[repr(transparent)]
@ -138,7 +137,7 @@ where
U: MaskElement,
{
// Safety: masks are simply integer vectors of 0 and -1, and we can cast the element type.
unsafe { Mask(intrinsics::simd_cast(self.0)) }
unsafe { Mask(core::intrinsics::simd::simd_cast(self.0)) }
}
#[inline]
@ -150,7 +149,7 @@ where
unsafe {
// Compute the bitmask
let mut bytes: <LaneCount<N> as SupportedLaneCount>::BitMask =
intrinsics::simd_bitmask(self.0);
core::intrinsics::simd::simd_bitmask(self.0);
// LLVM assumes bit order should match endianness
if cfg!(target_endian = "big") {
@ -183,7 +182,7 @@ where
}
// Compute the regular mask
Self::from_int_unchecked(intrinsics::simd_select_bitmask(
Self::from_int_unchecked(core::intrinsics::simd::simd_select_bitmask(
bytes,
Self::splat(true).to_int(),
Self::splat(false).to_int(),
@ -199,7 +198,7 @@ where
let resized = self.to_int().resize::<M>(T::FALSE);
// Safety: `resized` is an integer vector with length M, which must match T
let bitmask: U = unsafe { intrinsics::simd_bitmask(resized) };
let bitmask: U = unsafe { core::intrinsics::simd::simd_bitmask(resized) };
// LLVM assumes bit order should match endianness
if cfg!(target_endian = "big") {
@ -223,7 +222,7 @@ where
// SAFETY: `mask` is the correct bitmask type for a u64 bitmask
let mask: Simd<T, M> = unsafe {
intrinsics::simd_select_bitmask(
core::intrinsics::simd::simd_select_bitmask(
bitmask,
Simd::<T, M>::splat(T::TRUE),
Simd::<T, M>::splat(T::FALSE),
@ -274,14 +273,14 @@ where
#[must_use = "method returns a new bool and does not mutate the original value"]
pub fn any(self) -> bool {
// Safety: use `self` as an integer vector
unsafe { intrinsics::simd_reduce_any(self.to_int()) }
unsafe { core::intrinsics::simd::simd_reduce_any(self.to_int()) }
}
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn all(self) -> bool {
// Safety: use `self` as an integer vector
unsafe { intrinsics::simd_reduce_all(self.to_int()) }
unsafe { core::intrinsics::simd::simd_reduce_all(self.to_int()) }
}
}
@ -306,7 +305,7 @@ where
#[must_use = "method returns a new mask and does not mutate the original value"]
fn bitand(self, rhs: Self) -> Self {
// Safety: `self` is an integer vector
unsafe { Self(intrinsics::simd_and(self.0, rhs.0)) }
unsafe { Self(core::intrinsics::simd::simd_and(self.0, rhs.0)) }
}
}
@ -320,7 +319,7 @@ where
#[must_use = "method returns a new mask and does not mutate the original value"]
fn bitor(self, rhs: Self) -> Self {
// Safety: `self` is an integer vector
unsafe { Self(intrinsics::simd_or(self.0, rhs.0)) }
unsafe { Self(core::intrinsics::simd::simd_or(self.0, rhs.0)) }
}
}
@ -334,7 +333,7 @@ where
#[must_use = "method returns a new mask and does not mutate the original value"]
fn bitxor(self, rhs: Self) -> Self {
// Safety: `self` is an integer vector
unsafe { Self(intrinsics::simd_xor(self.0, rhs.0)) }
unsafe { Self(core::intrinsics::simd::simd_xor(self.0, rhs.0)) }
}
}

4
crates/core_simd/src/mod.rs
View file

@ -1,8 +1,6 @@
#[macro_use]
mod swizzle;
pub(crate) mod intrinsics;
mod alias;
mod cast;
mod fmt;
@ -27,8 +25,6 @@ pub mod simd {
pub mod cmp;
pub(crate) use crate::core_simd::intrinsics;
pub use crate::core_simd::alias::*;
pub use crate::core_simd::cast::*;
pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount};

6
crates/core_simd/src/ops.rs
View file

@ -37,7 +37,7 @@ where
macro_rules! unsafe_base {
($lhs:ident, $rhs:ident, {$simd_call:ident}, $($_:tt)*) => {
// Safety: $lhs and $rhs are vectors
unsafe { $crate::simd::intrinsics::$simd_call($lhs, $rhs) }
unsafe { core::intrinsics::simd::$simd_call($lhs, $rhs) }
};
}
@ -55,7 +55,7 @@ macro_rules! wrap_bitshift {
#[allow(clippy::suspicious_arithmetic_impl)]
// Safety: $lhs and the bitand result are vectors
unsafe {
$crate::simd::intrinsics::$simd_call(
core::intrinsics::simd::$simd_call(
$lhs,
$rhs.bitand(Simd::splat(<$int>::BITS as $int - 1)),
)
@ -97,7 +97,7 @@ macro_rules! int_divrem_guard {
$rhs
};
// Safety: $lhs and rhs are vectors
unsafe { $crate::simd::intrinsics::$simd_call($lhs, rhs) }
unsafe { core::intrinsics::simd::$simd_call($lhs, rhs) }
}
};
}

3
crates/core_simd/src/ops/unary.rs
View file

@ -1,4 +1,3 @@
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount};
use core::ops::{Neg, Not}; // unary ops
@ -15,7 +14,7 @@ macro_rules! neg {
#[must_use = "operator returns a new vector without mutating the input"]
fn neg(self) -> Self::Output {
// Safety: `self` is a signed vector
unsafe { intrinsics::simd_neg(self) }
unsafe { core::intrinsics::simd::simd_neg(self) }
}
})*
}

3
crates/core_simd/src/select.rs
View file

@ -1,4 +1,3 @@
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Mask, MaskElement, Simd, SimdElement, SupportedLaneCount};
impl<T, const N: usize> Mask<T, N>
@ -29,7 +28,7 @@ where
{
// Safety: The mask has been cast to a vector of integers,
// and the operands to select between are vectors of the same type and length.
unsafe { intrinsics::simd_select(self.to_int(), true_values, false_values) }
unsafe { core::intrinsics::simd::simd_select(self.to_int(), true_values, false_values) }
}
/// Choose elements from two masks.

9
crates/core_simd/src/simd/cmp/eq.rs
View file

@ -1,5 +1,4 @@
use crate::simd::{
intrinsics,
ptr::{SimdConstPtr, SimdMutPtr},
LaneCount, Mask, Simd, SimdElement, SupportedLaneCount,
};
@ -31,14 +30,14 @@ macro_rules! impl_number {
fn simd_eq(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) }
unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_eq(self, other)) }
}
#[inline]
fn simd_ne(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) }
unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_ne(self, other)) }
}
}
)*
@ -60,14 +59,14 @@ macro_rules! impl_mask {
fn simd_eq(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_eq(self.to_int(), other.to_int())) }
unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_eq(self.to_int(), other.to_int())) }
}
#[inline]
fn simd_ne(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_ne(self.to_int(), other.to_int())) }
unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_ne(self.to_int(), other.to_int())) }
}
}
)*

25
crates/core_simd/src/simd/cmp/ord.rs
View file

@ -1,6 +1,5 @@
use crate::simd::{
cmp::SimdPartialEq,
intrinsics,
ptr::{SimdConstPtr, SimdMutPtr},
LaneCount, Mask, Simd, SupportedLaneCount,
};
@ -57,28 +56,28 @@ macro_rules! impl_integer {
fn simd_lt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) }
unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_lt(self, other)) }
}
#[inline]
fn simd_le(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) }
unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_le(self, other)) }
}
#[inline]
fn simd_gt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) }
unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_gt(self, other)) }
}
#[inline]
fn simd_ge(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) }
unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_ge(self, other)) }
}
}
@ -123,28 +122,28 @@ macro_rules! impl_float {
fn simd_lt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) }
unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_lt(self, other)) }
}
#[inline]
fn simd_le(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) }
unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_le(self, other)) }
}
#[inline]
fn simd_gt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) }
unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_gt(self, other)) }
}
#[inline]
fn simd_ge(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) }
unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_ge(self, other)) }
}
}
)*
@ -164,28 +163,28 @@ macro_rules! impl_mask {
fn simd_lt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_lt(self.to_int(), other.to_int())) }
unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_lt(self.to_int(), other.to_int())) }
}
#[inline]
fn simd_le(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_le(self.to_int(), other.to_int())) }
unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_le(self.to_int(), other.to_int())) }
}
#[inline]
fn simd_gt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_gt(self.to_int(), other.to_int())) }
unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_gt(self.to_int(), other.to_int())) }
}
#[inline]
fn simd_ge(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_ge(self.to_int(), other.to_int())) }
unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_ge(self.to_int(), other.to_int())) }
}
}

20
crates/core_simd/src/simd/num/float.rs
View file

@ -1,7 +1,7 @@
use super::sealed::Sealed;
use crate::simd::{
cmp::{SimdPartialEq, SimdPartialOrd},
intrinsics, LaneCount, Mask, Simd, SimdCast, SimdElement, SupportedLaneCount,
LaneCount, Mask, Simd, SimdCast, SimdElement, SupportedLaneCount,
};
/// Operations on SIMD vectors of floats.
@ -259,7 +259,7 @@ macro_rules! impl_trait {
fn cast<T: SimdCast>(self) -> Self::Cast<T>
{
// Safety: supported types are guaranteed by SimdCast
unsafe { intrinsics::simd_as(self) }
unsafe { core::intrinsics::simd::simd_as(self) }
}
#[inline]
@ -269,7 +269,7 @@ macro_rules! impl_trait {
Self::Scalar: core::convert::FloatToInt<I>,
{
// Safety: supported types are guaranteed by SimdCast, the caller is responsible for the extra invariants
unsafe { intrinsics::simd_cast(self) }
unsafe { core::intrinsics::simd::simd_cast(self) }
}
#[inline]
@ -289,7 +289,7 @@ macro_rules! impl_trait {
#[inline]
fn abs(self) -> Self {
// Safety: `self` is a float vector
unsafe { intrinsics::simd_fabs(self) }
unsafe { core::intrinsics::simd::simd_fabs(self) }
}
#[inline]
@ -363,13 +363,13 @@ macro_rules! impl_trait {
#[inline]
fn simd_min(self, other: Self) -> Self {
// Safety: `self` and `other` are float vectors
unsafe { intrinsics::simd_fmin(self, other) }
unsafe { core::intrinsics::simd::simd_fmin(self, other) }
}
#[inline]
fn simd_max(self, other: Self) -> Self {
// Safety: `self` and `other` are floating point vectors
unsafe { intrinsics::simd_fmax(self, other) }
unsafe { core::intrinsics::simd::simd_fmax(self, other) }
}
#[inline]
@ -391,7 +391,7 @@ macro_rules! impl_trait {
self.as_array().iter().sum()
} else {
// Safety: `self` is a float vector
unsafe { intrinsics::simd_reduce_add_ordered(self, 0.) }
unsafe { core::intrinsics::simd::simd_reduce_add_ordered(self, 0.) }
}
}
@ -402,20 +402,20 @@ macro_rules! impl_trait {
self.as_array().iter().product()
} else {
// Safety: `self` is a float vector
unsafe { intrinsics::simd_reduce_mul_ordered(self, 1.) }
unsafe { core::intrinsics::simd::simd_reduce_mul_ordered(self, 1.) }
}
}
#[inline]
fn reduce_max(self) -> Self::Scalar {
// Safety: `self` is a float vector
unsafe { intrinsics::simd_reduce_max(self) }
unsafe { core::intrinsics::simd::simd_reduce_max(self) }
}
#[inline]
fn reduce_min(self) -> Self::Scalar {
// Safety: `self` is a float vector
unsafe { intrinsics::simd_reduce_min(self) }
unsafe { core::intrinsics::simd::simd_reduce_min(self) }
}
}
)*

26
crates/core_simd/src/simd/num/int.rs
View file

@ -1,6 +1,6 @@
use super::sealed::Sealed;
use crate::simd::{
cmp::SimdPartialOrd, intrinsics, num::SimdUint, LaneCount, Mask, Simd, SimdCast, SimdElement,
cmp::SimdPartialOrd, num::SimdUint, LaneCount, Mask, Simd, SimdCast, SimdElement,
SupportedLaneCount,
};
@ -237,19 +237,19 @@ macro_rules! impl_trait {
#[inline]
fn cast<T: SimdCast>(self) -> Self::Cast<T> {
// Safety: supported types are guaranteed by SimdCast
unsafe { intrinsics::simd_as(self) }
unsafe { core::intrinsics::simd::simd_as(self) }
}
#[inline]
fn saturating_add(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { intrinsics::simd_saturating_add(self, second) }
unsafe { core::intrinsics::simd::simd_saturating_add(self, second) }
}
#[inline]
fn saturating_sub(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { intrinsics::simd_saturating_sub(self, second) }
unsafe { core::intrinsics::simd::simd_saturating_sub(self, second) }
}
#[inline]
@ -293,55 +293,55 @@ macro_rules! impl_trait {
#[inline]
fn reduce_sum(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_add_ordered(self, 0) }
unsafe { core::intrinsics::simd::simd_reduce_add_ordered(self, 0) }
}
#[inline]
fn reduce_product(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) }
unsafe { core::intrinsics::simd::simd_reduce_mul_ordered(self, 1) }
}
#[inline]
fn reduce_max(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_max(self) }
unsafe { core::intrinsics::simd::simd_reduce_max(self) }
}
#[inline]
fn reduce_min(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_min(self) }
unsafe { core::intrinsics::simd::simd_reduce_min(self) }
}
#[inline]
fn reduce_and(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_and(self) }
unsafe { core::intrinsics::simd::simd_reduce_and(self) }
}
#[inline]
fn reduce_or(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_or(self) }
unsafe { core::intrinsics::simd::simd_reduce_or(self) }
}
#[inline]
fn reduce_xor(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_xor(self) }
unsafe { core::intrinsics::simd::simd_reduce_xor(self) }
}
#[inline]
fn swap_bytes(self) -> Self {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_bswap(self) }
unsafe { core::intrinsics::simd::simd_bswap(self) }
}
#[inline]
fn reverse_bits(self) -> Self {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_bitreverse(self) }
unsafe { core::intrinsics::simd::simd_bitreverse(self) }
}
#[inline]

30
crates/core_simd/src/simd/num/uint.rs
View file

@ -1,5 +1,5 @@
use super::sealed::Sealed;
use crate::simd::{intrinsics, LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount};
use crate::simd::{LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount};
/// Operations on SIMD vectors of unsigned integers.
pub trait SimdUint: Copy + Sealed {
@ -117,7 +117,7 @@ macro_rules! impl_trait {
#[inline]
fn cast<T: SimdCast>(self) -> Self::Cast<T> {
// Safety: supported types are guaranteed by SimdCast
unsafe { intrinsics::simd_as(self) }
unsafe { core::intrinsics::simd::simd_as(self) }
}
#[inline]
@ -129,79 +129,79 @@ macro_rules! impl_trait {
#[inline]
fn saturating_add(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { intrinsics::simd_saturating_add(self, second) }
unsafe { core::intrinsics::simd::simd_saturating_add(self, second) }
}
#[inline]
fn saturating_sub(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { intrinsics::simd_saturating_sub(self, second) }
unsafe { core::intrinsics::simd::simd_saturating_sub(self, second) }
}
#[inline]
fn reduce_sum(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_add_ordered(self, 0) }
unsafe { core::intrinsics::simd::simd_reduce_add_ordered(self, 0) }
}
#[inline]
fn reduce_product(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) }
unsafe { core::intrinsics::simd::simd_reduce_mul_ordered(self, 1) }
}
#[inline]
fn reduce_max(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_max(self) }
unsafe { core::intrinsics::simd::simd_reduce_max(self) }
}
#[inline]
fn reduce_min(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_min(self) }
unsafe { core::intrinsics::simd::simd_reduce_min(self) }
}
#[inline]
fn reduce_and(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_and(self) }
unsafe { core::intrinsics::simd::simd_reduce_and(self) }
}
#[inline]
fn reduce_or(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_or(self) }
unsafe { core::intrinsics::simd::simd_reduce_or(self) }
}
#[inline]
fn reduce_xor(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_xor(self) }
unsafe { core::intrinsics::simd::simd_reduce_xor(self) }
}
#[inline]
fn swap_bytes(self) -> Self {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_bswap(self) }
unsafe { core::intrinsics::simd::simd_bswap(self) }
}
#[inline]
fn reverse_bits(self) -> Self {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_bitreverse(self) }
unsafe { core::intrinsics::simd::simd_bitreverse(self) }
}
#[inline]
fn leading_zeros(self) -> Self {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_ctlz(self) }
unsafe { core::intrinsics::simd::simd_ctlz(self) }
}
#[inline]
fn trailing_zeros(self) -> Self {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_cttz(self) }
unsafe { core::intrinsics::simd::simd_cttz(self) }
}
#[inline]

14
crates/core_simd/src/simd/ptr/const_ptr.rs
View file

@ -1,7 +1,5 @@
use super::sealed::Sealed;
use crate::simd::{
cmp::SimdPartialEq, intrinsics, num::SimdUint, LaneCount, Mask, Simd, SupportedLaneCount,
};
use crate::simd::{cmp::SimdPartialEq, num::SimdUint, LaneCount, Mask, Simd, SupportedLaneCount};
/// Operations on SIMD vectors of constant pointers.
pub trait SimdConstPtr: Copy + Sealed {
@ -103,13 +101,13 @@ where
assert_eq!(size_of::<<U as Pointee>::Metadata>(), 0);
// Safety: pointers can be cast
unsafe { intrinsics::simd_cast_ptr(self) }
unsafe { core::intrinsics::simd::simd_cast_ptr(self) }
}
#[inline]
fn cast_mut(self) -> Self::MutPtr {
// Safety: pointers can be cast
unsafe { intrinsics::simd_cast_ptr(self) }
unsafe { core::intrinsics::simd::simd_cast_ptr(self) }
}
#[inline]
@ -135,19 +133,19 @@ where
#[inline]
fn expose_addr(self) -> Self::Usize {
// Safety: `self` is a pointer vector
unsafe { intrinsics::simd_expose_addr(self) }
unsafe { core::intrinsics::simd::simd_expose_addr(self) }
}
#[inline]
fn from_exposed_addr(addr: Self::Usize) -> Self {
// Safety: `self` is a pointer vector
unsafe { intrinsics::simd_from_exposed_addr(addr) }
unsafe { core::intrinsics::simd::simd_from_exposed_addr(addr) }
}
#[inline]
fn wrapping_offset(self, count: Self::Isize) -> Self {
// Safety: simd_arith_offset takes a vector of pointers and a vector of offsets
unsafe { intrinsics::simd_arith_offset(self, count) }
unsafe { core::intrinsics::simd::simd_arith_offset(self, count) }
}
#[inline]

14
crates/core_simd/src/simd/ptr/mut_ptr.rs
View file

@ -1,7 +1,5 @@
use super::sealed::Sealed;
use crate::simd::{
cmp::SimdPartialEq, intrinsics, num::SimdUint, LaneCount, Mask, Simd, SupportedLaneCount,
};
use crate::simd::{cmp::SimdPartialEq, num::SimdUint, LaneCount, Mask, Simd, SupportedLaneCount};
/// Operations on SIMD vectors of mutable pointers.
pub trait SimdMutPtr: Copy + Sealed {
@ -100,13 +98,13 @@ where
assert_eq!(size_of::<<U as Pointee>::Metadata>(), 0);
// Safety: pointers can be cast
unsafe { intrinsics::simd_cast_ptr(self) }
unsafe { core::intrinsics::simd::simd_cast_ptr(self) }
}
#[inline]
fn cast_const(self) -> Self::ConstPtr {
// Safety: pointers can be cast
unsafe { intrinsics::simd_cast_ptr(self) }
unsafe { core::intrinsics::simd::simd_cast_ptr(self) }
}
#[inline]
@ -132,19 +130,19 @@ where
#[inline]
fn expose_addr(self) -> Self::Usize {
// Safety: `self` is a pointer vector
unsafe { intrinsics::simd_expose_addr(self) }
unsafe { core::intrinsics::simd::simd_expose_addr(self) }
}
#[inline]
fn from_exposed_addr(addr: Self::Usize) -> Self {
// Safety: `self` is a pointer vector
unsafe { intrinsics::simd_from_exposed_addr(addr) }
unsafe { core::intrinsics::simd::simd_from_exposed_addr(addr) }
}
#[inline]
fn wrapping_offset(self, count: Self::Isize) -> Self {
// Safety: simd_arith_offset takes a vector of pointers and a vector of offsets
unsafe { intrinsics::simd_arith_offset(self, count) }
unsafe { core::intrinsics::simd::simd_arith_offset(self, count) }
}
#[inline]

5
crates/core_simd/src/swizzle.rs
View file

@ -1,4 +1,3 @@
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Mask, MaskElement, Simd, SimdElement, SupportedLaneCount};
/// Constructs a new SIMD vector by copying elements from selected elements in other vectors.
@ -88,7 +87,7 @@ pub trait Swizzle<const N: usize> {
{
// Safety: `vector` is a vector, and the index is a const array of u32.
unsafe {
intrinsics::simd_shuffle(
core::intrinsics::simd::simd_shuffle(
vector,
vector,
const {
@ -124,7 +123,7 @@ pub trait Swizzle<const N: usize> {
{
// Safety: `first` and `second` are vectors, and the index is a const array of u32.
unsafe {
intrinsics::simd_shuffle(
core::intrinsics::simd::simd_shuffle(
first,
second,
const {

11
crates/core_simd/src/vector.rs
View file

@ -1,6 +1,5 @@
use crate::simd::{
cmp::SimdPartialOrd,
intrinsics,
ptr::{SimdConstPtr, SimdMutPtr},
LaneCount, Mask, MaskElement, SupportedLaneCount, Swizzle,
};
@ -491,7 +490,7 @@ where
or: Self,
) -> Self {
// Safety: The caller is responsible for upholding all invariants
unsafe { intrinsics::simd_gather(or, source, enable.to_int()) }
unsafe { core::intrinsics::simd::simd_gather(or, source, enable.to_int()) }
}
/// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`.
@ -650,7 +649,7 @@ where
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub unsafe fn scatter_select_ptr(self, dest: Simd<*mut T, N>, enable: Mask<isize, N>) {
// Safety: The caller is responsible for upholding all invariants
unsafe { intrinsics::simd_scatter(self, dest, enable.to_int()) }
unsafe { core::intrinsics::simd::simd_scatter(self, dest, enable.to_int()) }
}
}
@ -692,7 +691,8 @@ where
fn eq(&self, other: &Self) -> bool {
// Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask.
let mask = unsafe {
let tfvec: Simd<<T as SimdElement>::Mask, N> = intrinsics::simd_eq(*self, *other);
let tfvec: Simd<<T as SimdElement>::Mask, N> =
core::intrinsics::simd::simd_eq(*self, *other);
Mask::from_int_unchecked(tfvec)
};
@ -705,7 +705,8 @@ where
fn ne(&self, other: &Self) -> bool {
// Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask.
let mask = unsafe {
let tfvec: Simd<<T as SimdElement>::Mask, N> = intrinsics::simd_ne(*self, *other);
let tfvec: Simd<<T as SimdElement>::Mask, N> =
core::intrinsics::simd::simd_ne(*self, *other);
Mask::from_int_unchecked(tfvec)
};