make prefetch intrinsics safe
This commit is contained in:
Folkert de Vries
parent
51df7aabbe
commit
d25910eaeb
5 changed files with 70 additions and 72 deletions
|
|
@ -330,10 +330,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
|
|||
_ => bug!(),
|
||||
};
|
||||
let ptr = args[0].immediate();
|
||||
let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_u32() as i32;
|
||||
self.call_intrinsic(
|
||||
"llvm.prefetch",
|
||||
&[self.val_ty(ptr)],
|
||||
&[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)],
|
||||
&[
|
||||
ptr,
|
||||
self.const_i32(rw),
|
||||
self.const_i32(locality),
|
||||
self.const_i32(cache_type),
|
||||
],
|
||||
)
|
||||
}
|
||||
sym::carrying_mul_add => {
|
||||
|
|
|
|||
|
|
@ -136,6 +136,10 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi
|
|||
| sym::round_ties_even_f64
|
||||
| sym::round_ties_even_f128
|
||||
| sym::autodiff
|
||||
| sym::prefetch_read_data
|
||||
| sym::prefetch_write_data
|
||||
| sym::prefetch_read_instruction
|
||||
| sym::prefetch_write_instruction
|
||||
| sym::const_eval_select => hir::Safety::Safe,
|
||||
_ => hir::Safety::Unsafe,
|
||||
};
|
||||
|
|
@ -218,7 +222,7 @@ pub(crate) fn check_intrinsic_type(
|
|||
| sym::prefetch_write_data
|
||||
| sym::prefetch_read_instruction
|
||||
| sym::prefetch_write_instruction => {
|
||||
(1, 0, vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.i32], tcx.types.unit)
|
||||
(1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.types.unit)
|
||||
}
|
||||
sym::needs_drop => (1, 0, vec![], tcx.types.bool),
|
||||
|
||||
|
|
|
|||
|
|
@ -265,17 +265,16 @@ pub unsafe fn atomic_singlethreadfence<const ORD: AtomicOrdering>();
|
|||
/// Prefetches have no effect on the behavior of the program but can change its performance
|
||||
/// characteristics.
|
||||
///
|
||||
/// The `locality` argument must be a constant integer and is a temporal locality specifier
|
||||
/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
|
||||
/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
|
||||
/// to (3) - extremely local keep in cache.
|
||||
///
|
||||
/// This intrinsic does not have a stable counterpart.
|
||||
#[rustc_intrinsic]
|
||||
#[rustc_nounwind]
|
||||
#[miri::intrinsic_fallback_is_spec]
|
||||
pub unsafe fn prefetch_read_data<T>(data: *const T, locality: i32) {
|
||||
pub const fn prefetch_read_data<T, const LOCALITY: i32>(data: *const T) {
|
||||
// This operation is a no-op, unless it is overridden by the backend.
|
||||
let _ = data;
|
||||
let _ = locality;
|
||||
}
|
||||
|
||||
/// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
|
||||
|
|
@ -283,17 +282,16 @@ pub unsafe fn prefetch_read_data<T>(data: *const T, locality: i32) {
|
|||
/// Prefetches have no effect on the behavior of the program but can change its performance
|
||||
/// characteristics.
|
||||
///
|
||||
/// The `locality` argument must be a constant integer and is a temporal locality specifier
|
||||
/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
|
||||
/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
|
||||
/// to (3) - extremely local keep in cache.
|
||||
///
|
||||
/// This intrinsic does not have a stable counterpart.
|
||||
#[rustc_intrinsic]
|
||||
#[rustc_nounwind]
|
||||
#[miri::intrinsic_fallback_is_spec]
|
||||
pub unsafe fn prefetch_write_data<T>(data: *const T, locality: i32) {
|
||||
pub const fn prefetch_write_data<T, const LOCALITY: i32>(data: *const T) {
|
||||
// This operation is a no-op, unless it is overridden by the backend.
|
||||
let _ = data;
|
||||
let _ = locality;
|
||||
}
|
||||
|
||||
/// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
|
||||
|
|
@ -301,17 +299,16 @@ pub unsafe fn prefetch_write_data<T>(data: *const T, locality: i32) {
|
|||
/// Prefetches have no effect on the behavior of the program but can change its performance
|
||||
/// characteristics.
|
||||
///
|
||||
/// The `locality` argument must be a constant integer and is a temporal locality specifier
|
||||
/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
|
||||
/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
|
||||
/// to (3) - extremely local keep in cache.
|
||||
///
|
||||
/// This intrinsic does not have a stable counterpart.
|
||||
#[rustc_intrinsic]
|
||||
#[rustc_nounwind]
|
||||
#[miri::intrinsic_fallback_is_spec]
|
||||
pub unsafe fn prefetch_read_instruction<T>(data: *const T, locality: i32) {
|
||||
pub const fn prefetch_read_instruction<T, const LOCALITY: i32>(data: *const T) {
|
||||
// This operation is a no-op, unless it is overridden by the backend.
|
||||
let _ = data;
|
||||
let _ = locality;
|
||||
}
|
||||
|
||||
/// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
|
||||
|
|
@ -319,17 +316,16 @@ pub unsafe fn prefetch_read_instruction<T>(data: *const T, locality: i32) {
|
|||
/// Prefetches have no effect on the behavior of the program but can change its performance
|
||||
/// characteristics.
|
||||
///
|
||||
/// The `locality` argument must be a constant integer and is a temporal locality specifier
|
||||
/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
|
||||
/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
|
||||
/// to (3) - extremely local keep in cache.
|
||||
///
|
||||
/// This intrinsic does not have a stable counterpart.
|
||||
#[rustc_intrinsic]
|
||||
#[rustc_nounwind]
|
||||
#[miri::intrinsic_fallback_is_spec]
|
||||
pub unsafe fn prefetch_write_instruction<T>(data: *const T, locality: i32) {
|
||||
pub const fn prefetch_write_instruction<T, const LOCALITY: i32>(data: *const T) {
|
||||
// This operation is a no-op, unless it is overridden by the backend.
|
||||
let _ = data;
|
||||
let _ = locality;
|
||||
}
|
||||
|
||||
/// Executes a breakpoint trap, for inspection by a debugger.
|
||||
|
|
|
|||
|
|
@ -1,23 +1,23 @@
|
|||
#![feature(core_intrinsics)]
|
||||
|
||||
// Test that these intrinsics work. Their behavior should be a no-op.
|
||||
|
||||
fn main() {
|
||||
static X: [u8; 8] = [0; 8];
|
||||
|
||||
unsafe {
|
||||
::std::intrinsics::prefetch_read_data(::std::ptr::null::<u8>(), 1);
|
||||
::std::intrinsics::prefetch_read_data(::std::ptr::dangling::<u8>(), 2);
|
||||
::std::intrinsics::prefetch_read_data(X.as_ptr(), 3);
|
||||
::std::intrinsics::prefetch_read_data::<_, 1>(::std::ptr::null::<u8>());
|
||||
::std::intrinsics::prefetch_read_data::<_, 2>(::std::ptr::dangling::<u8>());
|
||||
::std::intrinsics::prefetch_read_data::<_, 3>(X.as_ptr());
|
||||
|
||||
::std::intrinsics::prefetch_write_data(::std::ptr::null::<u8>(), 1);
|
||||
::std::intrinsics::prefetch_write_data(::std::ptr::dangling::<u8>(), 2);
|
||||
::std::intrinsics::prefetch_write_data(X.as_ptr(), 3);
|
||||
::std::intrinsics::prefetch_write_data::<_, 1>(::std::ptr::null::<u8>());
|
||||
::std::intrinsics::prefetch_write_data::<_, 2>(::std::ptr::dangling::<u8>());
|
||||
::std::intrinsics::prefetch_write_data::<_, 3>(X.as_ptr());
|
||||
|
||||
::std::intrinsics::prefetch_read_instruction(::std::ptr::null::<u8>(), 1);
|
||||
::std::intrinsics::prefetch_read_instruction(::std::ptr::dangling::<u8>(), 2);
|
||||
::std::intrinsics::prefetch_read_instruction(X.as_ptr(), 3);
|
||||
::std::intrinsics::prefetch_read_instruction::<_, 1>(::std::ptr::null::<u8>());
|
||||
::std::intrinsics::prefetch_read_instruction::<_, 2>(::std::ptr::dangling::<u8>());
|
||||
::std::intrinsics::prefetch_read_instruction::<_, 3>(X.as_ptr());
|
||||
|
||||
::std::intrinsics::prefetch_write_instruction(::std::ptr::null::<u8>(), 1);
|
||||
::std::intrinsics::prefetch_write_instruction(::std::ptr::dangling::<u8>(), 2);
|
||||
::std::intrinsics::prefetch_write_instruction(X.as_ptr(), 3);
|
||||
}
|
||||
::std::intrinsics::prefetch_write_instruction::<_, 1>(::std::ptr::null::<u8>());
|
||||
::std::intrinsics::prefetch_write_instruction::<_, 2>(::std::ptr::dangling::<u8>());
|
||||
::std::intrinsics::prefetch_write_instruction::<_, 3>(X.as_ptr());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,56 +9,48 @@ use std::intrinsics::{
|
|||
|
||||
#[no_mangle]
|
||||
pub fn check_prefetch_read_data(data: &[i8]) {
|
||||
unsafe {
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1)
|
||||
prefetch_read_data(data.as_ptr(), 0);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1)
|
||||
prefetch_read_data(data.as_ptr(), 1);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1)
|
||||
prefetch_read_data(data.as_ptr(), 2);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1)
|
||||
prefetch_read_data(data.as_ptr(), 3);
|
||||
}
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1)
|
||||
prefetch_read_data::<_, 0>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1)
|
||||
prefetch_read_data::<_, 1>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1)
|
||||
prefetch_read_data::<_, 2>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1)
|
||||
prefetch_read_data::<_, 3>(data.as_ptr());
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub fn check_prefetch_write_data(data: &[i8]) {
|
||||
unsafe {
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1)
|
||||
prefetch_write_data(data.as_ptr(), 0);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1)
|
||||
prefetch_write_data(data.as_ptr(), 1);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1)
|
||||
prefetch_write_data(data.as_ptr(), 2);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1)
|
||||
prefetch_write_data(data.as_ptr(), 3);
|
||||
}
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1)
|
||||
prefetch_write_data::<_, 0>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1)
|
||||
prefetch_write_data::<_, 1>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1)
|
||||
prefetch_write_data::<_, 2>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1)
|
||||
prefetch_write_data::<_, 3>(data.as_ptr());
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub fn check_prefetch_read_instruction(data: &[i8]) {
|
||||
unsafe {
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0)
|
||||
prefetch_read_instruction(data.as_ptr(), 0);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0)
|
||||
prefetch_read_instruction(data.as_ptr(), 1);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0)
|
||||
prefetch_read_instruction(data.as_ptr(), 2);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0)
|
||||
prefetch_read_instruction(data.as_ptr(), 3);
|
||||
}
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0)
|
||||
prefetch_read_instruction::<_, 0>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0)
|
||||
prefetch_read_instruction::<_, 1>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0)
|
||||
prefetch_read_instruction::<_, 2>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0)
|
||||
prefetch_read_instruction::<_, 3>(data.as_ptr());
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub fn check_prefetch_write_instruction(data: &[i8]) {
|
||||
unsafe {
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0)
|
||||
prefetch_write_instruction(data.as_ptr(), 0);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0)
|
||||
prefetch_write_instruction(data.as_ptr(), 1);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0)
|
||||
prefetch_write_instruction(data.as_ptr(), 2);
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0)
|
||||
prefetch_write_instruction(data.as_ptr(), 3);
|
||||
}
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0)
|
||||
prefetch_write_instruction::<_, 0>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0)
|
||||
prefetch_write_instruction::<_, 1>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0)
|
||||
prefetch_write_instruction::<_, 2>(data.as_ptr());
|
||||
// CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0)
|
||||
prefetch_write_instruction::<_, 3>(data.as_ptr());
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue