diff --git a/src/tools/miri/src/shims/intrinsics/atomic.rs b/src/tools/miri/src/intrinsics/atomic.rs similarity index 100% rename from src/tools/miri/src/shims/intrinsics/atomic.rs rename to src/tools/miri/src/intrinsics/atomic.rs diff --git a/src/tools/miri/src/shims/intrinsics/mod.rs b/src/tools/miri/src/intrinsics/mod.rs similarity index 100% rename from src/tools/miri/src/shims/intrinsics/mod.rs rename to src/tools/miri/src/intrinsics/mod.rs diff --git a/src/tools/miri/src/shims/intrinsics/simd.rs b/src/tools/miri/src/intrinsics/simd.rs similarity index 100% rename from src/tools/miri/src/shims/intrinsics/simd.rs rename to src/tools/miri/src/intrinsics/simd.rs diff --git a/src/tools/miri/src/lib.rs b/src/tools/miri/src/lib.rs index f47c84842b17..f01c2da25d9e 100644 --- a/src/tools/miri/src/lib.rs +++ b/src/tools/miri/src/lib.rs @@ -79,6 +79,7 @@ mod concurrency; mod diagnostics; mod eval; mod helpers; +mod intrinsics; mod machine; mod mono_hash_map; mod operator; @@ -95,9 +96,9 @@ pub use rustc_const_eval::interpret::*; #[doc(no_inline)] pub use rustc_const_eval::interpret::{self, AllocMap, PlaceTy, Provenance as _}; +pub use crate::intrinsics::EvalContextExt as _; pub use crate::shims::env::{EnvVars, EvalContextExt as _}; pub use crate::shims::foreign_items::{DynSym, EvalContextExt as _}; -pub use crate::shims::intrinsics::EvalContextExt as _; pub use crate::shims::os_str::EvalContextExt as _; pub use crate::shims::panic::{CatchUnwindData, EvalContextExt as _}; pub use crate::shims::time::EvalContextExt as _; diff --git a/src/tools/miri/src/shims/foreign_items.rs b/src/tools/miri/src/shims/foreign_items.rs index 4b96ff18b74c..6a12180b836b 100644 --- a/src/tools/miri/src/shims/foreign_items.rs +++ b/src/tools/miri/src/shims/foreign_items.rs @@ -893,6 +893,38 @@ trait EvalContextExtPriv<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { throw_unsup_format!("unsupported `llvm.prefetch` type argument: {}", ty); } } + // Used to implement the x86 `_mm{,256,512}_popcnt_epi{8,16,32,64}` and wasm + // `{i,u}8x16_popcnt` functions. + name if name.starts_with("llvm.ctpop.v") => { + let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; + + let (op, op_len) = this.operand_to_simd(op)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + + assert_eq!(dest_len, op_len); + + for i in 0..dest_len { + let op = this.read_immediate(&this.project_index(&op, i)?)?; + // Use `to_uint` to get a zero-extended `u128`. Those + // extra zeros will not affect `count_ones`. + let res = op.to_scalar().to_uint(op.layout.size)?.count_ones(); + + this.write_scalar( + Scalar::from_uint(res, op.layout.size), + &this.project_index(&dest, i)?, + )?; + } + } + + // Target-specific shims + name if name.starts_with("llvm.x86.") + && (this.tcx.sess.target.arch == "x86" + || this.tcx.sess.target.arch == "x86_64") => + { + return shims::x86::EvalContextExt::emulate_x86_intrinsic( + this, link_name, abi, args, dest, + ); + } // FIXME: Move these to an `arm` submodule. "llvm.aarch64.isb" if this.tcx.sess.target.arch == "aarch64" => { let [arg] = this.check_shim(abi, Abi::Unadjusted, link_name, args)?; @@ -923,38 +955,6 @@ trait EvalContextExtPriv<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } } - // Used to implement the x86 `_mm{,256,512}_popcnt_epi{8,16,32,64}` and wasm - // `{i,u}8x16_popcnt` functions. - name if name.starts_with("llvm.ctpop.v") => { - let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - - assert_eq!(dest_len, op_len); - - for i in 0..dest_len { - let op = this.read_immediate(&this.project_index(&op, i)?)?; - // Use `to_uint` to get a zero-extended `u128`. Those - // extra zeros will not affect `count_ones`. - let res = op.to_scalar().to_uint(op.layout.size)?.count_ones(); - - this.write_scalar( - Scalar::from_uint(res, op.layout.size), - &this.project_index(&dest, i)?, - )?; - } - } - - name if name.starts_with("llvm.x86.") - && (this.tcx.sess.target.arch == "x86" - || this.tcx.sess.target.arch == "x86_64") => - { - return shims::x86::EvalContextExt::emulate_x86_intrinsic( - this, link_name, abi, args, dest, - ); - } - // Platform-specific shims _ => return match this.tcx.sess.target.os.as_ref() { diff --git a/src/tools/miri/src/shims/mod.rs b/src/tools/miri/src/shims/mod.rs index 85c9a202f7da..ab5a8c5877ec 100644 --- a/src/tools/miri/src/shims/mod.rs +++ b/src/tools/miri/src/shims/mod.rs @@ -5,7 +5,6 @@ mod backtrace; #[cfg(target_os = "linux")] pub mod ffi_support; pub mod foreign_items; -pub mod intrinsics; pub mod unix; pub mod windows; mod x86; diff --git a/src/tools/miri/tests/fail/shims/intrinsic_target_feature.rs b/src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.rs similarity index 100% rename from src/tools/miri/tests/fail/shims/intrinsic_target_feature.rs rename to src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.rs diff --git a/src/tools/miri/tests/fail/shims/intrinsic_target_feature.stderr b/src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.stderr similarity index 100% rename from src/tools/miri/tests/fail/shims/intrinsic_target_feature.stderr rename to src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.stderr diff --git a/src/tools/miri/tests/pass/ints.rs b/src/tools/miri/tests/pass/integers.rs similarity index 100% rename from src/tools/miri/tests/pass/ints.rs rename to src/tools/miri/tests/pass/integers.rs diff --git a/src/tools/miri/tests/pass/intrinsics-integer.rs b/src/tools/miri/tests/pass/intrinsics/integer.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-integer.rs rename to src/tools/miri/tests/pass/intrinsics/integer.rs diff --git a/src/tools/miri/tests/pass/intrinsics.rs b/src/tools/miri/tests/pass/intrinsics/intrinsics.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics.rs rename to src/tools/miri/tests/pass/intrinsics/intrinsics.rs diff --git a/src/tools/miri/tests/pass/portable-simd-ptrs.rs b/src/tools/miri/tests/pass/intrinsics/portable-simd-ptrs.rs similarity index 100% rename from src/tools/miri/tests/pass/portable-simd-ptrs.rs rename to src/tools/miri/tests/pass/intrinsics/portable-simd-ptrs.rs diff --git a/src/tools/miri/tests/pass/portable-simd.rs b/src/tools/miri/tests/pass/intrinsics/portable-simd.rs similarity index 100% rename from src/tools/miri/tests/pass/portable-simd.rs rename to src/tools/miri/tests/pass/intrinsics/portable-simd.rs diff --git a/src/tools/miri/tests/pass/volatile.rs b/src/tools/miri/tests/pass/intrinsics/volatile.rs similarity index 100% rename from src/tools/miri/tests/pass/volatile.rs rename to src/tools/miri/tests/pass/intrinsics/volatile.rs diff --git a/src/tools/miri/tests/pass/available-parallelism-miri-num-cpus.rs b/src/tools/miri/tests/pass/shims/available-parallelism-miri-num-cpus.rs similarity index 100% rename from src/tools/miri/tests/pass/available-parallelism-miri-num-cpus.rs rename to src/tools/miri/tests/pass/shims/available-parallelism-miri-num-cpus.rs diff --git a/src/tools/miri/tests/pass/available-parallelism.rs b/src/tools/miri/tests/pass/shims/available-parallelism.rs similarity index 100% rename from src/tools/miri/tests/pass/available-parallelism.rs rename to src/tools/miri/tests/pass/shims/available-parallelism.rs diff --git a/src/tools/miri/tests/pass/intrinsics-x86-aes-vaes.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-aes-vaes.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-x86-aes-vaes.rs rename to src/tools/miri/tests/pass/shims/x86/intrinsics-x86-aes-vaes.rs diff --git a/src/tools/miri/tests/pass/intrinsics-x86-avx.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-x86-avx.rs rename to src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx.rs diff --git a/src/tools/miri/tests/pass/intrinsics-x86-avx2.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx2.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-x86-avx2.rs rename to src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx2.rs diff --git a/src/tools/miri/tests/pass/intrinsics-x86-avx512.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-x86-avx512.rs rename to src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs diff --git a/src/tools/miri/tests/pass/intrinsics-x86-pause-without-sse2.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-pause-without-sse2.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-x86-pause-without-sse2.rs rename to src/tools/miri/tests/pass/shims/x86/intrinsics-x86-pause-without-sse2.rs diff --git a/src/tools/miri/tests/pass/intrinsics-x86-sse.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-x86-sse.rs rename to src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse.rs diff --git a/src/tools/miri/tests/pass/intrinsics-x86-sse2.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse2.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-x86-sse2.rs rename to src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse2.rs diff --git a/src/tools/miri/tests/pass/intrinsics-x86-sse3-ssse3.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse3-ssse3.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-x86-sse3-ssse3.rs rename to src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse3-ssse3.rs diff --git a/src/tools/miri/tests/pass/intrinsics-x86-sse41.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse41.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-x86-sse41.rs rename to src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse41.rs diff --git a/src/tools/miri/tests/pass/intrinsics-x86.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86.rs similarity index 100% rename from src/tools/miri/tests/pass/intrinsics-x86.rs rename to src/tools/miri/tests/pass/shims/x86/intrinsics-x86.rs