user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #452 from Amanieu/intrinsic_mod

Browse source
This commit is contained in:
Amanieu d'Antras 2022-02-06 09:49:01 +00:00 • committed by GitHub
commit e346c7c408
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 482 additions and 478 deletions
Download patch file Download diff file Expand all files Collapse all files

2
library/compiler-builtins/ci/run-docker.sh
View file

@ -19,7 +19,7 @@ run() {
-e CARGO_HOME=/cargo \
-e CARGO_TARGET_DIR=/target \
-e RUST_COMPILER_RT_ROOT \
-v $(dirname $(dirname `which cargo`)):/cargo \
-v "${HOME}/.cargo":/cargo \
-v `pwd`/target:/target \
-v `pwd`:/checkout:ro \
-v `rustc --print sysroot`:/rust:ro \

398
library/compiler-builtins/src/arm.rs
View file

@ -3,251 +3,181 @@
use core::intrinsics;
// NOTE This function and the ones below are implemented using assembly because they are using a
// custom calling convention which can't be implemented using a normal Rust function.
// NOTE The only difference between the iOS and non-iOS versions of those functions is that the iOS
// versions use 3 leading underscores in the names of called functions instead of 2.
#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
#[naked]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn __aeabi_uidivmod() {
core::arch::asm!(
"push {{lr}}",
"sub sp, sp, #4",
"mov r2, sp",
"bl __udivmodsi4",
"ldr r1, [sp]",
"add sp, sp, #4",
"pop {{pc}}",
options(noreturn)
);
}
// iOS symbols have a leading underscore.
#[cfg(target_os = "ios")]
#[naked]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn __aeabi_uidivmod() {
core::arch::asm!(
"push {{lr}}",
"sub sp, sp, #4",
"mov r2, sp",
"bl ___udivmodsi4",
"ldr r1, [sp]",
"add sp, sp, #4",
"pop {{pc}}",
options(noreturn)
);
macro_rules! bl {
($func:literal) => {
concat!("bl _", $func)
};
}
#[cfg(not(target_os = "ios"))]
#[naked]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn __aeabi_uldivmod() {
core::arch::asm!(
"push {{r4, lr}}",
"sub sp, sp, #16",
"add r4, sp, #8",
"str r4, [sp]",
"bl __udivmoddi4",
"ldr r2, [sp, #8]",
"ldr r3, [sp, #12]",
"add sp, sp, #16",
"pop {{r4, pc}}",
options(noreturn)
);
macro_rules! bl {
($func:literal) => {
concat!("bl ", $func)
};
}
#[cfg(target_os = "ios")]
#[naked]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn __aeabi_uldivmod() {
core::arch::asm!(
"push {{r4, lr}}",
"sub sp, sp, #16",
"add r4, sp, #8",
"str r4, [sp]",
"bl ___udivmoddi4",
"ldr r2, [sp, #8]",
"ldr r3, [sp, #12]",
"add sp, sp, #16",
"pop {{r4, pc}}",
options(noreturn)
);
}
#[cfg(not(target_os = "ios"))]
#[naked]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn __aeabi_idivmod() {
core::arch::asm!(
"push {{r0, r1, r4, lr}}",
"bl __aeabi_idiv",
"pop {{r1, r2}}",
"muls r2, r2, r0",
"subs r1, r1, r2",
"pop {{r4, pc}}",
options(noreturn)
);
}
#[cfg(target_os = "ios")]
#[naked]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn __aeabi_idivmod() {
core::arch::asm!(
"push {{r0, r1, r4, lr}}",
"bl ___aeabi_idiv",
"pop {{r1, r2}}",
"muls r2, r2, r0",
"subs r1, r1, r2",
"pop {{r4, pc}}",
options(noreturn)
);
}
#[cfg(not(target_os = "ios"))]
#[naked]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn __aeabi_ldivmod() {
core::arch::asm!(
"push {{r4, lr}}",
"sub sp, sp, #16",
"add r4, sp, #8",
"str r4, [sp]",
"bl __divmoddi4",
"ldr r2, [sp, #8]",
"ldr r3, [sp, #12]",
"add sp, sp, #16",
"pop {{r4, pc}}",
options(noreturn)
);
}
#[cfg(target_os = "ios")]
#[naked]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn __aeabi_ldivmod() {
core::arch::asm!(
"push {{r4, lr}}",
"sub sp, sp, #16",
"add r4, sp, #8",
"str r4, [sp]",
"bl ___divmoddi4",
"ldr r2, [sp, #8]",
"ldr r3, [sp, #12]",
"add sp, sp, #16",
"pop {{r4, pc}}",
options(noreturn)
);
}
// The following functions use weak linkage to allow users to override
// with custom implementation.
// FIXME: The `*4` and `*8` variants should be defined as aliases.
#[cfg(not(target_os = "ios"))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) {
::mem::memcpy(dest, src, n);
}
#[cfg(not(target_os = "ios"))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, mut n: usize) {
// We are guaranteed 4-alignment, so accessing at u32 is okay.
let mut dest = dest as *mut u32;
let mut src = src as *mut u32;
while n >= 4 {
*dest = *src;
dest = dest.offset(1);
src = src.offset(1);
n -= 4;
intrinsics! {
// NOTE This function and the ones below are implemented using assembly because they are using a
// custom calling convention which can't be implemented using a normal Rust function.
#[naked]
#[cfg(not(target_env = "msvc"))]
pub unsafe extern "C" fn __aeabi_uidivmod() {
core::arch::asm!(
"push {{lr}}",
"sub sp, sp, #4",
"mov r2, sp",
bl!("__udivmodsi4"),
"ldr r1, [sp]",
"add sp, sp, #4",
"pop {{pc}}",
options(noreturn)
);
}
__aeabi_memcpy(dest as *mut u8, src as *const u8, n);
}
#[cfg(not(target_os = "ios"))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) {
__aeabi_memcpy4(dest, src, n);
}
#[cfg(not(target_os = "ios"))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) {
::mem::memmove(dest, src, n);
}
#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) {
__aeabi_memmove(dest, src, n);
}
#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) {
__aeabi_memmove(dest, src, n);
}
#[cfg(not(target_os = "ios"))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) {
// Note the different argument order
::mem::memset(dest, c, n);
}
#[cfg(not(target_os = "ios"))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, mut n: usize, c: i32) {
let mut dest = dest as *mut u32;
let byte = (c as u32) & 0xff;
let c = (byte << 24) | (byte << 16) | (byte << 8) | byte;
while n >= 4 {
*dest = c;
dest = dest.offset(1);
n -= 4;
#[naked]
pub unsafe extern "C" fn __aeabi_uldivmod() {
core::arch::asm!(
"push {{r4, lr}}",
"sub sp, sp, #16",
"add r4, sp, #8",
"str r4, [sp]",
bl!("__udivmodsi4"),
"ldr r2, [sp, #8]",
"ldr r3, [sp, #12]",
"add sp, sp, #16",
"pop {{r4, pc}}",
options(noreturn)
);
}
__aeabi_memset(dest as *mut u8, n, byte as i32);
}
#[naked]
pub unsafe extern "C" fn __aeabi_idivmod() {
core::arch::asm!(
"push {{r0, r1, r4, lr}}",
bl!("__aeabi_idiv"),
"pop {{r1, r2}}",
"muls r2, r2, r0",
"subs r1, r1, r2",
"pop {{r4, pc}}",
options(noreturn)
);
}
#[cfg(not(target_os = "ios"))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) {
__aeabi_memset4(dest, n, c);
}
#[naked]
pub unsafe extern "C" fn __aeabi_ldivmod() {
core::arch::asm!(
"push {{r4, lr}}",
"sub sp, sp, #16",
"add r4, sp, #8",
"str r4, [sp]",
bl!("__divmoddi4"),
"ldr r2, [sp, #8]",
"ldr r3, [sp, #12]",
"add sp, sp, #16",
"pop {{r4, pc}}",
options(noreturn)
);
}
#[cfg(not(target_os = "ios"))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) {
__aeabi_memset(dest, n, 0);
}
// The following functions use weak linkage to allow users to override
// with custom implementation.
// FIXME: The `*4` and `*8` variants should be defined as aliases.
#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) {
__aeabi_memset4(dest, n, 0);
}
#[cfg(not(target_os = "ios"))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) {
::mem::memcpy(dest, src, n);
}
#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) {
__aeabi_memset4(dest, n, 0);
#[cfg(not(target_os = "ios"))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, n: usize) {
// We are guaranteed 4-alignment, so accessing at u32 is okay.
let mut dest = dest as *mut u32;
let mut src = src as *mut u32;
let mut n = n;
while n >= 4 {
*dest = *src;
dest = dest.offset(1);
src = src.offset(1);
n -= 4;
}
__aeabi_memcpy(dest as *mut u8, src as *const u8, n);
}
#[cfg(not(target_os = "ios"))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) {
__aeabi_memcpy4(dest, src, n);
}
#[cfg(not(target_os = "ios"))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) {
::mem::memmove(dest, src, n);
}
#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) {
__aeabi_memmove(dest, src, n);
}
#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) {
__aeabi_memmove(dest, src, n);
}
#[cfg(not(target_os = "ios"))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) {
// Note the different argument order
::mem::memset(dest, c, n);
}
#[cfg(not(target_os = "ios"))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, n: usize, c: i32) {
let mut dest = dest as *mut u32;
let mut n = n;
let byte = (c as u32) & 0xff;
let c = (byte << 24) | (byte << 16) | (byte << 8) | byte;
while n >= 4 {
*dest = c;
dest = dest.offset(1);
n -= 4;
}
__aeabi_memset(dest as *mut u8, n, byte as i32);
}
#[cfg(not(target_os = "ios"))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) {
__aeabi_memset4(dest, n, c);
}
#[cfg(not(target_os = "ios"))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) {
__aeabi_memset(dest, n, 0);
}
#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) {
__aeabi_memset4(dest, n, 0);
}
#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
#[linkage = "weak"]
pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) {
__aeabi_memset4(dest, n, 0);
}
}

21
library/compiler-builtins/src/arm_linux.rs
View file

@ -90,17 +90,19 @@ unsafe fn atomic_cmpxchg<T>(ptr: *mut T, oldval: u32, newval: u32) -> u32 {
macro_rules! atomic_rmw {
($name:ident, $ty:ty, $op:expr) => {
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty {
atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty
intrinsics! {
pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty {
atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty
}
}
};
}
macro_rules! atomic_cmpxchg {
($name:ident, $ty:ty) => {
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty {
atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty
intrinsics! {
pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty {
atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty
}
}
};
}
@ -205,7 +207,8 @@ atomic_cmpxchg!(__sync_val_compare_and_swap_1, u8);
atomic_cmpxchg!(__sync_val_compare_and_swap_2, u16);
atomic_cmpxchg!(__sync_val_compare_and_swap_4, u32);
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern "C" fn __sync_synchronize() {
__kuser_memory_barrier();
intrinsics! {
pub unsafe extern "C" fn __sync_synchronize() {
__kuser_memory_barrier();
}
}

119
library/compiler-builtins/src/macros.rs
View file

@ -76,7 +76,7 @@ macro_rules! intrinsics {
(
#[maybe_use_optimized_c_shim]
$(#[$($attr:tt)*])*
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty {
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
$($body:tt)*
}
@ -84,9 +84,9 @@ macro_rules! intrinsics {
) => (
#[cfg($name = "optimized-c")]
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
extern $abi {
fn $name($($argname: $ty),*) -> $ret;
fn $name($($argname: $ty),*) $(-> $ret)?;
}
unsafe {
$name($($argname),*)
@ -96,7 +96,7 @@ macro_rules! intrinsics {
#[cfg(not($name = "optimized-c"))]
intrinsics! {
$(#[$($attr)*])*
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
}
@ -110,7 +110,7 @@ macro_rules! intrinsics {
(
#[aapcs_on_arm]
$(#[$($attr:tt)*])*
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty {
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
$($body:tt)*
}
@ -119,7 +119,7 @@ macro_rules! intrinsics {
#[cfg(target_arch = "arm")]
intrinsics! {
$(#[$($attr)*])*
pub extern "aapcs" fn $name( $($argname: $ty),* ) -> $ret {
pub extern "aapcs" fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
}
@ -127,7 +127,7 @@ macro_rules! intrinsics {
#[cfg(not(target_arch = "arm"))]
intrinsics! {
$(#[$($attr)*])*
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
}
@ -140,7 +140,7 @@ macro_rules! intrinsics {
(
#[unadjusted_on_win64]
$(#[$($attr:tt)*])*
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty {
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
$($body:tt)*
}
@ -149,7 +149,7 @@ macro_rules! intrinsics {
#[cfg(all(windows, target_pointer_width = "64"))]
intrinsics! {
$(#[$($attr)*])*
pub extern "unadjusted" fn $name( $($argname: $ty),* ) -> $ret {
pub extern "unadjusted" fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
}
@ -157,7 +157,7 @@ macro_rules! intrinsics {
#[cfg(not(all(windows, target_pointer_width = "64")))]
intrinsics! {
$(#[$($attr)*])*
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
}
@ -175,7 +175,7 @@ macro_rules! intrinsics {
(
#[win64_128bit_abi_hack]
$(#[$($attr:tt)*])*
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty {
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
$($body:tt)*
}
@ -183,7 +183,7 @@ macro_rules! intrinsics {
) => (
#[cfg(all(windows, target_arch = "x86_64"))]
$(#[$($attr)*])*
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
@ -193,7 +193,7 @@ macro_rules! intrinsics {
pub extern $abi fn $name( $($argname: $ty),* )
-> ::macros::win64_128bit_abi_hack::U64x2
{
let e: $ret = super::$name($($argname),*);
let e: $($ret)? = super::$name($($argname),*);
::macros::win64_128bit_abi_hack::U64x2::from(e)
}
}
@ -201,7 +201,7 @@ macro_rules! intrinsics {
#[cfg(not(all(windows, target_arch = "x86_64")))]
intrinsics! {
$(#[$($attr)*])*
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
}
@ -216,21 +216,21 @@ macro_rules! intrinsics {
(
#[arm_aeabi_alias = $alias:ident]
$(#[$($attr:tt)*])*
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty {
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
$($body:tt)*
}
$($rest:tt)*
) => (
#[cfg(target_arch = "arm")]
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
#[cfg(target_arch = "arm")]
pub mod $name {
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
super::$name($($argname),*)
}
}
@ -238,7 +238,7 @@ macro_rules! intrinsics {
#[cfg(target_arch = "arm")]
pub mod $alias {
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub extern "aapcs" fn $alias( $($argname: $ty),* ) -> $ret {
pub extern "aapcs" fn $alias( $($argname: $ty),* ) $(-> $ret)? {
super::$name($($argname),*)
}
}
@ -246,7 +246,57 @@ macro_rules! intrinsics {
#[cfg(not(target_arch = "arm"))]
intrinsics! {
$(#[$($attr)*])*
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
}
intrinsics!($($rest)*);
);
// C mem* functions are only generated when the "mem" feature is enabled.
(
#[mem_builtin]
$(#[$($attr:tt)*])*
pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
$($body:tt)*
}
$($rest:tt)*
) => (
$(#[$($attr)*])*
pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
#[cfg(feature = "mem")]
pub mod $name {
$(#[$($attr)*])*
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
super::$name($($argname),*)
}
}
intrinsics!($($rest)*);
);
// Naked functions are special: we can't generate wrappers for them since
// they use a custom calling convention.
(
#[naked]
$(#[$($attr:tt)*])*
pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
$($body:tt)*
}
$($rest:tt)*
) => (
pub mod $name {
#[naked]
$(#[$($attr)*])*
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
}
@ -268,21 +318,46 @@ macro_rules! intrinsics {
// input we were given.
(
$(#[$($attr:tt)*])*
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty {
pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
$($body:tt)*
}
$($rest:tt)*
) => (
$(#[$($attr)*])*
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
pub mod $name {
$(#[$($attr)*])*
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
super::$name($($argname),*)
}
}
intrinsics!($($rest)*);
);
// Same as the above for unsafe functions.
(
$(#[$($attr:tt)*])*
pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
$($body:tt)*
}
$($rest:tt)*
) => (
$(#[$($attr)*])*
pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
$($body)*
}
pub mod $name {
$(#[$($attr)*])*
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
super::$name($($argname),*)
}
}

114
library/compiler-builtins/src/mem/mod.rs
View file

@ -19,53 +19,55 @@ use core::ops::{BitOr, Shl};
)]
mod impls;
#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
impls::copy_forward(dest, src, n);
dest
}
#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
let delta = (dest as usize).wrapping_sub(src as usize);
if delta >= n {
// We can copy forwards because either dest is far enough ahead of src,
// or src is ahead of dest (and delta overflowed).
intrinsics! {
#[mem_builtin]
#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
impls::copy_forward(dest, src, n);
} else {
impls::copy_backward(dest, src, n);
dest
}
dest
}
#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
pub unsafe extern "C" fn memset(s: *mut u8, c: c_int, n: usize) -> *mut u8 {
impls::set_bytes(s, c as u8, n);
s
}
#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
let mut i = 0;
while i < n {
let a = *s1.add(i);
let b = *s2.add(i);
if a != b {
return a as i32 - b as i32;
#[mem_builtin]
#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
let delta = (dest as usize).wrapping_sub(src as usize);
if delta >= n {
// We can copy forwards because either dest is far enough ahead of src,
// or src is ahead of dest (and delta overflowed).
impls::copy_forward(dest, src, n);
} else {
impls::copy_backward(dest, src, n);
}
i += 1;
dest
}
0
}
#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
memcmp(s1, s2, n)
#[mem_builtin]
#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
pub unsafe extern "C" fn memset(s: *mut u8, c: crate::mem::c_int, n: usize) -> *mut u8 {
impls::set_bytes(s, c as u8, n);
s
}
#[mem_builtin]
#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
let mut i = 0;
while i < n {
let a = *s1.add(i);
let b = *s2.add(i);
if a != b {
return a as i32 - b as i32;
}
i += 1;
}
0
}
#[mem_builtin]
#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
memcmp(s1, s2, n)
}
}
// `bytes` must be a multiple of `mem::size_of::<T>()`
@ -133,65 +135,65 @@ where
intrinsics! {
#[cfg(target_has_atomic_load_store = "8")]
pub extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
memcpy_element_unordered_atomic(dest, src, bytes);
}
#[cfg(target_has_atomic_load_store = "16")]
pub extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
memcpy_element_unordered_atomic(dest, src, bytes);
}
#[cfg(target_has_atomic_load_store = "32")]
pub extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
memcpy_element_unordered_atomic(dest, src, bytes);
}
#[cfg(target_has_atomic_load_store = "64")]
pub extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
memcpy_element_unordered_atomic(dest, src, bytes);
}
#[cfg(target_has_atomic_load_store = "128")]
pub extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
memcpy_element_unordered_atomic(dest, src, bytes);
}
#[cfg(target_has_atomic_load_store = "8")]
pub extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
memmove_element_unordered_atomic(dest, src, bytes);
}
#[cfg(target_has_atomic_load_store = "16")]
pub extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
memmove_element_unordered_atomic(dest, src, bytes);
}
#[cfg(target_has_atomic_load_store = "32")]
pub extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
memmove_element_unordered_atomic(dest, src, bytes);
}
#[cfg(target_has_atomic_load_store = "64")]
pub extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
memmove_element_unordered_atomic(dest, src, bytes);
}
#[cfg(target_has_atomic_load_store = "128")]
pub extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
memmove_element_unordered_atomic(dest, src, bytes);
}
#[cfg(target_has_atomic_load_store = "8")]
pub extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
memset_element_unordered_atomic(s, c, bytes);
}
#[cfg(target_has_atomic_load_store = "16")]
pub extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () {
memset_element_unordered_atomic(s, c, bytes);
}
#[cfg(target_has_atomic_load_store = "32")]
pub extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () {
memset_element_unordered_atomic(s, c, bytes);
}
#[cfg(target_has_atomic_load_store = "64")]
pub extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
memset_element_unordered_atomic(s, c, bytes);
}
#[cfg(target_has_atomic_load_store = "128")]
pub extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
memset_element_unordered_atomic(s, c, bytes);
}
}

148
library/compiler-builtins/src/x86.rs
View file

@ -8,82 +8,78 @@ use core::intrinsics;
// NOTE These functions are never mangled as they are not tested against compiler-rt
// and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm"),
not(feature = "mangled-names")
))]
#[naked]
#[no_mangle]
pub unsafe extern "C" fn ___chkstk_ms() {
core::arch::asm!(
"push %ecx",
"push %eax",
"cmp $0x1000,%eax",
"lea 12(%esp),%ecx",
"jb 1f",
"2:",
"sub $0x1000,%ecx",
"test %ecx,(%ecx)",
"sub $0x1000,%eax",
"cmp $0x1000,%eax",
"ja 2b",
"1:",
"sub %eax,%ecx",
"test %ecx,(%ecx)",
"pop %eax",
"pop %ecx",
"ret",
options(noreturn, att_syntax)
);
}
intrinsics! {
#[naked]
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm")
))]
pub unsafe extern "C" fn ___chkstk_ms() {
core::arch::asm!(
"push %ecx",
"push %eax",
"cmp $0x1000,%eax",
"lea 12(%esp),%ecx",
"jb 1f",
"2:",
"sub $0x1000,%ecx",
"test %ecx,(%ecx)",
"sub $0x1000,%eax",
"cmp $0x1000,%eax",
"ja 2b",
"1:",
"sub %eax,%ecx",
"test %ecx,(%ecx)",
"pop %eax",
"pop %ecx",
"ret",
options(noreturn, att_syntax)
);
}
// FIXME: __alloca should be an alias to __chkstk
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm"),
not(feature = "mangled-names")
))]
#[naked]
#[no_mangle]
pub unsafe extern "C" fn __alloca() {
core::arch::asm!(
"jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
options(noreturn, att_syntax)
);
}
// FIXME: __alloca should be an alias to __chkstk
#[naked]
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm")
))]
pub unsafe extern "C" fn __alloca() {
core::arch::asm!(
"jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
options(noreturn, att_syntax)
);
}
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm"),
not(feature = "mangled-names")
))]
#[naked]
#[no_mangle]
pub unsafe extern "C" fn ___chkstk() {
core::arch::asm!(
"push %ecx",
"cmp $0x1000,%eax",
"lea 8(%esp),%ecx", // esp before calling this routine -> ecx
"jb 1f",
"2:",
"sub $0x1000,%ecx",
"test %ecx,(%ecx)",
"sub $0x1000,%eax",
"cmp $0x1000,%eax",
"ja 2b",
"1:",
"sub %eax,%ecx",
"test %ecx,(%ecx)",
"lea 4(%esp),%eax", // load pointer to the return address into eax
"mov %ecx,%esp", // install the new top of stack pointer into esp
"mov -4(%eax),%ecx", // restore ecx
"push (%eax)", // push return address onto the stack
"sub %esp,%eax", // restore the original value in eax
"ret",
options(noreturn, att_syntax)
);
#[naked]
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm")
))]
pub unsafe extern "C" fn ___chkstk() {
core::arch::asm!(
"push %ecx",
"cmp $0x1000,%eax",
"lea 8(%esp),%ecx", // esp before calling this routine -> ecx
"jb 1f",
"2:",
"sub $0x1000,%ecx",
"test %ecx,(%ecx)",
"sub $0x1000,%eax",
"cmp $0x1000,%eax",
"ja 2b",
"1:",
"sub %eax,%ecx",
"test %ecx,(%ecx)",
"lea 4(%esp),%eax", // load pointer to the return address into eax
"mov %ecx,%esp", // install the new top of stack pointer into esp
"mov -4(%eax),%ecx", // restore ecx
"push (%eax)", // push return address onto the stack
"sub %esp,%eax", // restore the original value in eax
"ret",
options(noreturn, att_syntax)
);
}
}

158
library/compiler-builtins/src/x86_64.rs
View file

@ -8,89 +8,87 @@ use core::intrinsics;
// NOTE These functions are never mangled as they are not tested against compiler-rt
// and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm"),
not(feature = "mangled-names")
))]
#[naked]
#[no_mangle]
pub unsafe extern "C" fn ___chkstk_ms() {
core::arch::asm!(
"push %rcx",
"push %rax",
"cmp $0x1000,%rax",
"lea 24(%rsp),%rcx",
"jb 1f",
"2:",
"sub $0x1000,%rcx",
"test %rcx,(%rcx)",
"sub $0x1000,%rax",
"cmp $0x1000,%rax",
"ja 2b",
"1:",
"sub %rax,%rcx",
"test %rcx,(%rcx)",
"pop %rax",
"pop %rcx",
"ret",
options(noreturn, att_syntax)
);
}
intrinsics! {
#[naked]
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm")
))]
pub unsafe extern "C" fn ___chkstk_ms() {
core::arch::asm!(
"push %rcx",
"push %rax",
"cmp $0x1000,%rax",
"lea 24(%rsp),%rcx",
"jb 1f",
"2:",
"sub $0x1000,%rcx",
"test %rcx,(%rcx)",
"sub $0x1000,%rax",
"cmp $0x1000,%rax",
"ja 2b",
"1:",
"sub %rax,%rcx",
"test %rcx,(%rcx)",
"pop %rax",
"pop %rcx",
"ret",
options(noreturn, att_syntax)
);
}
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm"),
not(feature = "mangled-names")
))]
#[naked]
#[no_mangle]
pub unsafe extern "C" fn __alloca() {
core::arch::asm!(
"mov %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx
"jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
options(noreturn, att_syntax)
);
}
#[naked]
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm")
))]
pub unsafe extern "C" fn __alloca() {
core::arch::asm!(
"mov %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx
"jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
options(noreturn, att_syntax)
);
}
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm"),
not(feature = "mangled-names")
))]
#[naked]
#[no_mangle]
pub unsafe extern "C" fn ___chkstk() {
core::arch::asm!(
"push %rcx",
"cmp $0x1000,%rax",
"lea 16(%rsp),%rcx", // rsp before calling this routine -> rcx
"jb 1f",
"2:",
"sub $0x1000,%rcx",
"test %rcx,(%rcx)",
"sub $0x1000,%rax",
"cmp $0x1000,%rax",
"ja 2b",
"1:",
"sub %rax,%rcx",
"test %rcx,(%rcx)",
"lea 8(%rsp),%rax", // load pointer to the return address into rax
"mov %rcx,%rsp", // install the new top of stack pointer into rsp
"mov -8(%rax),%rcx", // restore rcx
"push (%rax)", // push return address onto the stack
"sub %rsp,%rax", // restore the original value in rax
"ret",
options(noreturn, att_syntax)
);
#[naked]
#[cfg(all(
windows,
target_env = "gnu",
not(feature = "no-asm")
))]
pub unsafe extern "C" fn ___chkstk() {
core::arch::asm!(
"push %rcx",
"cmp $0x1000,%rax",
"lea 16(%rsp),%rcx", // rsp before calling this routine -> rcx
"jb 1f",
"2:",
"sub $0x1000,%rcx",
"test %rcx,(%rcx)",
"sub $0x1000,%rax",
"cmp $0x1000,%rax",
"ja 2b",
"1:",
"sub %rax,%rcx",
"test %rcx,(%rcx)",
"lea 8(%rsp),%rax", // load pointer to the return address into rax
"mov %rcx,%rsp", // install the new top of stack pointer into rsp
"mov -8(%rax),%rcx", // restore rcx
"push (%rax)", // push return address onto the stack
"sub %rsp,%rax", // restore the original value in rax
"ret",
options(noreturn, att_syntax)
);
}
}
// HACK(https://github.com/rust-lang/rust/issues/62785): x86_64-unknown-uefi needs special LLVM
// support unless we emit the _fltused
#[no_mangle]
#[used]
#[cfg(target_os = "uefi")]
static _fltused: i32 = 0;
mod _fltused {
#[no_mangle]
#[used]
#[cfg(target_os = "uefi")]
static _fltused: i32 = 0;
}