user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #713 from saethlin/permissive-memcpy

Browse source 
Use wrapping pointer arithmetic in mem/impls.rs
This commit is contained in:
Trevor Gross 2024-10-17 18:55:02 -05:00 • committed by GitHub
commit 26ca96a2f9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 61 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

105
library/compiler-builtins/src/mem/impls.rs
View file

@ -1,3 +1,20 @@
// In C and Rust it is UB to read or write to usize::MAX because if an allocation extends to the
// last byte of address space (there must be an allocation to do the read or write), in C computing
// its one-past-the-end pointer would be equal to NULL and in Rust computing the address of a
// trailing ZST member with a safe place projection would wrap (place projection address computation
// is non-wrapping).
//
// However, some embedded systems have special memory at usize::MAX, and need to access that
// memory. If they do that with the intrinsics provided by compiler-builtins (such as memcpy!), the
// ptr::add in these loops will wrap. And if compiler-builtins is compiled with cfg(ub_checks),
// this will fail a UB check at runtime.
//
// Since this scenario is UB, we are within our rights hit this check and halt execution...
// But we are also within our rights to try to make it work.
// We use wrapping_add/wrapping_sub for pointer arithmetic in this module in an attempt to support
// this use. Of course this is not a guarantee that such use will work, it just means that this
// crate doing wrapping pointer arithmetic with a method that must not wrap won't be the problem if
// something does go wrong at runtime.
use core::intrinsics::likely;
const WORD_SIZE: usize = core::mem::size_of::<usize>();
@ -9,7 +26,7 @@ const WORD_MASK: usize = WORD_SIZE - 1;
// word-wise copy.
// * The word-wise copy logic needs to perform some checks so it has some small overhead.
// ensures that even on 32-bit platforms we have copied at least 8 bytes through
// word-wise copy so the saving of word-wise copy outweights the fixed overhead.
// word-wise copy so the saving of word-wise copy outweighs the fixed overhead.
const WORD_COPY_THRESHOLD: usize = if 2 * WORD_SIZE > 16 {
2 * WORD_SIZE
} else {
@ -28,11 +45,11 @@ unsafe fn read_usize_unaligned(x: *const usize) -> usize {
pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) {
#[inline(always)]
unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) {
let dest_end = dest.add(n);
let dest_end = dest.wrapping_add(n);
while dest < dest_end {
*dest = *src;
dest = dest.add(1);
src = src.add(1);
dest = dest.wrapping_add(1);
src = src.wrapping_add(1);
}
}
@ -40,12 +57,12 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
unsafe fn copy_forward_aligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let mut src_usize = src as *mut usize;
let dest_end = dest.add(n) as *mut usize;
let dest_end = dest.wrapping_add(n) as *mut usize;
while dest_usize < dest_end {
*dest_usize = *src_usize;
dest_usize = dest_usize.add(1);
src_usize = src_usize.add(1);
dest_usize = dest_usize.wrapping_add(1);
src_usize = src_usize.wrapping_add(1);
}
}
@ -53,7 +70,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
#[inline(always)]
unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let dest_end = dest.add(n) as *mut usize;
let dest_end = dest.wrapping_add(n) as *mut usize;
// Calculate the misalignment offset and shift needed to reassemble value.
let offset = src as usize & WORD_MASK;
@ -70,7 +87,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
let mut prev_word = core::ptr::read_volatile(src_aligned);
while dest_usize < dest_end {
src_aligned = src_aligned.add(1);
src_aligned = src_aligned.wrapping_add(1);
let cur_word = *src_aligned;
#[cfg(target_endian = "little")]
let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift);
@ -79,7 +96,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
prev_word = cur_word;
*dest_usize = resembled;
dest_usize = dest_usize.add(1);
dest_usize = dest_usize.wrapping_add(1);
}
}
@ -88,12 +105,12 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let mut src_usize = src as *mut usize;
let dest_end = dest.add(n) as *mut usize;
let dest_end = dest.wrapping_add(n) as *mut usize;
while dest_usize < dest_end {
*dest_usize = read_usize_unaligned(src_usize);
dest_usize = dest_usize.add(1);
src_usize = src_usize.add(1);
dest_usize = dest_usize.wrapping_add(1);
src_usize = src_usize.wrapping_add(1);
}
}
@ -102,8 +119,8 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
// Because of n >= 2 * WORD_SIZE, dst_misalignment < n
let dest_misalignment = (dest as usize).wrapping_neg() & WORD_MASK;
copy_forward_bytes(dest, src, dest_misalignment);
dest = dest.add(dest_misalignment);
src = src.add(dest_misalignment);
dest = dest.wrapping_add(dest_misalignment);
src = src.wrapping_add(dest_misalignment);
n -= dest_misalignment;
let n_words = n & !WORD_MASK;
@ -113,8 +130,8 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
} else {
copy_forward_misaligned_words(dest, src, n_words);
}
dest = dest.add(n_words);
src = src.add(n_words);
dest = dest.wrapping_add(n_words);
src = src.wrapping_add(n_words);
n -= n_words;
}
copy_forward_bytes(dest, src, n);
@ -126,10 +143,10 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
// as their inputs instead of pointers to the start!
#[inline(always)]
unsafe fn copy_backward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) {
let dest_start = dest.sub(n);
let dest_start = dest.wrapping_sub(n);
while dest_start < dest {
dest = dest.sub(1);
src = src.sub(1);
dest = dest.wrapping_sub(1);
src = src.wrapping_sub(1);
*dest = *src;
}
}
@ -138,11 +155,11 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
unsafe fn copy_backward_aligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let mut src_usize = src as *mut usize;
let dest_start = dest.sub(n) as *mut usize;
let dest_start = dest.wrapping_sub(n) as *mut usize;
while dest_start < dest_usize {
dest_usize = dest_usize.sub(1);
src_usize = src_usize.sub(1);
dest_usize = dest_usize.wrapping_sub(1);
src_usize = src_usize.wrapping_sub(1);
*dest_usize = *src_usize;
}
}
@ -151,7 +168,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
#[inline(always)]
unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let dest_start = dest.sub(n) as *mut usize;
let dest_start = dest.wrapping_sub(n) as *mut usize;
// Calculate the misalignment offset and shift needed to reassemble value.
let offset = src as usize & WORD_MASK;
@ -168,7 +185,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
let mut prev_word = core::ptr::read_volatile(src_aligned);
while dest_start < dest_usize {
src_aligned = src_aligned.sub(1);
src_aligned = src_aligned.wrapping_sub(1);
let cur_word = *src_aligned;
#[cfg(target_endian = "little")]
let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift;
@ -176,7 +193,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift;
prev_word = cur_word;
dest_usize = dest_usize.sub(1);
dest_usize = dest_usize.wrapping_sub(1);
*dest_usize = resembled;
}
}
@ -186,25 +203,25 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let mut src_usize = src as *mut usize;
let dest_start = dest.sub(n) as *mut usize;
let dest_start = dest.wrapping_sub(n) as *mut usize;
while dest_start < dest_usize {
dest_usize = dest_usize.sub(1);
src_usize = src_usize.sub(1);
dest_usize = dest_usize.wrapping_sub(1);
src_usize = src_usize.wrapping_sub(1);
*dest_usize = read_usize_unaligned(src_usize);
}
}
let mut dest = dest.add(n);
let mut src = src.add(n);
let mut dest = dest.wrapping_add(n);
let mut src = src.wrapping_add(n);
if n >= WORD_COPY_THRESHOLD {
// Align dest
// Because of n >= 2 * WORD_SIZE, dst_misalignment < n
let dest_misalignment = dest as usize & WORD_MASK;
copy_backward_bytes(dest, src, dest_misalignment);
dest = dest.sub(dest_misalignment);
src = src.sub(dest_misalignment);
dest = dest.wrapping_sub(dest_misalignment);
src = src.wrapping_sub(dest_misalignment);
n -= dest_misalignment;
let n_words = n & !WORD_MASK;
@ -214,8 +231,8 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
} else {
copy_backward_misaligned_words(dest, src, n_words);
}
dest = dest.sub(n_words);
src = src.sub(n_words);
dest = dest.wrapping_sub(n_words);
src = src.wrapping_sub(n_words);
n -= n_words;
}
copy_backward_bytes(dest, src, n);
@ -225,10 +242,10 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
#[inline(always)]
pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) {
let end = s.add(n);
let end = s.wrapping_add(n);
while s < end {
*s = c;
s = s.add(1);
s = s.wrapping_add(1);
}
}
@ -242,11 +259,11 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
}
let mut s_usize = s as *mut usize;
let end = s.add(n) as *mut usize;
let end = s.wrapping_add(n) as *mut usize;
while s_usize < end {
*s_usize = broadcast;
s_usize = s_usize.add(1);
s_usize = s_usize.wrapping_add(1);
}
}
@ -255,12 +272,12 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
// Because of n >= 2 * WORD_SIZE, dst_misalignment < n
let misalignment = (s as usize).wrapping_neg() & WORD_MASK;
set_bytes_bytes(s, c, misalignment);
s = s.add(misalignment);
s = s.wrapping_add(misalignment);
n -= misalignment;
let n_words = n & !WORD_MASK;
set_bytes_words(s, c, n_words);
s = s.add(n_words);
s = s.wrapping_add(n_words);
n -= n_words;
}
set_bytes_bytes(s, c, n);
@ -270,8 +287,8 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
pub unsafe fn compare_bytes(s1: *const u8, s2: *const u8, n: usize) -> i32 {
let mut i = 0;
while i < n {
let a = *s1.add(i);
let b = *s2.add(i);
let a = *s1.wrapping_add(i);
let b = *s2.wrapping_add(i);
if a != b {
return a as i32 - b as i32;
}
@ -285,7 +302,7 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize {
let mut n = 0;
while *s != 0 {
n += 1;
s = s.add(1);
s = s.wrapping_add(1);
}
n
}