user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Less syscalls for the copy_file_range probe

Browse source 
If it's obvious from the actual syscall results themselves that the
syscall is supported or unsupported, don't do an extra syscall with an
invalid file descriptor.

CC #122052
This commit is contained in:
Tobias Bucher 2024-03-06 12:42:32 +01:00
parent 80399064af
commit 214c49837a
1 changed files with 48 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

71
library/std/src/sys/pal/unix/kernel_copy.rs
View file

@ -560,6 +560,12 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
// We store the availability in a global to avoid unnecessary syscalls
static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED);
let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
NOT_PROBED => false,
UNAVAILABLE => return CopyResult::Fallback(0),
_ => true,
};
syscall! {
fn copy_file_range(
fd_in: libc::c_int,
@ -571,26 +577,6 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
) -> libc::ssize_t
}
match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
NOT_PROBED => {
// EPERM can indicate seccomp filters or an immutable file.
// To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported
// and some other error (ENOSYS or EPERM) if it's not available
let result = unsafe {
cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
};
if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) {
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
} else {
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
return CopyResult::Fallback(0);
}
}
UNAVAILABLE => return CopyResult::Fallback(0),
_ => {}
};
let mut written = 0u64;
while written < max_len {
let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64);
@ -604,6 +590,11 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0))
};
if !have_probed && copy_result.is_ok() {
have_probed = true;
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
}
match copy_result {
Ok(0) if written == 0 => {
// fallback to work around several kernel bugs where copy_file_range will fail to
@ -616,10 +607,44 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
Ok(0) => return CopyResult::Ended(written), // reached EOF
Ok(ret) => written += ret as u64,
Err(err) => {
return match err.raw_os_error() {
let raw_os_error = match err.raw_os_error() {
Some(raw) => raw,
_ => return CopyResult::Error(err, written),
};
return match raw_os_error {
// when file offset + max_length > u64::MAX
Some(EOVERFLOW) => CopyResult::Fallback(written),
Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => {
EOVERFLOW => CopyResult::Fallback(written),
ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF if written == 0 => {
if !have_probed {
if raw_os_error == ENOSYS {
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
} else {
// EPERM can indicate seccomp filters or an
// immutable file. To distinguish these cases
// we probe with invalid file descriptors which
// should result in EBADF if the syscall is
// supported and some other error (ENOSYS or
// EPERM) if it's not available.
let result = unsafe {
cvt(copy_file_range(
INVALID_FD,
ptr::null_mut(),
INVALID_FD,
ptr::null_mut(),
1,
0,
))
};
if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF)))
{
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
} else {
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
}
}
}
// Try fallback io::copy if either:
// - Kernel version is < 4.5 (ENOSYS¹)
// - Files are mounted on different fs (EXDEV)