695dee063b
This PR is an implementation of [RFC 1974] which specifies a new method of defining a global allocator for a program. This obsoletes the old `#![allocator]` attribute and also removes support for it. [RFC 1974]: https://github.com/rust-lang/rfcs/pull/197 The new `#[global_allocator]` attribute solves many issues encountered with the `#![allocator]` attribute such as composition and restrictions on the crate graph itself. The compiler now has much more control over the ABI of the allocator and how it's implemented, allowing much more freedom in terms of how this feature is implemented. cc #27389
255 lines
9.9 KiB
Rust
255 lines
9.9 KiB
Rust
// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
|
|
// file at the top-level directory of this distribution and at
|
|
// http://rust-lang.org/COPYRIGHT.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
|
// option. This file may not be copied, modified, or distributed
|
|
// except according to those terms.
|
|
|
|
#![crate_name = "alloc_jemalloc"]
|
|
#![crate_type = "rlib"]
|
|
#![no_std]
|
|
#![unstable(feature = "alloc_jemalloc",
|
|
reason = "this library is unlikely to be stabilized in its current \
|
|
form or name",
|
|
issue = "27783")]
|
|
#![deny(warnings)]
|
|
#![feature(libc)]
|
|
#![feature(staged_api)]
|
|
#![feature(linkage)]
|
|
#![cfg_attr(stage0, allocator)]
|
|
#![cfg_attr(stage0, feature(allocator))]
|
|
#![cfg_attr(not(stage0), feature(global_allocator))]
|
|
#![cfg_attr(all(not(stage0), not(dummy_jemalloc)), feature(allocator_api))]
|
|
#![cfg_attr(not(stage0), feature(alloc))]
|
|
#![cfg_attr(not(stage0), feature(alloc_system))]
|
|
#![cfg_attr(dummy_jemalloc, allow(dead_code))]
|
|
|
|
#[cfg(not(stage0))]
|
|
extern crate alloc;
|
|
#[cfg(not(stage0))]
|
|
extern crate alloc_system;
|
|
extern crate libc;
|
|
|
|
#[cfg(all(not(stage0), not(dummy_jemalloc)))]
|
|
pub use contents::*;
|
|
#[cfg(all(not(stage0), not(dummy_jemalloc)))]
|
|
mod contents {
|
|
use core::ptr;
|
|
|
|
use alloc::heap::{Alloc, AllocErr, Layout};
|
|
use alloc_system::System;
|
|
use libc::{c_int, c_void, size_t};
|
|
|
|
// Note that the symbols here are prefixed by default on macOS and Windows (we
|
|
// don't explicitly request it), and on Android and DragonFly we explicitly
|
|
// request it as unprefixing cause segfaults (mismatches in allocators).
|
|
extern "C" {
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_mallocx")]
|
|
fn mallocx(size: size_t, flags: c_int) -> *mut c_void;
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_calloc")]
|
|
fn calloc(size: size_t, flags: c_int) -> *mut c_void;
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_rallocx")]
|
|
fn rallocx(ptr: *mut c_void, size: size_t, flags: c_int) -> *mut c_void;
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_xallocx")]
|
|
fn xallocx(ptr: *mut c_void, size: size_t, extra: size_t, flags: c_int) -> size_t;
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_sdallocx")]
|
|
fn sdallocx(ptr: *mut c_void, size: size_t, flags: c_int);
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_nallocx")]
|
|
fn nallocx(size: size_t, flags: c_int) -> size_t;
|
|
}
|
|
|
|
const MALLOCX_ZERO: c_int = 0x40;
|
|
|
|
// The minimum alignment guaranteed by the architecture. This value is used to
|
|
// add fast paths for low alignment values. In practice, the alignment is a
|
|
// constant at the call site and the branch will be optimized out.
|
|
#[cfg(all(any(target_arch = "arm",
|
|
target_arch = "mips",
|
|
target_arch = "powerpc")))]
|
|
const MIN_ALIGN: usize = 8;
|
|
#[cfg(all(any(target_arch = "x86",
|
|
target_arch = "x86_64",
|
|
target_arch = "aarch64",
|
|
target_arch = "powerpc64",
|
|
target_arch = "mips64",
|
|
target_arch = "s390x",
|
|
target_arch = "sparc64")))]
|
|
const MIN_ALIGN: usize = 16;
|
|
|
|
// MALLOCX_ALIGN(a) macro
|
|
fn mallocx_align(a: usize) -> c_int {
|
|
a.trailing_zeros() as c_int
|
|
}
|
|
|
|
fn align_to_flags(align: usize) -> c_int {
|
|
if align <= MIN_ALIGN {
|
|
0
|
|
} else {
|
|
mallocx_align(align)
|
|
}
|
|
}
|
|
|
|
// for symbol names src/librustc/middle/allocator.rs
|
|
// for signatures src/librustc_allocator/lib.rs
|
|
|
|
// linkage directives are provided as part of the current compiler allocator
|
|
// ABI
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_alloc(size: usize,
|
|
align: usize,
|
|
err: *mut u8) -> *mut u8 {
|
|
let flags = align_to_flags(align);
|
|
let ptr = mallocx(size as size_t, flags) as *mut u8;
|
|
if ptr.is_null() {
|
|
let layout = Layout::from_size_align_unchecked(size, align);
|
|
ptr::write(err as *mut AllocErr,
|
|
AllocErr::Exhausted { request: layout });
|
|
}
|
|
ptr
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_oom(err: *const u8) -> ! {
|
|
System.oom((*(err as *const AllocErr)).clone())
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_dealloc(ptr: *mut u8,
|
|
size: usize,
|
|
align: usize) {
|
|
let flags = align_to_flags(align);
|
|
sdallocx(ptr as *mut c_void, size, flags);
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_usable_size(layout: *const u8,
|
|
min: *mut usize,
|
|
max: *mut usize) {
|
|
let layout = &*(layout as *const Layout);
|
|
let flags = align_to_flags(layout.align());
|
|
let size = nallocx(layout.size(), flags) as usize;
|
|
*min = layout.size();
|
|
if size > 0 {
|
|
*max = size;
|
|
} else {
|
|
*max = layout.size();
|
|
}
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_realloc(ptr: *mut u8,
|
|
_old_size: usize,
|
|
old_align: usize,
|
|
new_size: usize,
|
|
new_align: usize,
|
|
err: *mut u8) -> *mut u8 {
|
|
if new_align != old_align {
|
|
ptr::write(err as *mut AllocErr,
|
|
AllocErr::Unsupported { details: "can't change alignments" });
|
|
return 0 as *mut u8
|
|
}
|
|
|
|
let flags = align_to_flags(new_align);
|
|
let ptr = rallocx(ptr as *mut c_void, new_size, flags) as *mut u8;
|
|
if ptr.is_null() {
|
|
let layout = Layout::from_size_align_unchecked(new_size, new_align);
|
|
ptr::write(err as *mut AllocErr,
|
|
AllocErr::Exhausted { request: layout });
|
|
}
|
|
ptr
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_alloc_zeroed(size: usize,
|
|
align: usize,
|
|
err: *mut u8) -> *mut u8 {
|
|
let ptr = if align <= MIN_ALIGN {
|
|
calloc(size as size_t, 1) as *mut u8
|
|
} else {
|
|
let flags = align_to_flags(align) | MALLOCX_ZERO;
|
|
mallocx(size as size_t, flags) as *mut u8
|
|
};
|
|
if ptr.is_null() {
|
|
let layout = Layout::from_size_align_unchecked(size, align);
|
|
ptr::write(err as *mut AllocErr,
|
|
AllocErr::Exhausted { request: layout });
|
|
}
|
|
ptr
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_alloc_excess(size: usize,
|
|
align: usize,
|
|
excess: *mut usize,
|
|
err: *mut u8) -> *mut u8 {
|
|
let p = __rde_alloc(size, align, err);
|
|
if !p.is_null() {
|
|
*excess = size;
|
|
}
|
|
return p
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_realloc_excess(ptr: *mut u8,
|
|
old_size: usize,
|
|
old_align: usize,
|
|
new_size: usize,
|
|
new_align: usize,
|
|
excess: *mut usize,
|
|
err: *mut u8) -> *mut u8 {
|
|
let p = __rde_realloc(ptr, old_size, old_align, new_size, new_align, err);
|
|
if !p.is_null() {
|
|
*excess = new_size;
|
|
}
|
|
return p
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_grow_in_place(ptr: *mut u8,
|
|
old_size: usize,
|
|
old_align: usize,
|
|
new_size: usize,
|
|
new_align: usize) -> u8 {
|
|
__rde_shrink_in_place(ptr, old_size, old_align, new_size, new_align)
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_shrink_in_place(ptr: *mut u8,
|
|
_old_size: usize,
|
|
old_align: usize,
|
|
new_size: usize,
|
|
new_align: usize) -> u8 {
|
|
if old_align == new_align {
|
|
let flags = align_to_flags(new_align);
|
|
(xallocx(ptr as *mut c_void, new_size, 0, flags) == new_size) as u8
|
|
} else {
|
|
0
|
|
}
|
|
}
|
|
}
|