user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

micro-optimize dynamic allocation alignment

Browse source 
Previously, some parts of this optimization were impossible because the
alignment passed to the free function was not correct. That was fully
fixed by #17012.

Closes #17092
This commit is contained in:
Daniel Micay 2014-09-08 01:46:33 -04:00
parent b625d43f8f
commit 92b09261e7
2 changed files with 65 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/jemalloc

@ -1 +1 @@
Subproject commit 024c67ad651e1a3ca228936c4cfb13a37329baf2
Subproject commit aae04170ccbfeea620502106b581c3c216cd132a

98
src/liballoc/heap.rs
View file

@ -149,12 +149,24 @@ unsafe fn closure_exchange_malloc(drop_glue: fn(*mut u8), size: uint,
alloc as *mut u8
}
// The minimum alignment guaranteed by the architecture. This value is used to
// add fast paths for low alignment values. In practice, the alignment is a
// constant at the call site and the branch will be optimized out.
#[cfg(target_arch = "arm")]
#[cfg(target_arch = "mips")]
#[cfg(target_arch = "mipsel")]
static MIN_ALIGN: uint = 8;
#[cfg(target_arch = "x86")]
#[cfg(target_arch = "x86_64")]
static MIN_ALIGN: uint = 16;
#[cfg(jemalloc)]
mod imp {
use core::option::{None, Option};
use core::ptr::{RawPtr, mut_null, null};
use core::num::Int;
use libc::{c_char, c_int, c_void, size_t};
use super::MIN_ALIGN;
#[link(name = "jemalloc", kind = "static")]
#[cfg(not(test))]
@ -183,9 +195,15 @@ mod imp {
#[inline(always)]
fn mallocx_align(a: uint) -> c_int { a.trailing_zeros() as c_int }
#[inline(always)]
fn align_to_flags(align: uint) -> c_int {
if align <= MIN_ALIGN { 0 } else { mallocx_align(align) }
}
#[inline]
pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
let ptr = je_mallocx(size as size_t, mallocx_align(align)) as *mut u8;
let flags = align_to_flags(align);
let ptr = je_mallocx(size as size_t, flags) as *mut u8;
if ptr.is_null() {
::oom()
}
@ -195,8 +213,8 @@ mod imp {
#[inline]
pub unsafe fn reallocate(ptr: *mut u8, size: uint, align: uint,
_old_size: uint) -> *mut u8 {
let ptr = je_rallocx(ptr as *mut c_void, size as size_t,
mallocx_align(align)) as *mut u8;
let flags = align_to_flags(align);
let ptr = je_rallocx(ptr as *mut c_void, size as size_t, flags) as *mut u8;
if ptr.is_null() {
::oom()
}
@ -206,18 +224,20 @@ mod imp {
#[inline]
pub unsafe fn reallocate_inplace(ptr: *mut u8, size: uint, align: uint,
_old_size: uint) -> bool {
je_xallocx(ptr as *mut c_void, size as size_t, 0,
mallocx_align(align)) == size as size_t
let flags = align_to_flags(align);
je_xallocx(ptr as *mut c_void, size as size_t, 0, flags) == size as size_t
}
#[inline]
pub unsafe fn deallocate(ptr: *mut u8, _size: uint, align: uint) {
je_dallocx(ptr as *mut c_void, mallocx_align(align))
let flags = align_to_flags(align);
je_dallocx(ptr as *mut c_void, flags)
}
#[inline]
pub fn usable_size(size: uint, align: uint) -> uint {
unsafe { je_nallocx(size as size_t, mallocx_align(align)) as uint }
let flags = align_to_flags(align);
unsafe { je_nallocx(size as size_t, flags) as uint }
}
pub fn stats_print() {
@ -234,6 +254,7 @@ mod imp {
use core::ptr;
use libc;
use libc_heap;
use super::MIN_ALIGN;
extern {
fn posix_memalign(memptr: *mut *mut libc::c_void,
@ -243,16 +264,7 @@ mod imp {
#[inline]
pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
// The posix_memalign manpage states
//
// alignment [...] must be a power of and a multiple of
// sizeof(void *)
//
// The `align` parameter to this function is the *minimum* alignment for
// a block of memory, so we special case everything under `*uint` to
// just pass it to malloc, which is guaranteed to align to at least the
// size of `*uint`.
if align < mem::size_of::<uint>() {
if align <= MIN_ALIGN {
libc_heap::malloc_raw(size)
} else {
let mut out = 0 as *mut libc::c_void;
@ -269,10 +281,14 @@ mod imp {
#[inline]
pub unsafe fn reallocate(ptr: *mut u8, size: uint, align: uint,
old_size: uint) -> *mut u8 {
let new_ptr = allocate(size, align);
ptr::copy_memory(new_ptr, ptr as *const u8, cmp::min(size, old_size));
deallocate(ptr, old_size, align);
return new_ptr;
if align <= MIN_ALIGN {
libc_heap::realloc_raw(ptr, size)
} else {
let new_ptr = allocate(size, align);
ptr::copy_memory(new_ptr, ptr as *const u8, cmp::min(size, old_size));
deallocate(ptr, old_size, align);
new_ptr
}
}
#[inline]
@ -291,14 +307,16 @@ mod imp {
size
}
pub fn stats_print() {
}
pub fn stats_print() {}
}
#[cfg(not(jemalloc), windows)]
mod imp {
use libc::{c_void, size_t};
use libc;
use libc_heap;
use core::ptr::RawPtr;
use super::MIN_ALIGN;
extern {
fn _aligned_malloc(size: size_t, align: size_t) -> *mut c_void;
@ -309,22 +327,30 @@ mod imp {
#[inline]
pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
let ptr = _aligned_malloc(size as size_t, align as size_t);
if ptr.is_null() {
::oom();
if align <= MIN_ALIGN {
libc_heap::malloc_raw(size)
} else {
let ptr = _aligned_malloc(size as size_t, align as size_t);
if ptr.is_null() {
::oom();
}
ptr as *mut u8
}
ptr as *mut u8
}
#[inline]
pub unsafe fn reallocate(ptr: *mut u8, size: uint, align: uint,
_old_size: uint) -> *mut u8 {
let ptr = _aligned_realloc(ptr as *mut c_void, size as size_t,
align as size_t);
if ptr.is_null() {
::oom();
if align <= MIN_ALIGN {
libc_heap::realloc_raw(ptr, size)
} else {
let ptr = _aligned_realloc(ptr as *mut c_void, size as size_t,
align as size_t);
if ptr.is_null() {
::oom();
}
ptr as *mut u8
}
ptr as *mut u8
}
#[inline]
@ -334,8 +360,12 @@ mod imp {
}
#[inline]
pub unsafe fn deallocate(ptr: *mut u8, _size: uint, _align: uint) {
_aligned_free(ptr as *mut c_void)
pub unsafe fn deallocate(ptr: *mut u8, _size: uint, align: uint) {
if align <= MIN_ALIGN {
libc::free(ptr as *mut libc::c_void)
} else {
_aligned_free(ptr as *mut c_void)
}
}
#[inline]