diff --git a/src/override/malloc.cc b/src/override/malloc.cc index 2bdd552..37c2d9f 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -4,18 +4,62 @@ using namespace snmalloc; +#ifndef SNMALLOC_EXPORT +# define SNMALLOC_EXPORT +#endif + #ifndef SNMALLOC_NAME_MANGLE # define SNMALLOC_NAME_MANGLE(a) a #endif +namespace +{ + /** + * RAII wrapper around an `Alloc`. This class gets an allocator from the + * global pool and wraps it so that `Alloc` methods can be called + * directly via the `->` operator on this class. When this object is + * destroyed, it returns the allocator to the global pool. + */ + struct slow_allocator + { + /** + * The allocator that this wrapper will use. + */ + Alloc* alloc; + /** + * Constructor. Claims an allocator from the global pool + */ + slow_allocator() : alloc(current_alloc_pool()->acquire()) {} + /** + * Destructor. Returns the allocator to the pool. + */ + ~slow_allocator() + { + current_alloc_pool()->release(alloc); + } + /** + * Arrow operator, allows methods exposed by `Alloc` to be called on the + * wrapper. + */ + Alloc* operator->() + { + return alloc; + } + }; + slow_allocator bootstrap_alloc() + { + return slow_allocator{}; + } +} + extern "C" { - void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr) { return Alloc::external_pointer(ptr); } - void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) { // Include size 0 in the first sizeclass. size = ((size - 1) >> (bits::BITS - 1)) + size; @@ -23,7 +67,7 @@ extern "C" return ThreadAlloc::get()->alloc(size); } - void SNMALLOC_NAME_MANGLE(free)(void* ptr) + SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free)(void* ptr) { if (ptr == nullptr) return; @@ -31,7 +75,7 @@ extern "C" ThreadAlloc::get()->dealloc(ptr); } - void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -45,12 +89,12 @@ extern "C" return ThreadAlloc::get()->alloc(sz); } - size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)(void* ptr) + SNMALLOC_EXPORT size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)(void* ptr) { return Alloc::alloc_size(ptr); } - void* SNMALLOC_NAME_MANGLE(realloc)(void* ptr, size_t size) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(realloc)(void* ptr, size_t size) { if (size == (size_t)-1) { @@ -88,7 +132,8 @@ extern "C" } #ifndef __FreeBSD__ - void* SNMALLOC_NAME_MANGLE(reallocarray)(void* ptr, size_t nmemb, size_t size) + SNMALLOC_EXPORT void* + SNMALLOC_NAME_MANGLE(reallocarray)(void* ptr, size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -101,14 +146,16 @@ extern "C" } #endif - void* SNMALLOC_NAME_MANGLE(aligned_alloc)(size_t alignment, size_t size) + SNMALLOC_EXPORT void* + SNMALLOC_NAME_MANGLE(aligned_alloc)(size_t alignment, size_t size) { assert((size % alignment) == 0); (void)alignment; return SNMALLOC_NAME_MANGLE(malloc)(size); } - void* SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size) + SNMALLOC_EXPORT void* + SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size) { if ( (alignment == 0) || (alignment == size_t(-1)) || @@ -141,7 +188,7 @@ extern "C" return nullptr; } - int SNMALLOC_NAME_MANGLE(posix_memalign)( + SNMALLOC_EXPORT int SNMALLOC_NAME_MANGLE(posix_memalign)( void** memptr, size_t alignment, size_t size) { if ( @@ -161,13 +208,13 @@ extern "C" } #ifndef __FreeBSD__ - void* SNMALLOC_NAME_MANGLE(valloc)(size_t size) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(valloc)(size_t size) { return SNMALLOC_NAME_MANGLE(memalign)(OS_PAGE_SIZE, size); } #endif - void* SNMALLOC_NAME_MANGLE(pvalloc)(size_t size) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(pvalloc)(size_t size) { if (size == size_t(-1)) { @@ -178,11 +225,46 @@ extern "C" OS_PAGE_SIZE, (size + OS_PAGE_SIZE - 1) & ~(OS_PAGE_SIZE - 1)); } - void SNMALLOC_NAME_MANGLE(_malloc_prefork)(void) {} - void SNMALLOC_NAME_MANGLE(_malloc_postfork)(void) {} - void SNMALLOC_NAME_MANGLE(_malloc_first_thread)(void) {} - int SNMALLOC_NAME_MANGLE(mallctl)(const char*, void*, size_t*, void*, size_t) + // Stub implementations for jemalloc compatibility. + // These are called by FreeBSD's libthr (pthreads) to notify malloc of + // various events. They are currently unused, though we may wish to reset + // statistics on fork if built with statistics. + + SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_prefork)(void) {} + SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_postfork)(void) {} + SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_first_thread)(void) {} + + SNMALLOC_EXPORT int + SNMALLOC_NAME_MANGLE(mallctl)(const char*, void*, size_t*, void*, size_t) { return ENOENT; } + +#ifndef __PIC__ + // The following functions are required to work before TLS is set up, in + // statically-linked programs. These temporarily grab an allocator from the + // pool and return it. + + void* __je_bootstrap_malloc(size_t size) + { + return bootstrap_alloc()->alloc(size); + } + void* __je_bootstrap_calloc(size_t nmemb, size_t size) + { + bool overflow = false; + size_t sz = bits::umul(size, nmemb, overflow); + if (overflow) + { + errno = ENOMEM; + return 0; + } + // Include size 0 in the first sizeclass. + sz = ((sz - 1) >> (bits::BITS - 1)) + sz; + return bootstrap_alloc()->alloc(sz); + } + void __je_bootstrap_free(void* ptr) + { + bootstrap_alloc()->dealloc(ptr); + } +#endif }