diff --git a/src/snmalloc/ds_core/redblacktree.h b/src/snmalloc/ds_core/redblacktree.h index f69114b..8b981ab 100644 --- a/src/snmalloc/ds_core/redblacktree.h +++ b/src/snmalloc/ds_core/redblacktree.h @@ -68,7 +68,9 @@ namespace snmalloc template concept RBRep = // - RBRepTypes && RBRepMethods && + RBRepTypes // + && RBRepMethods // + && ConceptSame>; #endif diff --git a/src/snmalloc/mem/localalloc.h b/src/snmalloc/mem/localalloc.h index c85d30b..75be56e 100644 --- a/src/snmalloc/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -184,6 +184,13 @@ namespace snmalloc } return check_init([&](CoreAlloc* core_alloc) { + if (size > bits::one_at_bit(bits::BITS - 1)) + { + // Cannot allocate something that is more that half the size of the + // address space + errno = ENOMEM; + return capptr::Alloc{nullptr}; + } // Grab slab of correct size // Set remote as large allocator remote. auto [chunk, meta] = Config::Backend::alloc_chunk( @@ -712,7 +719,7 @@ namespace snmalloc auto pm_size = sizeclass_full_to_size(pm_sc); snmalloc_check_client( mitigations(sanity_checks), - sc == pm_sc, + (sc == pm_sc) || (p == nullptr), "Dealloc rounded size mismatch: {} != {}", rsize, pm_size); diff --git a/src/snmalloc/mem/sizeclasstable.h b/src/snmalloc/mem/sizeclasstable.h index dd2be97..2d13d9c 100644 --- a/src/snmalloc/mem/sizeclasstable.h +++ b/src/snmalloc/mem/sizeclasstable.h @@ -478,6 +478,12 @@ namespace snmalloc { if (size > sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1)) { + if (size > bits::one_at_bit(bits::BITS - 1)) + { + // This size is too large, no rounding should occur as will result in a + // failed allocation later. + return size; + } return bits::next_pow2(size); } // If realloc(ptr, 0) returns nullptr, some consumers treat this as a diff --git a/src/snmalloc/override/libc.h b/src/snmalloc/override/libc.h new file mode 100644 index 0000000..05a82ec --- /dev/null +++ b/src/snmalloc/override/libc.h @@ -0,0 +1,177 @@ +#pragma once + +#include "../global/global.h" + +#include +#include + +namespace snmalloc::libc +{ + SNMALLOC_SLOW_PATH inline void* set_error(int err = ENOMEM) + { + errno = err; + return nullptr; + } + + SNMALLOC_SLOW_PATH inline int set_error_and_return(int err = ENOMEM) + { + errno = err; + return err; + } + + inline void* __malloc_end_pointer(void* ptr) + { + return ThreadAlloc::get().external_pointer(ptr); + } + + SNMALLOC_FAST_PATH_INLINE void* malloc(size_t size) + { + return ThreadAlloc::get().alloc(size); + } + + SNMALLOC_FAST_PATH_INLINE void free(void* ptr) + { + ThreadAlloc::get().dealloc(ptr); + } + + SNMALLOC_FAST_PATH_INLINE void free_sized(void* ptr, size_t size) + { + ThreadAlloc::get().dealloc(ptr, size); + } + + SNMALLOC_FAST_PATH_INLINE void* calloc(size_t nmemb, size_t size) + { + bool overflow = false; + size_t sz = bits::umul(size, nmemb, overflow); + if (SNMALLOC_UNLIKELY(overflow)) + { + return set_error(); + } + return ThreadAlloc::get().alloc(sz); + } + + SNMALLOC_FAST_PATH_INLINE void* realloc(void* ptr, size_t size) + { + auto& a = ThreadAlloc::get(); + size_t sz = a.alloc_size(ptr); + // Keep the current allocation if the given size is in the same sizeclass. + if (sz == round_size(size)) + { +#ifdef SNMALLOC_PASS_THROUGH + // snmallocs alignment guarantees can be broken by realloc in pass-through + // this is not exercised, by existing clients, but is tested. + if (pointer_align_up(ptr, natural_alignment(size)) == ptr) + return ptr; +#else + return ptr; +#endif + } + + void* p = a.alloc(size); + if (SNMALLOC_LIKELY(p != nullptr)) + { + sz = bits::min(size, sz); + // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy + // otherwise. + if (SNMALLOC_UNLIKELY(sz != 0)) + ::memcpy(p, ptr, sz); + a.dealloc(ptr); + } + else if (SNMALLOC_LIKELY(size == 0)) + { + a.dealloc(ptr); + } + else + { + return set_error(); + } + return p; + } + + inline size_t malloc_usable_size(const void* ptr) + { + return ThreadAlloc::get().alloc_size(ptr); + } + + inline void* reallocarray(void* ptr, size_t nmemb, size_t size) + { + bool overflow = false; + size_t sz = bits::umul(size, nmemb, overflow); + if (SNMALLOC_UNLIKELY(overflow)) + { + return set_error(); + } + return realloc(ptr, sz); + } + + inline int reallocarr(void* ptr_, size_t nmemb, size_t size) + { + int err = errno; + auto& a = ThreadAlloc::get(); + bool overflow = false; + size_t sz = bits::umul(size, nmemb, overflow); + if (SNMALLOC_UNLIKELY(sz == 0)) + { + errno = err; + return 0; + } + if (SNMALLOC_UNLIKELY(overflow)) + { + return set_error_and_return(EOVERFLOW); + } + + void** ptr = reinterpret_cast(ptr_); + void* p = a.alloc(sz); + if (SNMALLOC_UNLIKELY(p == nullptr)) + { + return set_error_and_return(ENOMEM); + } + + sz = bits::min(sz, a.alloc_size(*ptr)); + + SNMALLOC_ASSUME(*ptr != nullptr || sz == 0); + // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy + // otherwise. + if (SNMALLOC_UNLIKELY(sz != 0)) + ::memcpy(p, *ptr, sz); + errno = err; + a.dealloc(*ptr); + *ptr = p; + return 0; + } + + inline void* memalign(size_t alignment, size_t size) + { + if (SNMALLOC_UNLIKELY( + alignment < sizeof(uintptr_t) || !bits::is_pow2(alignment))) + { + return set_error(EINVAL); + } + + return malloc(aligned_size(alignment, size)); + } + + inline void* aligned_alloc(size_t alignment, size_t size) + { + SNMALLOC_ASSERT((size % alignment) == 0); + return memalign(alignment, size); + } + + inline int posix_memalign(void** memptr, size_t alignment, size_t size) + { + if (SNMALLOC_UNLIKELY( + (alignment < sizeof(uintptr_t) || !bits::is_pow2(alignment)))) + { + return EINVAL; + } + + void* p = memalign(alignment, size); + if (SNMALLOC_UNLIKELY(p == nullptr)) + { + if (size != 0) + return ENOMEM; + } + *memptr = p; + return 0; + } +} // namespace snmalloc::libc \ No newline at end of file diff --git a/src/snmalloc/override/malloc.cc b/src/snmalloc/override/malloc.cc index b975848..fd5eca1 100644 --- a/src/snmalloc/override/malloc.cc +++ b/src/snmalloc/override/malloc.cc @@ -1,8 +1,6 @@ +#include "libc.h" #include "override.h" -#include -#include - using namespace snmalloc; #ifndef MALLOC_USABLE_SIZE_QUALIFIER @@ -13,52 +11,34 @@ extern "C" { SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr) { - return ThreadAlloc::get().external_pointer(ptr); + return snmalloc::libc::__malloc_end_pointer(ptr); } SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) { - return ThreadAlloc::get().alloc(size); + return snmalloc::libc::malloc(size); } SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free)(void* ptr) { - ThreadAlloc::get().dealloc(ptr); + snmalloc::libc::free(ptr); } SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(cfree)(void* ptr) { - ThreadAlloc::get().dealloc(ptr); - } - - /** - * Clang was helpfully inlining the constant return value, and - * thus converting from a tail call to an ordinary call. - */ - SNMALLOC_EXPORT inline void* snmalloc_not_allocated = nullptr; - - static SNMALLOC_SLOW_PATH void* SNMALLOC_NAME_MANGLE(snmalloc_set_error)() - { - errno = ENOMEM; - return snmalloc_not_allocated; + snmalloc::libc::free(ptr); } SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size) { - bool overflow = false; - size_t sz = bits::umul(size, nmemb, overflow); - if (SNMALLOC_UNLIKELY(overflow)) - { - return SNMALLOC_NAME_MANGLE(snmalloc_set_error)(); - } - return ThreadAlloc::get().alloc(sz); + return snmalloc::libc::calloc(nmemb, size); } SNMALLOC_EXPORT size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)( MALLOC_USABLE_SIZE_QUALIFIER void* ptr) { - return ThreadAlloc::get().alloc_size(ptr); + return snmalloc::libc::malloc_usable_size(ptr); } SNMALLOC_EXPORT @@ -69,162 +49,53 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(realloc)(void* ptr, size_t size) { - auto& a = ThreadAlloc::get(); - size_t sz = a.alloc_size(ptr); - // Keep the current allocation if the given size is in the same sizeclass. - if (sz == round_size(size)) - { -#ifdef SNMALLOC_PASS_THROUGH - // snmallocs alignment guarantees can be broken by realloc in pass-through - // this is not exercised, by existing clients, but is tested. - if (pointer_align_up(ptr, natural_alignment(size)) == ptr) - return ptr; -#else - return ptr; -#endif - } - - if (size == (size_t)-1) - { - errno = ENOMEM; - return nullptr; - } - - void* p = a.alloc(size); - if (SNMALLOC_LIKELY(p != nullptr)) - { - sz = bits::min(size, sz); - // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy - // otherwise. - if (sz != 0) - memcpy(p, ptr, sz); - a.dealloc(ptr); - } - else if (SNMALLOC_LIKELY(size == 0)) - { - a.dealloc(ptr); - } - else - { - errno = ENOMEM; - } - return p; + return snmalloc::libc::realloc(ptr, size); } #if !defined(SNMALLOC_NO_REALLOCARRAY) SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(reallocarray)(void* ptr, size_t nmemb, size_t size) { - bool overflow = false; - size_t sz = bits::umul(size, nmemb, overflow); - if (overflow) - { - errno = ENOMEM; - return nullptr; - } - return SNMALLOC_NAME_MANGLE(realloc)(ptr, sz); + return snmalloc::libc::reallocarray(ptr, nmemb, size); } #endif #if !defined(SNMALLOC_NO_REALLOCARR) SNMALLOC_EXPORT int - SNMALLOC_NAME_MANGLE(reallocarr)(void* ptr_, size_t nmemb, size_t size) + SNMALLOC_NAME_MANGLE(reallocarr)(void* ptr, size_t nmemb, size_t size) { - int err = errno; - auto& a = ThreadAlloc::get(); - bool overflow = false; - size_t sz = bits::umul(size, nmemb, overflow); - if (sz == 0) - { - errno = err; - return 0; - } - if (overflow) - { - errno = err; - return EOVERFLOW; - } - - void** ptr = reinterpret_cast(ptr_); - void* p = a.alloc(sz); - if (p == nullptr) - { - errno = ENOMEM; - return ENOMEM; - } - - sz = bits::min(sz, a.alloc_size(*ptr)); - - SNMALLOC_ASSUME(*ptr != nullptr || sz == 0); - // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy - // otherwise. - if (sz != 0) - memcpy(p, *ptr, sz); - errno = err; - a.dealloc(*ptr); - *ptr = p; - return 0; + return snmalloc::libc::reallocarr(ptr, nmemb, size); } #endif SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size) { - if ((alignment == 0) || (alignment == size_t(-1))) - { - errno = EINVAL; - return nullptr; - } - - if ((size + alignment) < size) - { - errno = ENOMEM; - return nullptr; - } - - return SNMALLOC_NAME_MANGLE(malloc)(aligned_size(alignment, size)); + return snmalloc::libc::memalign(alignment, size); } SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(aligned_alloc)(size_t alignment, size_t size) { - SNMALLOC_ASSERT((size % alignment) == 0); - return SNMALLOC_NAME_MANGLE(memalign)(alignment, size); + return snmalloc::libc::memalign(alignment, size); } SNMALLOC_EXPORT int SNMALLOC_NAME_MANGLE(posix_memalign)( void** memptr, size_t alignment, size_t size) { - if ((alignment < sizeof(uintptr_t) || ((alignment & (alignment - 1)) != 0))) - { - return EINVAL; - } - - void* p = SNMALLOC_NAME_MANGLE(memalign)(alignment, size); - if (SNMALLOC_UNLIKELY(p == nullptr)) - { - if (size != 0) - return ENOMEM; - } - *memptr = p; - return 0; + return snmalloc::libc::posix_memalign(memptr, alignment, size); } #if !defined(__FreeBSD__) && !defined(__OpenBSD__) SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(valloc)(size_t size) { - return SNMALLOC_NAME_MANGLE(memalign)(OS_PAGE_SIZE, size); + return snmalloc::libc::memalign(OS_PAGE_SIZE, size); } #endif SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(pvalloc)(size_t size) { - if (size == size_t(-1)) - { - errno = ENOMEM; - return nullptr; - } - return SNMALLOC_NAME_MANGLE(memalign)( + return snmalloc::libc::memalign( OS_PAGE_SIZE, (size + OS_PAGE_SIZE - 1) & ~(OS_PAGE_SIZE - 1)); } diff --git a/src/snmalloc/override/new.cc b/src/snmalloc/override/new.cc index d3e084d..85636fe 100644 --- a/src/snmalloc/override/new.cc +++ b/src/snmalloc/override/new.cc @@ -20,102 +20,96 @@ using namespace snmalloc; void* operator new(size_t size) { - return ThreadAlloc::get().alloc(size); + return snmalloc::libc::malloc(size); } void* operator new[](size_t size) { - return ThreadAlloc::get().alloc(size); + return snmalloc::libc::malloc(size); } void* operator new(size_t size, std::nothrow_t&) { - return ThreadAlloc::get().alloc(size); + return snmalloc::libc::malloc(size); } void* operator new[](size_t size, std::nothrow_t&) { - return ThreadAlloc::get().alloc(size); + return snmalloc::libc::malloc(size); } void operator delete(void* p) EXCEPTSPEC { - ThreadAlloc::get().dealloc(p); + snmalloc::libc::free(p); } void operator delete(void* p, size_t size) EXCEPTSPEC { - if (p == nullptr) - return; - ThreadAlloc::get().dealloc(p, size); + snmalloc::libc::free_sized(p, size); } void operator delete(void* p, std::nothrow_t&) { - ThreadAlloc::get().dealloc(p); + snmalloc::libc::free(p); } void operator delete[](void* p) EXCEPTSPEC { - ThreadAlloc::get().dealloc(p); + snmalloc::libc::free(p); } void operator delete[](void* p, size_t size) EXCEPTSPEC { - if (p == nullptr) - return; - ThreadAlloc::get().dealloc(p, size); + snmalloc::libc::free_sized(p, size); } void operator delete[](void* p, std::nothrow_t&) { - ThreadAlloc::get().dealloc(p); + snmalloc::libc::free(p); } void* operator new(size_t size, std::align_val_t val) { size = aligned_size(size_t(val), size); - return ThreadAlloc::get().alloc(size); + return snmalloc::libc::malloc(size); } void* operator new[](size_t size, std::align_val_t val) { size = aligned_size(size_t(val), size); - return ThreadAlloc::get().alloc(size); + return snmalloc::libc::malloc(size); } void* operator new(size_t size, std::align_val_t val, std::nothrow_t&) { size = aligned_size(size_t(val), size); - return ThreadAlloc::get().alloc(size); + return snmalloc::libc::malloc(size); } void* operator new[](size_t size, std::align_val_t val, std::nothrow_t&) { size = aligned_size(size_t(val), size); - return ThreadAlloc::get().alloc(size); + return snmalloc::libc::malloc(size); } void operator delete(void* p, std::align_val_t) EXCEPTSPEC { - ThreadAlloc::get().dealloc(p); + snmalloc::libc::free(p); } void operator delete[](void* p, std::align_val_t) EXCEPTSPEC { - ThreadAlloc::get().dealloc(p); + snmalloc::libc::free(p); } void operator delete(void* p, size_t size, std::align_val_t val) EXCEPTSPEC { size = aligned_size(size_t(val), size); - ThreadAlloc::get().dealloc(p, size); + snmalloc::libc::free_sized(p, size); } void operator delete[](void* p, size_t size, std::align_val_t val) EXCEPTSPEC { - if (p == nullptr) - return; size = aligned_size(size_t(val), size); - ThreadAlloc::get().dealloc(p, size); + snmalloc::libc::free_sized(p, size); } diff --git a/src/snmalloc/snmalloc_front.h b/src/snmalloc/snmalloc_front.h index 4c5aa60..7a16b6c 100644 --- a/src/snmalloc/snmalloc_front.h +++ b/src/snmalloc/snmalloc_front.h @@ -1 +1,2 @@ #include "global/global.h" +#include "override/libc.h" \ No newline at end of file