Add jemalloc compat functions. (#456)
This adds the full set of jemalloc functions that FreeBSD's libc exposes, including some (the `*allocm` family) that are gone from newer versions of jemalloc and the `*allocx` family that replaced them. These are not necessarily efficient implementations but they should allow snmalloc to replace jemalloc without any ABI breakage (in the loosest possible sense). Jemalloc provides a very generic sysctl-like mechanism for setting and getting some values. These are all implemented to return the not-supported error code. This may break code that expects that they will succeed. In particular, these APIs are used to register custom backing-store allocators and to manage caches and arenas. These concepts don't map directly onto snmalloc and attempting to do so would almost certainly not provide the same performance characteristics and so it's better to `LD_PRELOAD` jemalloc (or explicitly link to it) for programs that gain a significant speedup from this.
This commit is contained in:
David Chisnall
388
src/override/jemalloc_compat.cc
Normal file
388
src/override/jemalloc_compat.cc
Normal file
@@ -0,0 +1,388 @@
|
||||
#include "override.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <string.h>
|
||||
|
||||
using namespace snmalloc;
|
||||
namespace
|
||||
{
|
||||
/**
|
||||
* Helper for JEMalloc-compatible non-standard APIs. These take a flags
|
||||
* argument as an `int`. This class provides a wrapper for extracting the
|
||||
* fields embedded in this API.
|
||||
*/
|
||||
class JEMallocFlags
|
||||
{
|
||||
/**
|
||||
* The raw flags.
|
||||
*/
|
||||
int flags;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Constructor, takes a `flags` parameter from one of the `*allocx()`
|
||||
* JEMalloc APIs.
|
||||
*/
|
||||
constexpr JEMallocFlags(int flags) : flags(flags) {}
|
||||
|
||||
/**
|
||||
* Jemalloc's *allocx APIs store the alignment in the low 6 bits of the
|
||||
* flags, allowing any alignment up to 2^63.
|
||||
*/
|
||||
constexpr int log2align()
|
||||
{
|
||||
return flags & 0x3f;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc's *allocx APIs use bit 6 to indicate whether memory should be
|
||||
* zeroed.
|
||||
*/
|
||||
constexpr bool should_zero()
|
||||
{
|
||||
return (flags & 0x40) == 0x40;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc's *allocm APIs use bit 7 to indicate whether reallocation may
|
||||
* move. This is ignored by the `*allocx` functions.
|
||||
*/
|
||||
constexpr bool may_not_move()
|
||||
{
|
||||
return (flags & 0x80) == 0x80;
|
||||
}
|
||||
|
||||
size_t aligned_size(size_t size)
|
||||
{
|
||||
return ::aligned_size(bits::one_at_bit<size_t>(log2align()), size);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Error codes from Jemalloc 3's experimental API.
|
||||
*/
|
||||
enum JEMalloc3Result
|
||||
{
|
||||
/**
|
||||
* Allocation succeeded.
|
||||
*/
|
||||
allocm_success = 0,
|
||||
|
||||
/**
|
||||
* Allocation failed because memory was not available.
|
||||
*/
|
||||
allocm_err_oom = 1,
|
||||
|
||||
/**
|
||||
* Reallocation failed because it would have required moving.
|
||||
*/
|
||||
allocm_err_not_moved = 2
|
||||
};
|
||||
} // namespace
|
||||
|
||||
extern "C"
|
||||
{
|
||||
// Stub implementations for jemalloc compatibility.
|
||||
// These are called by FreeBSD's libthr (pthreads) to notify malloc of
|
||||
// various events. They are currently unused, though we may wish to reset
|
||||
// statistics on fork if built with statistics.
|
||||
|
||||
SNMALLOC_EXPORT SNMALLOC_USED_FUNCTION inline void _malloc_prefork(void) {}
|
||||
SNMALLOC_EXPORT SNMALLOC_USED_FUNCTION inline void _malloc_postfork(void) {}
|
||||
SNMALLOC_EXPORT SNMALLOC_USED_FUNCTION inline void _malloc_first_thread(void)
|
||||
{}
|
||||
|
||||
/**
|
||||
* Jemalloc API provides a way of avoiding name lookup when calling
|
||||
* `mallctl`. For now, always return an error.
|
||||
*/
|
||||
int SNMALLOC_NAME_MANGLE(mallctlnametomib)(const char*, size_t*, size_t*)
|
||||
{
|
||||
return ENOENT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc API provides a generic entry point for various functions. For
|
||||
* now, this is always implemented to return an error.
|
||||
*/
|
||||
int SNMALLOC_NAME_MANGLE(mallctlbymib)(
|
||||
const size_t*, size_t, void*, size_t*, void*, size_t)
|
||||
{
|
||||
return ENOENT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc API provides a generic entry point for various functions. For
|
||||
* now, this is always implemented to return an error.
|
||||
*/
|
||||
SNMALLOC_EXPORT int
|
||||
SNMALLOC_NAME_MANGLE(mallctl)(const char*, void*, size_t*, void*, size_t)
|
||||
{
|
||||
return ENOENT;
|
||||
}
|
||||
|
||||
#ifdef SNMALLOC_JEMALLOC3_EXPERIMENTAL
|
||||
/**
|
||||
* Jemalloc 3 experimental API. Allocates at least `size` bytes and returns
|
||||
* the result in `*ptr`, if `rsize` is not null then writes the allocated size
|
||||
* into `*rsize`. `flags` controls whether the memory is zeroed and what
|
||||
* alignment is requested.
|
||||
*/
|
||||
int SNMALLOC_NAME_MANGLE(allocm)(
|
||||
void** ptr, size_t* rsize, size_t size, int flags)
|
||||
{
|
||||
auto f = JEMallocFlags(flags);
|
||||
size = f.aligned_size(size);
|
||||
if (rsize != nullptr)
|
||||
{
|
||||
*rsize = round_size(size);
|
||||
}
|
||||
if (f.should_zero())
|
||||
{
|
||||
*ptr = ThreadAlloc::get().alloc<ZeroMem::YesZero>(size);
|
||||
}
|
||||
else
|
||||
{
|
||||
*ptr = ThreadAlloc::get().alloc(size);
|
||||
}
|
||||
return (*ptr != nullptr) ? allocm_success : allocm_err_oom;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc 3 experimental API. Reallocates the allocation in `*ptr` to be at
|
||||
* least `size` bytes and returns the result in `*ptr`, if `rsize` is not null
|
||||
* then writes the allocated size into `*rsize`. `flags` controls whether the
|
||||
* memory is zeroed and what alignment is requested and whether reallocation
|
||||
* is permitted. If reallocating, the size will be at least `size` + `extra`
|
||||
* bytes.
|
||||
*/
|
||||
int SNMALLOC_NAME_MANGLE(rallocm)(
|
||||
void** ptr, size_t* rsize, size_t size, size_t extra, int flags)
|
||||
{
|
||||
auto f = JEMallocFlags(flags);
|
||||
auto alloc_size = f.aligned_size(size);
|
||||
|
||||
auto& a = ThreadAlloc::get();
|
||||
size_t sz = a.alloc_size(*ptr);
|
||||
// Keep the current allocation if the given size is in the same sizeclass.
|
||||
if (sz == round_size(alloc_size))
|
||||
{
|
||||
if (rsize != nullptr)
|
||||
{
|
||||
*rsize = sz;
|
||||
}
|
||||
return allocm_success;
|
||||
}
|
||||
|
||||
if (f.may_not_move())
|
||||
{
|
||||
return allocm_err_not_moved;
|
||||
}
|
||||
|
||||
if (std::numeric_limits<size_t>::max() - size > extra)
|
||||
{
|
||||
alloc_size = f.aligned_size(size + extra);
|
||||
}
|
||||
|
||||
void* p =
|
||||
f.should_zero() ? a.alloc<YesZero>(alloc_size) : a.alloc(alloc_size);
|
||||
if (SNMALLOC_LIKELY(p != nullptr))
|
||||
{
|
||||
sz = bits::min(alloc_size, sz);
|
||||
// Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy
|
||||
// otherwise.
|
||||
if (sz != 0)
|
||||
{
|
||||
memcpy(p, *ptr, sz);
|
||||
}
|
||||
a.dealloc(*ptr);
|
||||
*ptr = p;
|
||||
if (rsize != nullptr)
|
||||
{
|
||||
*rsize = alloc_size;
|
||||
}
|
||||
return allocm_success;
|
||||
}
|
||||
return allocm_err_oom;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc 3 experimental API. Sets `*rsize` to the size of the allocation
|
||||
* at `*ptr`. The third argument contains some flags relating to arenas that
|
||||
* we ignore.
|
||||
*/
|
||||
int SNMALLOC_NAME_MANGLE(sallocm)(const void* ptr, size_t* rsize, int)
|
||||
{
|
||||
*rsize = ThreadAlloc::get().alloc_size(ptr);
|
||||
return allocm_success;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc 3 experimental API. Deallocates the allocation
|
||||
* at `*ptr`. The second argument contains some flags relating to arenas that
|
||||
* we ignore.
|
||||
*/
|
||||
int SNMALLOC_NAME_MANGLE(dallocm)(void* ptr, int)
|
||||
{
|
||||
ThreadAlloc::get().dealloc(ptr);
|
||||
return allocm_success;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc 3 experimental API. Returns in `*rsize` the size of the
|
||||
* allocation that would be returned if `size` and `flags` are passed to
|
||||
* `allocm`.
|
||||
*/
|
||||
int SNMALLOC_NAME_MANGLE(nallocm)(size_t* rsize, size_t size, int flags)
|
||||
{
|
||||
*rsize = round_size(JEMallocFlags(flags).aligned_size(size));
|
||||
return allocm_success;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef SNMALLOC_JEMALLOC_NONSTANDARD
|
||||
/**
|
||||
* Jemalloc function that provides control over alignment and zeroing
|
||||
* behaviour via the `flags` argument. This argument also includes control
|
||||
* over the thread cache and arena to use. These don't translate directly to
|
||||
* snmalloc and so are ignored.
|
||||
*/
|
||||
SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(mallocx)(size_t size, int flags)
|
||||
{
|
||||
auto f = JEMallocFlags(flags);
|
||||
size = f.aligned_size(size);
|
||||
if (f.should_zero())
|
||||
{
|
||||
return ThreadAlloc::get().alloc<ZeroMem::YesZero>(size);
|
||||
}
|
||||
return ThreadAlloc::get().alloc(size);
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc non-standard function that is similar to `realloc`. This can
|
||||
* request zeroed memory for any newly allocated memory, though only if the
|
||||
* object grows (which, for snmalloc, means if it's copied). The flags
|
||||
* controlling the thread cache and arena are ignored.
|
||||
*/
|
||||
SNMALLOC_EXPORT void*
|
||||
SNMALLOC_NAME_MANGLE(rallocx)(void* ptr, size_t size, int flags)
|
||||
{
|
||||
auto f = JEMallocFlags(flags);
|
||||
size = f.aligned_size(size);
|
||||
|
||||
auto& a = ThreadAlloc::get();
|
||||
size_t sz = round_size(a.alloc_size(ptr));
|
||||
// Keep the current allocation if the given size is in the same sizeclass.
|
||||
if (sz == size)
|
||||
{
|
||||
return ptr;
|
||||
}
|
||||
|
||||
if (size == (size_t)-1)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// We have a choice here of either asking for zeroed memory, or trying to
|
||||
// zero the remainder. The former is *probably* faster for large
|
||||
// allocations, because we get zeroed memory from the PAL and don't zero it
|
||||
// twice. This is not profiled and so should be considered for refactoring
|
||||
// if anyone cares about the performance of these APIs.
|
||||
void* p = f.should_zero() ? a.alloc<YesZero>(size) : a.alloc(size);
|
||||
if (SNMALLOC_LIKELY(p != nullptr))
|
||||
{
|
||||
sz = bits::min(size, sz);
|
||||
// Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy
|
||||
// otherwise.
|
||||
if (sz != 0)
|
||||
memcpy(p, ptr, sz);
|
||||
a.dealloc(ptr);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc non-standard API that performs a `realloc` only if it can do so
|
||||
* without copying and returns the size of the underlying object. With
|
||||
* snmalloc, this simply returns the size of the sizeclass backing the
|
||||
* object.
|
||||
*/
|
||||
size_t SNMALLOC_NAME_MANGLE(xallocx)(void* ptr, size_t, size_t, int)
|
||||
{
|
||||
auto& a = ThreadAlloc::get();
|
||||
return a.alloc_size(ptr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc non-standard API that queries the underlying size of the
|
||||
* allocation.
|
||||
*/
|
||||
size_t SNMALLOC_NAME_MANGLE(sallocx)(const void* ptr, int)
|
||||
{
|
||||
auto& a = ThreadAlloc::get();
|
||||
return a.alloc_size(ptr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc non-standard API that frees `ptr`. The second argument allows
|
||||
* specifying a thread cache or arena but this is currently unused in
|
||||
* snmalloc.
|
||||
*/
|
||||
void SNMALLOC_NAME_MANGLE(dallocx)(void* ptr, int)
|
||||
{
|
||||
ThreadAlloc::get().dealloc(ptr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc non-standard API that frees `ptr`. The second argument specifies
|
||||
* a size, which is intended to speed up the operation. This could improve
|
||||
* performance for snmalloc, if we could guarantee that this is allocated by
|
||||
* the current thread but is otherwise not helpful. The third argument allows
|
||||
* specifying a thread cache or arena but this is currently unused in
|
||||
* snmalloc.
|
||||
*/
|
||||
void SNMALLOC_NAME_MANGLE(sdallocx)(void* ptr, size_t, int)
|
||||
{
|
||||
ThreadAlloc::get().dealloc(ptr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Jemalloc non-standard API that returns the size of memory that would be
|
||||
* allocated if the same arguments were passed to `mallocx`.
|
||||
*/
|
||||
size_t SNMALLOC_NAME_MANGLE(nallocx)(size_t size, int flags)
|
||||
{
|
||||
return round_size(JEMallocFlags(flags).aligned_size(size));
|
||||
}
|
||||
#endif
|
||||
|
||||
#if !defined(__PIC__) && defined(SNMALLOC_BOOTSTRAP_ALLOCATOR)
|
||||
// The following functions are required to work before TLS is set up, in
|
||||
// statically-linked programs. These temporarily grab an allocator from the
|
||||
// pool and return it.
|
||||
|
||||
void* __je_bootstrap_malloc(size_t size)
|
||||
{
|
||||
return get_scoped_allocator()->alloc(size);
|
||||
}
|
||||
|
||||
void* __je_bootstrap_calloc(size_t nmemb, size_t size)
|
||||
{
|
||||
bool overflow = false;
|
||||
size_t sz = bits::umul(size, nmemb, overflow);
|
||||
if (overflow)
|
||||
{
|
||||
errno = ENOMEM;
|
||||
return nullptr;
|
||||
}
|
||||
// Include size 0 in the first sizeclass.
|
||||
sz = ((sz - 1) >> (bits::BITS - 1)) + sz;
|
||||
return get_scoped_allocator()->alloc<ZeroMem::YesZero>(sz);
|
||||
}
|
||||
|
||||
void __je_bootstrap_free(void* ptr)
|
||||
{
|
||||
get_scoped_allocator()->dealloc(ptr);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -215,49 +215,4 @@ extern "C"
|
||||
return SNMALLOC_NAME_MANGLE(memalign)(
|
||||
OS_PAGE_SIZE, (size + OS_PAGE_SIZE - 1) & ~(OS_PAGE_SIZE - 1));
|
||||
}
|
||||
|
||||
// Stub implementations for jemalloc compatibility.
|
||||
// These are called by FreeBSD's libthr (pthreads) to notify malloc of
|
||||
// various events. They are currently unused, though we may wish to reset
|
||||
// statistics on fork if built with statistics.
|
||||
|
||||
SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_prefork)(void) {}
|
||||
SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_postfork)(void) {}
|
||||
SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_first_thread)(void) {}
|
||||
|
||||
SNMALLOC_EXPORT int
|
||||
SNMALLOC_NAME_MANGLE(mallctl)(const char*, void*, size_t*, void*, size_t)
|
||||
{
|
||||
return ENOENT;
|
||||
}
|
||||
|
||||
#if !defined(__PIC__) && defined(SNMALLOC_BOOTSTRAP_ALLOCATOR)
|
||||
// The following functions are required to work before TLS is set up, in
|
||||
// statically-linked programs. These temporarily grab an allocator from the
|
||||
// pool and return it.
|
||||
|
||||
void* __je_bootstrap_malloc(size_t size)
|
||||
{
|
||||
return get_scoped_allocator()->alloc(size);
|
||||
}
|
||||
|
||||
void* __je_bootstrap_calloc(size_t nmemb, size_t size)
|
||||
{
|
||||
bool overflow = false;
|
||||
size_t sz = bits::umul(size, nmemb, overflow);
|
||||
if (overflow)
|
||||
{
|
||||
errno = ENOMEM;
|
||||
return nullptr;
|
||||
}
|
||||
// Include size 0 in the first sizeclass.
|
||||
sz = ((sz - 1) >> (bits::BITS - 1)) + sz;
|
||||
return get_scoped_allocator()->alloc<ZeroMem::YesZero>(sz);
|
||||
}
|
||||
|
||||
void __je_bootstrap_free(void* ptr)
|
||||
{
|
||||
get_scoped_allocator()->dealloc(ptr);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
406
src/test/func/jemalloc/jemalloc.cc
Normal file
406
src/test/func/jemalloc/jemalloc.cc
Normal file
@@ -0,0 +1,406 @@
|
||||
#include <functional>
|
||||
#include <stdio.h>
|
||||
#include <test/setup.h>
|
||||
|
||||
#define SNMALLOC_NAME_MANGLE(a) our_##a
|
||||
#undef SNMALLOC_NO_REALLOCARRAY
|
||||
#undef SNMALLOC_NO_REALLOCARR
|
||||
#define SNMALLOC_BOOTSTRAP_ALLOCATOR
|
||||
#define SNMALLOC_JEMALLOC3_EXPERIMENTAL
|
||||
#define SNMALLOC_JEMALLOC_NONSTANDARD
|
||||
#include "../../../override/jemalloc_compat.cc"
|
||||
#include "../../../override/malloc.cc"
|
||||
|
||||
#if __has_include(<malloc_np.h>)
|
||||
# include <malloc_np.h>
|
||||
#endif
|
||||
|
||||
#ifdef __FreeBSD__
|
||||
/**
|
||||
* Enable testing against the versions that we get from libc or elsewhere.
|
||||
* Enabled by default on FreeBSD where all of the jemalloc functions are
|
||||
* exported from libc.
|
||||
*/
|
||||
# define TEST_JEMALLOC_MALLOCX
|
||||
#endif
|
||||
|
||||
#define OUR_MALLOCX_LG_ALIGN(la) (static_cast<int>(la))
|
||||
#define OUR_MALLOCX_ZERO (one_at_bit<int>(6))
|
||||
|
||||
#define OUR_ALLOCM_NO_MOVE (one_at_bit<int>(7))
|
||||
|
||||
#define OUR_ALLOCM_SUCCESS 0
|
||||
#define OUR_ALLOCM_ERR_OOM 1
|
||||
#define OUR_ALLOCM_ERR_NOT_MOVED 2
|
||||
|
||||
#ifndef MALLOCX_LG_ALIGN
|
||||
# define MALLOCX_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la)
|
||||
#endif
|
||||
#ifndef MALLOCX_ZERO
|
||||
# define MALLOCX_ZERO OUR_MALLOCX_ZERO
|
||||
#endif
|
||||
|
||||
#ifndef ALLOCM_LG_ALIGN
|
||||
# define ALLOCM_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la)
|
||||
#endif
|
||||
#ifndef ALLOCM_ZERO
|
||||
# define ALLOCM_ZERO OUR_MALLOCX_ZERO
|
||||
#endif
|
||||
#ifndef ALLOCM_NO_MOVE
|
||||
# define ALLOCM_NO_MOVE OUR_ALLOCM_NO_MOVE
|
||||
#endif
|
||||
#ifndef ALLOCM_SUCCESS
|
||||
# define ALLOCM_SUCCESS OUR_ALLOCM_SUCCESS
|
||||
#endif
|
||||
#ifndef ALLOCM_ERR_OOM
|
||||
# define ALLOCM_ERR_OOM OUR_ALLOCM_ERR_OOM
|
||||
#endif
|
||||
#ifndef ALLOCM_ERR_NOT_MOVED
|
||||
# define ALLOCM_ERR_NOT_MOVED OUR_ALLOCM_ERR_NOT_MOVED
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
# define __PRETTY_FUNCTION__ __FUNCSIG__
|
||||
#endif
|
||||
|
||||
using namespace snmalloc;
|
||||
using namespace snmalloc::bits;
|
||||
|
||||
namespace
|
||||
{
|
||||
/**
|
||||
* Test whether the MALLOCX_LG_ALIGN macro is defined correctly. This test
|
||||
* will pass trivially if we don't have the malloc_np.h header from
|
||||
* jemalloc, but at least the FreeBSD action runners in CI do have this
|
||||
* header.
|
||||
*/
|
||||
template<int Size>
|
||||
void check_lg_align_macro()
|
||||
{
|
||||
static_assert(
|
||||
OUR_MALLOCX_LG_ALIGN(Size) == MALLOCX_LG_ALIGN(Size),
|
||||
"Our definition of MALLOCX_LG_ALIGN is wrong");
|
||||
static_assert(
|
||||
OUR_MALLOCX_LG_ALIGN(Size) == ALLOCM_LG_ALIGN(Size),
|
||||
"Our definition of ALLOCM_LG_ALIGN is wrong");
|
||||
static_assert(
|
||||
JEMallocFlags(Size).log2align() == Size, "Out log2 align mask is wrong");
|
||||
if constexpr (Size > 0)
|
||||
{
|
||||
check_lg_align_macro<Size - 1>();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The name of the function under test. This is set in the START_TEST macro
|
||||
* and used for error reporting in EXPECT.
|
||||
*/
|
||||
const char* function = nullptr;
|
||||
|
||||
/**
|
||||
* Log that the test started.
|
||||
*/
|
||||
#define START_TEST(msg) \
|
||||
function = __PRETTY_FUNCTION__; \
|
||||
fprintf(stderr, "Starting test: " msg "\n");
|
||||
|
||||
/**
|
||||
* An assertion that fires even in debug builds. Uses the value set by
|
||||
* START_TEST.
|
||||
*/
|
||||
#define EXPECT(x, msg, ...) \
|
||||
if (!(x)) \
|
||||
{ \
|
||||
fprintf( \
|
||||
stderr, \
|
||||
"%s:%d in %s: " msg "\n", \
|
||||
__FILE__, \
|
||||
__LINE__, \
|
||||
function, \
|
||||
##__VA_ARGS__); \
|
||||
fflush(stderr); \
|
||||
abort(); \
|
||||
}
|
||||
|
||||
/**
|
||||
* The default maximum number of bits of address space to use for tests.
|
||||
* This is clamped on platforms without lazy commit because this much RAM
|
||||
* (or, at least, commit charge) will be used on such systems.
|
||||
*
|
||||
* Thread sanitizer makes these tests *very* slow, so reduce the size
|
||||
* significantly when it's enabled.
|
||||
*/
|
||||
constexpr size_t DefaultMax = 22;
|
||||
|
||||
/**
|
||||
* Run a test with a range of sizes and alignments. The `test` argument is
|
||||
* called with a size and log2 alignment as parameters.
|
||||
*/
|
||||
template<size_t Log2MaxSize = DefaultMax>
|
||||
void test_sizes_and_alignments(std::function<void(size_t, int)> test)
|
||||
{
|
||||
constexpr size_t low = 5;
|
||||
for (size_t base = low; base < Log2MaxSize; base++)
|
||||
{
|
||||
fprintf(stderr, "\tTrying 0x%zx-byte allocations\n", one_at_bit(base));
|
||||
for (size_t i = 0; i < one_at_bit(low); i++)
|
||||
{
|
||||
for (int align = 1; align < 20; align++)
|
||||
{
|
||||
test(one_at_bit(base) + (i << (base - low)), align);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that the size reported by nallocx corresponds to the size reported by
|
||||
* sallocx on the return value from mallocx.
|
||||
*/
|
||||
template<
|
||||
void*(Mallocx)(size_t, int),
|
||||
void(Dallocx)(void*, int),
|
||||
size_t(Sallocx)(const void*, int),
|
||||
size_t(Nallocx)(size_t, int)>
|
||||
void test_size()
|
||||
{
|
||||
START_TEST("nallocx and mallocx return the same size");
|
||||
test_sizes_and_alignments([](size_t size, int align) {
|
||||
int flags = MALLOCX_LG_ALIGN(align);
|
||||
size_t expected = Nallocx(size, flags);
|
||||
void* ptr = Mallocx(size, flags);
|
||||
EXPECT(
|
||||
ptr != nullptr,
|
||||
"Failed to allocate 0x%zx bytes with %d bit alignment",
|
||||
size,
|
||||
align);
|
||||
size_t allocated = Sallocx(ptr, 0);
|
||||
EXPECT(
|
||||
allocated == expected,
|
||||
"Expected to have allocated 0x%zx bytes, got 0x%zx bytes",
|
||||
expected,
|
||||
allocated);
|
||||
Dallocx(ptr, 0);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that, when we request zeroing in rallocx, we get zeroed memory.
|
||||
*/
|
||||
template<
|
||||
void*(Mallocx)(size_t, int),
|
||||
void(Dallocx)(void*, int),
|
||||
void*(Rallocx)(void*, size_t, int)>
|
||||
void test_zeroing()
|
||||
{
|
||||
START_TEST("rallocx can zero the remaining space.");
|
||||
// The Rallocx call will copy everything in the first malloc, so stay
|
||||
// fairly small.
|
||||
auto test = [](size_t size, int align) {
|
||||
int flags = MALLOCX_LG_ALIGN(align) | MALLOCX_ZERO;
|
||||
char* ptr = static_cast<char*>(Mallocx(size, flags));
|
||||
ptr = static_cast<char*>(Rallocx(ptr, size * 2, flags));
|
||||
EXPECT(
|
||||
ptr != nullptr,
|
||||
"Failed to reallocate for 0x%zx byte allocation",
|
||||
size * 2);
|
||||
EXPECT(
|
||||
ptr[size] == 0,
|
||||
"Memory not zero initialised for 0x%zx byte reallocation from 0x%zx "
|
||||
"byte allocation",
|
||||
size * 2,
|
||||
size);
|
||||
// The second time we run this test, we if we're allocating from a free
|
||||
// list then we will reuse this, so make sure it requires explicit
|
||||
// zeroing.
|
||||
ptr[size] = 12;
|
||||
Dallocx(ptr, 0);
|
||||
};
|
||||
test_sizes_and_alignments<22>(test);
|
||||
test_sizes_and_alignments<22>(test);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that xallocx reports a size that is at least the requested amount.
|
||||
*/
|
||||
template<
|
||||
void*(Mallocx)(size_t, int),
|
||||
void(Dallocx)(void*, int),
|
||||
size_t(Xallocx)(void*, size_t, size_t, int)>
|
||||
void test_xallocx()
|
||||
{
|
||||
START_TEST("xallocx returns a sensible value.");
|
||||
// The Rallocx call will copy all of these, so stay fairly small.
|
||||
auto test = [](size_t size, int align) {
|
||||
int flags = MALLOCX_LG_ALIGN(align);
|
||||
void* ptr = Mallocx(size, flags);
|
||||
EXPECT(
|
||||
ptr != nullptr, "Failed to allocate for 0x%zx byte allocation", size);
|
||||
size_t sz = Xallocx(ptr, size, 1024, flags);
|
||||
EXPECT(
|
||||
sz >= size, "xalloc returned 0x%zx, expected at least 0x%zx", sz, size);
|
||||
Dallocx(ptr, 0);
|
||||
};
|
||||
test_sizes_and_alignments(test);
|
||||
}
|
||||
|
||||
template<
|
||||
int(Allocm)(void**, size_t*, size_t, int),
|
||||
int(Sallocm)(const void*, size_t*, int),
|
||||
int(Dallocm)(void*, int),
|
||||
int(Nallocm)(size_t*, size_t, int)>
|
||||
void test_nallocm_size()
|
||||
{
|
||||
START_TEST("nallocm and allocm return the same size");
|
||||
test_sizes_and_alignments([](size_t size, int align) {
|
||||
int flags = ALLOCM_LG_ALIGN(align);
|
||||
size_t expected;
|
||||
int ret = Nallocm(&expected, size, flags);
|
||||
EXPECT(
|
||||
(ret == ALLOCM_SUCCESS),
|
||||
"nallocm(%zx, %d) failed with error %d",
|
||||
size,
|
||||
flags,
|
||||
ret);
|
||||
void* ptr;
|
||||
size_t allocated;
|
||||
ret = Allocm(&ptr, &allocated, size, flags);
|
||||
EXPECT(
|
||||
(ptr != nullptr) && (ret == ALLOCM_SUCCESS),
|
||||
"Failed to allocate 0x%zx bytes with %d bit alignment",
|
||||
size,
|
||||
align);
|
||||
EXPECT(
|
||||
allocated == expected,
|
||||
"Expected to have allocated 0x%zx bytes, got 0x%zx bytes",
|
||||
expected,
|
||||
allocated);
|
||||
ret = Sallocm(ptr, &expected, 0);
|
||||
EXPECT(
|
||||
(ret == ALLOCM_SUCCESS) && (allocated == expected),
|
||||
"Expected to have allocated 0x%zx bytes, got 0x%zx bytes",
|
||||
expected,
|
||||
allocated);
|
||||
|
||||
Dallocm(ptr, 0);
|
||||
});
|
||||
}
|
||||
|
||||
template<
|
||||
int(Allocm)(void**, size_t*, size_t, int),
|
||||
int(Rallocm)(void**, size_t*, size_t, size_t, int),
|
||||
int(Dallocm)(void*, int)>
|
||||
void test_rallocm_nomove()
|
||||
{
|
||||
START_TEST("rallocm non-moving behaviour");
|
||||
test_sizes_and_alignments([](size_t size, int align) {
|
||||
int flags = ALLOCM_LG_ALIGN(align);
|
||||
void* ptr;
|
||||
size_t allocated;
|
||||
int ret = Allocm(&ptr, &allocated, size, flags);
|
||||
void* orig = ptr;
|
||||
EXPECT(
|
||||
(ptr != nullptr) && (ret == ALLOCM_SUCCESS),
|
||||
"Failed to allocate 0x%zx bytes with %d bit alignment",
|
||||
size,
|
||||
align);
|
||||
ret = Rallocm(&ptr, nullptr, allocated + 1, 12, flags | ALLOCM_NO_MOVE);
|
||||
EXPECT(
|
||||
(ret == ALLOCM_ERR_NOT_MOVED) || (ptr == orig),
|
||||
"Expected rallocm not to be able to move or reallocate, but return was "
|
||||
"%d\n",
|
||||
ret);
|
||||
Dallocm(ptr, 0);
|
||||
});
|
||||
}
|
||||
|
||||
template<
|
||||
int(Allocm)(void**, size_t*, size_t, int),
|
||||
int(Rallocm)(void**, size_t*, size_t, size_t, int),
|
||||
int(Sallocm)(const void*, size_t*, int),
|
||||
int(Dallocm)(void*, int),
|
||||
int(Nallocm)(size_t*, size_t, int)>
|
||||
void test_legacy_experimental_apis()
|
||||
{
|
||||
START_TEST("allocm out-of-memory behaviour");
|
||||
void* ptr = nullptr;
|
||||
int ret = Allocm(&ptr, nullptr, std::numeric_limits<size_t>::max() / 2, 0);
|
||||
EXPECT(
|
||||
(ptr == nullptr) && (ret == OUR_ALLOCM_ERR_OOM),
|
||||
"Expected massive allocation to fail with out of memory (%d), received "
|
||||
"allocation %p, return code %d",
|
||||
OUR_ALLOCM_ERR_OOM,
|
||||
ptr,
|
||||
ret);
|
||||
test_nallocm_size<Allocm, Sallocm, Dallocm, Nallocm>();
|
||||
test_rallocm_nomove<Allocm, Rallocm, Dallocm>();
|
||||
}
|
||||
}
|
||||
|
||||
extern "C"
|
||||
{
|
||||
/**
|
||||
* The jemalloc 3.x experimental APIs are gone from the headers in newer
|
||||
* versions, but are still present in FreeBSD libc, so declare them here
|
||||
* for testing.
|
||||
*/
|
||||
int allocm(void**, size_t*, size_t, int);
|
||||
int rallocm(void**, size_t*, size_t, size_t, int);
|
||||
int sallocm(const void*, size_t*, int);
|
||||
int dallocm(void*, int);
|
||||
int nallocm(size_t*, size_t, int);
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
#ifdef SNMALLOC_PASS_THROUGH
|
||||
return 0;
|
||||
#endif
|
||||
check_lg_align_macro<63>();
|
||||
static_assert(
|
||||
OUR_MALLOCX_ZERO == MALLOCX_ZERO, "Our MALLOCX_ZERO macro is wrong");
|
||||
static_assert(
|
||||
OUR_MALLOCX_ZERO == ALLOCM_ZERO, "Our ALLOCM_ZERO macro is wrong");
|
||||
static_assert(
|
||||
OUR_ALLOCM_NO_MOVE == ALLOCM_NO_MOVE, "Our ALLOCM_NO_MOVE macro is wrong");
|
||||
static_assert(
|
||||
JEMallocFlags(MALLOCX_ZERO).should_zero(),
|
||||
"Our MALLOCX_ZERO is not the value that we are using");
|
||||
static_assert(
|
||||
!JEMallocFlags(~MALLOCX_ZERO).should_zero(),
|
||||
"Our MALLOCX_ZERO is not the value that we are using");
|
||||
static_assert(
|
||||
JEMallocFlags(ALLOCM_NO_MOVE).may_not_move(),
|
||||
"Our ALLOCM_NO_MOVE is not the value that we are using");
|
||||
static_assert(
|
||||
!JEMallocFlags(~ALLOCM_NO_MOVE).may_not_move(),
|
||||
"Our ALLOCM_NO_MOVE is not the value that we are using");
|
||||
test_size<our_mallocx, our_dallocx, our_sallocx, our_nallocx>();
|
||||
test_zeroing<our_mallocx, our_dallocx, our_rallocx>();
|
||||
test_xallocx<our_mallocx, our_dallocx, our_xallocx>();
|
||||
test_legacy_experimental_apis<
|
||||
our_allocm,
|
||||
our_rallocm,
|
||||
our_sallocm,
|
||||
our_dallocm,
|
||||
our_nallocm>();
|
||||
|
||||
#ifndef __PIC__
|
||||
void* bootstrap = __je_bootstrap_malloc(42);
|
||||
if (bootstrap == nullptr)
|
||||
{
|
||||
printf("Failed to allocate from bootstrap malloc\n");
|
||||
}
|
||||
__je_bootstrap_free(bootstrap);
|
||||
#endif
|
||||
|
||||
// These tests are for jemalloc compatibility and so should work with
|
||||
// jemalloc's implementation of these functions. If TEST_JEMALLOC is
|
||||
// defined then we try
|
||||
#ifdef TEST_JEMALLOC_MALLOCX
|
||||
test_size<mallocx, dallocx, sallocx, nallocx>();
|
||||
test_zeroing<mallocx, dallocx, rallocx>();
|
||||
test_xallocx<mallocx, dallocx, xallocx>();
|
||||
test_legacy_experimental_apis<allocm, rallocm, sallocm, dallocm, nallocm>();
|
||||
#endif
|
||||
}
|
||||
@@ -348,14 +348,6 @@ int main(int argc, char** argv)
|
||||
abort();
|
||||
}
|
||||
|
||||
#ifndef __PIC__
|
||||
snmalloc::debug_check_empty<snmalloc::Globals>();
|
||||
void* bootstrap = __je_bootstrap_malloc(42);
|
||||
if (bootstrap == nullptr)
|
||||
{
|
||||
printf("Failed to allocate from bootstrap malloc\n");
|
||||
}
|
||||
__je_bootstrap_free(bootstrap);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user