diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 6684e91..922c8f6 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -127,10 +127,11 @@ namespace snmalloc */ static void inner_release() { - if (get() != &GlobalPlaceHolder) + Alloc*& a = get_noncachable(); + if (a != &GlobalPlaceHolder) { - current_alloc_pool()->release(get()); - get() = &GlobalPlaceHolder; + current_alloc_pool()->release(a); + a = &GlobalPlaceHolder; } } @@ -152,13 +153,38 @@ namespace snmalloc /** * Public interface, returns the allocator for this thread, constructing * one if necessary. + * + * If no operations have been performed on an allocator returned by either + * `get()` nor `get_noncachable`, then the value contained in the return + * will be an Alloc* that will always use the slow path. The Alloc*& will + * be updated, and is performant to use for subsequent calls. + * + * Only use this API if you intend to use the returned allocator just once + * per call, you store the indirection (Alloc*&) rather than (Alloc*), or if + * you know other calls have already been made to the allocator. */ - static inline Alloc*& get() + static inline Alloc*& get_noncachable() { static thread_local Alloc* alloc = &GlobalPlaceHolder; return alloc; } + /** + * Public interface, returns the allocator for this thread, constructing + * one if necessary. + * + * The returned Alloc* is guaranteed to be initialised. This incurs a cost, + * so use `get_noncachable` if you can meet its criteria. + */ + static SNMALLOC_FAST_PATH Alloc* get() + { + auto alloc = get_noncachable(); + auto new_alloc = lazy_replacement(alloc); + return (likely(new_alloc == nullptr)) ? + alloc : + reinterpret_cast(new_alloc); + } + static void register_cleanup() { static thread_local OnDestruct @@ -195,7 +221,7 @@ namespace snmalloc */ SNMALLOC_SLOW_PATH inline void* lazy_replacement_slow() { - auto*& local_alloc = ThreadAlloc::get(); + auto*& local_alloc = ThreadAlloc::get_noncachable(); if ((local_alloc != nullptr) && (local_alloc != &GlobalPlaceHolder)) { return local_alloc; diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 8ac2338..fe1ae39 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -37,7 +37,7 @@ int main() oe_end = (uint8_t*)oe_base + size; std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl; - auto& a = ThreadAlloc::get(); + auto a = ThreadAlloc::get(); for (size_t i = 0; i < 1000; i++) { diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index cbfa26e..73f40f5 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -7,7 +7,7 @@ using namespace snmalloc; void test_alloc_dealloc_64k() { - auto& alloc = ThreadAlloc::get(); + auto alloc = ThreadAlloc::get(); constexpr size_t count = 1 << 12; constexpr size_t outer_count = 12; @@ -39,7 +39,7 @@ void test_alloc_dealloc_64k() void test_random_allocation() { - auto& alloc = ThreadAlloc::get(); + auto alloc = ThreadAlloc::get(); std::unordered_set allocated; constexpr size_t count = 10000; @@ -91,7 +91,7 @@ void test_random_allocation() void test_calloc() { - auto& alloc = ThreadAlloc::get(); + auto alloc = ThreadAlloc::get(); for (size_t size = 16; size <= (1 << 24); size <<= 1) { @@ -162,7 +162,7 @@ void test_double_alloc() void test_external_pointer() { // Malloc does not have an external pointer querying mechanism. - auto& alloc = ThreadAlloc::get(); + auto alloc = ThreadAlloc::get(); for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++) { @@ -208,7 +208,7 @@ void test_external_pointer_large() { xoroshiro::p128r64 r; - auto& alloc = ThreadAlloc::get(); + auto alloc = ThreadAlloc::get(); constexpr size_t count_log = snmalloc::bits::is64() ? 5 : 3; constexpr size_t count = 1 << count_log; @@ -244,7 +244,7 @@ void test_external_pointer_large() void test_external_pointer_dealloc_bug() { - auto& alloc = ThreadAlloc::get(); + auto alloc = ThreadAlloc::get(); constexpr size_t count = (SUPERSLAB_SIZE / SLAB_SIZE) * 2; void* allocs[count]; @@ -268,7 +268,7 @@ void test_external_pointer_dealloc_bug() void test_alloc_16M() { - auto& alloc = ThreadAlloc::get(); + auto alloc = ThreadAlloc::get(); // sizes >= 16M use large_alloc const size_t size = 16'000'000; @@ -279,7 +279,7 @@ void test_alloc_16M() void test_calloc_16M() { - auto& alloc = ThreadAlloc::get(); + auto alloc = ThreadAlloc::get(); // sizes >= 16M use large_alloc const size_t size = 16'000'000; diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 11cd631..1858724 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -75,7 +75,7 @@ size_t swapcount; void test_tasks_f(size_t id) { - Alloc*& a = ThreadAlloc::get(); + Alloc* a = ThreadAlloc::get(); xoroshiro::p128r32 r(id + 5000); for (size_t n = 0; n < swapcount; n++) @@ -100,7 +100,7 @@ void test_tasks_f(size_t id) void test_tasks(size_t num_tasks, size_t count, size_t size) { - Alloc*& a = ThreadAlloc::get(); + Alloc* a = ThreadAlloc::get(); contention = new std::atomic[size]; xoroshiro::p128r32 r; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 0105a47..d87baeb 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -49,7 +49,7 @@ namespace test void test_external_pointer(xoroshiro::p128r64& r) { - auto& alloc = ThreadAlloc::get(); + auto alloc = ThreadAlloc::get(); #ifdef NDEBUG static constexpr size_t iterations = 10000000; #else @@ -79,9 +79,6 @@ namespace test int main(int, char**) { xoroshiro::p128r64 r; - // Force a per-thread allocator to actually exist. - void* p = ThreadAlloc::get()->alloc(16); - ThreadAlloc::get()->dealloc(p); #if NDEBUG size_t nn = 30; #else diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index aef53f2..3b3700c 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -7,7 +7,7 @@ using namespace snmalloc; template void test_alloc_dealloc(size_t count, size_t size, bool write) { - auto*& alloc = ThreadAlloc::get(); + auto* alloc = ThreadAlloc::get(); DO_TIME( "Count: " << std::setw(6) << count << ", Size: " << std::setw(6) << size