From d900e294243ede0c1f4ccd2f04a8dd6fab78e1ed Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 31 Mar 2020 09:17:53 +0100 Subject: [PATCH] Improve slow path performance for allocation (#143) * Remote dealloc refactor. * Improve remote dealloc Change remote to count down to 0, so fast path does not need a constant. Use signed value so that branch does not depend on addition. * Inline remote_dealloc The fast path of remote_dealloc is sufficiently compact that it can be inlined. * Improve fast path in Slab::alloc Turn the internal structure into tail calls, to improve fast path. Should be no algorithmic changes. * Refactor initialisation to help fast path. Break lazy initialisation into two functions, so it is easier to codegen fast paths. * Minor tidy to statically sized dealloc. * Refactor semi-slow path for alloc Make the backup path a bit faster. Only algorithmic change is to delay checking for first allocation. Otherwise, should be unchanged. * Test initial operation of a thread The first operation a new thread takes is special. It results in allocating an allocator, and swinging it into the TLS. This makes this a very special path, that is rarely tested. This test generates a lot of threads to cover the first alloc and dealloc operations. * Correctly handle reusing get_noncachable * Fix large alloc stats Large alloc stats aren't necessarily balanced on a thread, this changes to tracking individual pushs and pops, rather than the net effect (with an unsigned value). * Fix TLS init on large alloc path * Add Bump ptrs to allocator Each allocator has a bump ptr for each size class. This is no longer slab local. Slabs that haven't been fully allocated no longer need to be in the DLL for this sizeclass. * Change to a cycle non-empty list This change reduces the branching in the case of finding a new free list. Using a non-empty cyclic list enables branch free add, and a single branch in remove to detect the empty case. * Update differences * Rename first allocation Use needs initialisation as makes more sense for other scenarios. * Use a ptrdiff to help with zero init. * Make GlobalPlaceholder zero init The GlobalPlaceholder allocator is now a zero init block of memory. This removes various issues for when things are initialised. It is made read-only to we detect write to it on some platforms. --- difference.md | 5 +- src/ds/address.h | 20 ++ src/ds/bits.h | 4 +- src/ds/cdllist.h | 122 ++++++++ src/ds/dllist.h | 8 +- src/ds/helpers.h | 2 +- src/mem/alloc.h | 281 +++++++++++++----- src/mem/allocstats.h | 32 +- src/mem/globalalloc.h | 22 +- src/mem/largealloc.h | 12 +- src/mem/metaslab.h | 24 +- src/mem/pagemap.h | 2 +- src/mem/pool.h | 2 +- src/mem/slab.h | 150 +++++----- src/mem/superslab.h | 24 +- src/mem/threadalloc.h | 91 ++++-- src/pal/pal_consts.h | 10 +- src/pal/pal_windows.h | 4 +- .../func/first_operation/first_operation.cc | 112 +++++++ src/test/perf/low_memory/low-memory.cc | 2 +- 20 files changed, 690 insertions(+), 239 deletions(-) create mode 100644 src/ds/cdllist.h create mode 100644 src/test/func/first_operation/first_operation.cc diff --git a/difference.md b/difference.md index aa8bb1c..273a74c 100644 --- a/difference.md +++ b/difference.md @@ -33,7 +33,10 @@ This document outlines the changes that have diverged from 4. We now store a direct pointer to the next element in each slabs free list rather than a relative offset into the slab. This enables list calculation on the fast path. - + + 5. There is a single bump-ptr per size class that is part of the + allocator structure. The per size class slab list now only contains slabs + with free list, and not if it only has a bump ptr. [2-4] Are changes that are directly inspired by (mimalloc)[http://github.com/microsoft/mimalloc]. \ No newline at end of file diff --git a/src/ds/address.h b/src/ds/address.h index ab896ee..a92ef36 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -24,6 +24,15 @@ namespace snmalloc return reinterpret_cast(reinterpret_cast(base) + diff); } + /** + * Perform pointer arithmetic and return the adjusted pointer. + */ + template + inline T* pointer_offset_signed(T* base, ptrdiff_t diff) + { + return reinterpret_cast(reinterpret_cast(base) + diff); + } + /** * Cast from a pointer type to an address. */ @@ -125,4 +134,15 @@ namespace snmalloc return static_cast( static_cast(cursor) - static_cast(base)); } + + /** + * Compute the difference in pointers in units of char. This can be used + * across allocations. + */ + inline ptrdiff_t pointer_diff_signed(void* base, void* cursor) + { + return static_cast( + static_cast(cursor) - static_cast(base)); + } + } // namespace snmalloc diff --git a/src/ds/bits.h b/src/ds/bits.h index 12c9bdc..50967fd 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -329,7 +329,7 @@ namespace snmalloc * * `std::min` is in ``, so pulls in a lot of unneccessary code * We write our own to reduce the code that potentially needs reviewing. - **/ + */ template constexpr inline T min(T t1, T t2) { @@ -341,7 +341,7 @@ namespace snmalloc * * `std::max` is in ``, so pulls in a lot of unneccessary code * We write our own to reduce the code that potentially needs reviewing. - **/ + */ template constexpr inline T max(T t1, T t2) { diff --git a/src/ds/cdllist.h b/src/ds/cdllist.h new file mode 100644 index 0000000..1bc39aa --- /dev/null +++ b/src/ds/cdllist.h @@ -0,0 +1,122 @@ +#pragma once + +#include "defines.h" + +#include +#include + +namespace snmalloc +{ + /** + * Special class for cyclic doubly linked non-empty linked list + * + * This code assumes there is always one element in the list. The client + * must ensure there is a sentinal element. + */ + class CDLLNode + { + /** + * to_next is used to handle a zero initialised data structure. + * This means that `is_empty` works even when the constructor hasn't + * been run. + */ + ptrdiff_t to_next = 0; + + // TODO: CHERI will need a real pointer too + // CDLLNode* next = nullptr; + CDLLNode* prev = nullptr; + + void set_next(CDLLNode* c) + { + // TODO: CHERI will need a real pointer too + // next = c; + to_next = pointer_diff_signed(this, c); + } + + public: + /** + * Single element cyclic list. This is the empty case. + */ + CDLLNode() + { + set_next(this); + prev = this; + } + + SNMALLOC_FAST_PATH bool is_empty() + { + return to_next == 0; + } + + /** + * Removes this element from the cyclic list is it part of. + */ + SNMALLOC_FAST_PATH void remove() + { + SNMALLOC_ASSERT(!is_empty()); + debug_check(); + get_next()->prev = prev; + prev->set_next(get_next()); + // As this is no longer in the list, check invariant for + // neighbouring element. + get_next()->debug_check(); + +#ifndef NDEBUG + set_next(nullptr); + prev = nullptr; +#endif + } + + SNMALLOC_FAST_PATH CDLLNode* get_next() + { + // TODO: CHERI will require a real pointer + // return next; + return pointer_offset_signed(this, to_next); + } + + SNMALLOC_FAST_PATH CDLLNode* get_prev() + { + return prev; + } + + SNMALLOC_FAST_PATH void insert_next(CDLLNode* item) + { + debug_check(); + item->set_next(get_next()); + get_next()->prev = item; + item->prev = this; + set_next(item); + debug_check(); + } + + SNMALLOC_FAST_PATH void insert_prev(CDLLNode* item) + { + debug_check(); + item->prev = prev; + prev->set_next(item); + item->set_next(this); + prev = item; + debug_check(); + } + + /** + * Checks the lists invariants + * x->next->prev = x + * for all x in the list. + */ + void debug_check() + { +#ifndef NDEBUG + CDLLNode* item = get_next(); + CDLLNode* p = this; + + do + { + SNMALLOC_ASSERT(item->prev == p); + p = item; + item = item->get_next(); + } while (item != this); +#endif + } + }; +} // namespace snmalloc diff --git a/src/ds/dllist.h b/src/ds/dllist.h index e4e70d2..4a7e9c7 100644 --- a/src/ds/dllist.h +++ b/src/ds/dllist.h @@ -94,12 +94,12 @@ namespace snmalloc return *this; } - bool is_empty() + SNMALLOC_FAST_PATH bool is_empty() { return head == Terminator(); } - T* get_head() + SNMALLOC_FAST_PATH T* get_head() { return head; } @@ -109,7 +109,7 @@ namespace snmalloc return tail; } - T* pop() + SNMALLOC_FAST_PATH T* pop() { T* item = head; @@ -169,7 +169,7 @@ namespace snmalloc #endif } - void remove(T* item) + SNMALLOC_FAST_PATH void remove(T* item) { #ifndef NDEBUG debug_check_contains(item); diff --git a/src/ds/helpers.h b/src/ds/helpers.h index 5434c33..e3c2290 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -48,7 +48,7 @@ namespace snmalloc * * Wraps on read. This allows code to trust the value is in range, even when * there is a memory corruption. - **/ + */ template class Mod { diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 87b9e81..afe5a10 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -49,14 +49,20 @@ namespace snmalloc FastFreeLists() : small_fast_free_lists() {} }; - SNMALLOC_FAST_PATH void* no_replacement(void*) - { - return nullptr; - } - /** - * Allocator. This class is parameterised on three template parameters. The - * `MemoryProvider` defines the source of memory for this allocator. + * Allocator. This class is parameterised on five template parameters. + * + * The first two template parameter provides a hook to allow the allocator in + * use to be dynamically modified. This is used to implement a trick from + * mimalloc that avoids a conditional branch on the fast path. We + * initialise the thread-local allocator pointer with the address of a global + * allocator, which never owns any memory. The first returns true, if is + * passed the global allocator. The second initialises the thread-local + * allocator if it is has been been initialised already. Splitting into two + * functions allows for the code to be structured into tail calls to improve + * codegen. + * + * The `MemoryProvider` defines the source of memory for this allocator. * Allocators try to reuse address space by allocating from existing slabs or * reusing freed large allocations. When they need to allocate a new chunk * of memory they request space from the `MemoryProvider`. @@ -65,31 +71,35 @@ namespace snmalloc * to associate metadata with large (16MiB, by default) regions, allowing an * allocator to find the allocator responsible for that region. * - * The next template parameter, `IsQueueInline`, defines whether the + * The final template parameter, `IsQueueInline`, defines whether the * message queue for this allocator should be stored as a field of the * allocator (`true`) or provided externally, allowing it to be anywhere else * in the address space (`false`). - * - * The final template parameter provides a hook to allow the allocator in use - * to be dynamically modified. This is used to implement a trick from - * mimalloc that avoids a conditional branch on the fast path. We initialise - * the thread-local allocator pointer with the address of a global allocator, - * which never owns any memory. When we try to allocate memory, we call the - * replacement function. */ template< + bool (*NeedsInitialisation)(void*), + void* (*InitThreadAllocator)(), class MemoryProvider = GlobalVirtual, class ChunkMap = SNMALLOC_DEFAULT_CHUNKMAP, - bool IsQueueInline = true, - void* (*Replacement)(void*) = no_replacement> - class Allocator - : public FastFreeLists, - public Pooled< - Allocator> + bool IsQueueInline = true> + class Allocator : public FastFreeLists, + public Pooled> { LargeAlloc large_allocator; ChunkMap chunk_map; + /** + * Per size class bumpptr for building new free lists + * If aligned to a SLAB start, then it is empty, and a new + * slab is required. + */ + void* bump_ptrs[NUM_SMALL_CLASSES] = {nullptr}; + public: Stats& stats() { @@ -155,8 +165,6 @@ namespace snmalloc else return calloc(1, size); #else - stats().alloc_request(size); - // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1))) @@ -202,17 +210,14 @@ namespace snmalloc UNUSED(size); return free(p); #else - constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size); - handle_message_queue(); - if (sizeclass < NUM_SMALL_CLASSES) { Superslab* super = Superslab::get(p); RemoteAllocator* target = super->get_allocator(); - if (target == public_state()) + if (likely(target == public_state())) small_dealloc(super, p, sizeclass); else remote_dealloc(target, p, sizeclass); @@ -222,7 +227,7 @@ namespace snmalloc Mediumslab* slab = Mediumslab::get(p); RemoteAllocator* target = slab->get_allocator(); - if (target == public_state()) + if (likely(target == public_state())) medium_dealloc(slab, p, sizeclass); else remote_dealloc(target, p, sizeclass); @@ -262,7 +267,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH void dealloc_sized_slow(void* p, size_t size) { if (size == 0) - dealloc(p, 1); + return dealloc(p, 1); if (likely(size <= sizeclass_to_size(NUM_SIZECLASSES - 1))) { @@ -505,8 +510,12 @@ namespace snmalloc /// r is used for which round of sending this is. inline size_t get_slot(size_t id, size_t r) { - constexpr size_t allocator_size = sizeof( - Allocator); + constexpr size_t allocator_size = sizeof(Allocator< + NeedsInitialisation, + InitThreadAllocator, + MemoryProvider, + ChunkMap, + IsQueueInline>); constexpr size_t initial_shift = bits::next_pow2_bits_const(allocator_size); SNMALLOC_ASSERT((initial_shift + (r * REMOTE_SLOT_BITS)) < 64); @@ -703,7 +712,7 @@ namespace snmalloc * * If result pointer is null, then this code raises a Pal::error on the * particular check that fails, if any do fail. - **/ + */ void debug_is_empty(bool* result) { auto test = [&result](auto& queue) { @@ -728,6 +737,26 @@ namespace snmalloc } } + // Dump bump allocators back into memory + for (size_t i = 0; i < NUM_SMALL_CLASSES; i++) + { + auto& bp = bump_ptrs[i]; + auto rsize = sizeclass_to_size(i); + FreeListHead ffl; + while (pointer_align_up(bp, SLAB_SIZE) != bp) + { + Slab::alloc_new_list(bp, ffl, rsize); + void* prev = ffl.value; + while (prev != nullptr) + { + auto n = Metaslab::follow_next(prev); + Superslab* super = Superslab::get(prev); + small_dealloc_offseted_inner(super, prev, i); + prev = n; + } + } + } + for (size_t i = 0; i < NUM_SMALL_CLASSES; i++) { auto prev = small_fast_free_lists[i].value; @@ -856,10 +885,19 @@ namespace snmalloc remote.post(id()); } + /** + * Check if this allocator has messages to deallocate blocks from another + * thread + */ + SNMALLOC_FAST_PATH bool has_messages() + { + return !(message_queue().is_empty()); + } + SNMALLOC_FAST_PATH void handle_message_queue() { // Inline the empty check, but not necessarily the full queue handling. - if (likely(message_queue().is_empty())) + if (likely(!has_messages())) return; handle_message_queue_inner(); @@ -921,7 +959,7 @@ namespace snmalloc } template - Slab* alloc_slab(sizeclass_t sizeclass) + SNMALLOC_SLOW_PATH Slab* alloc_slab(sizeclass_t sizeclass) { stats().sizeclass_alloc_slab(sizeclass); if (Superslab::is_short_sizeclass(sizeclass)) @@ -970,17 +1008,19 @@ namespace snmalloc SNMALLOC_ASSUME(size <= SLAB_SIZE); sizeclass_t sizeclass = size_to_sizeclass(size); - return small_alloc_inner(sizeclass); + return small_alloc_inner(sizeclass, size); } template - SNMALLOC_FAST_PATH void* small_alloc_inner(sizeclass_t sizeclass) + SNMALLOC_FAST_PATH void* + small_alloc_inner(sizeclass_t sizeclass, size_t size) { SNMALLOC_ASSUME(sizeclass < NUM_SMALL_CLASSES); auto& fl = small_fast_free_lists[sizeclass]; void* head = fl.value; if (likely(head != nullptr)) { + stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); // Read the next slot from the memory that's about to be allocated. fl.value = Metaslab::follow_next(head); @@ -993,46 +1033,140 @@ namespace snmalloc return p; } - return small_alloc_slow(sizeclass); + if (likely(!has_messages())) + return small_alloc_next_free_list( + sizeclass, size); + + return small_alloc_mq_slow(sizeclass, size); } + /** + * Slow path for handling message queue, before dealing with small + * allocation request. + */ template - SNMALLOC_SLOW_PATH void* small_alloc_slow(sizeclass_t sizeclass) + SNMALLOC_SLOW_PATH void* + small_alloc_mq_slow(sizeclass_t sizeclass, size_t size) { - if (void* replacement = Replacement(this)) - { - return reinterpret_cast(replacement) - ->template small_alloc_inner(sizeclass); - } + handle_message_queue_inner(); - stats().sizeclass_alloc(sizeclass); + return small_alloc_next_free_list( + sizeclass, size); + } - handle_message_queue(); + /** + * Attempt to find a new free list to allocate from + */ + template + SNMALLOC_SLOW_PATH void* + small_alloc_next_free_list(sizeclass_t sizeclass, size_t size) + { size_t rsize = sizeclass_to_size(sizeclass); auto& sl = small_classes[sizeclass]; Slab* slab; - if (!sl.is_empty()) + if (likely(!sl.is_empty())) { - SlabLink* link = sl.get_head(); - slab = link->get_slab(); + stats().alloc_request(size); + stats().sizeclass_alloc(sizeclass); + + SlabLink* link = sl.get_next(); + slab = get_slab(link); + auto& ffl = small_fast_free_lists[sizeclass]; + return slab->alloc( + sl, ffl, rsize, large_allocator.memory_provider); } - else + return small_alloc_rare(sizeclass, size); + } + + /** + * Called when there are no available free list to service this request + * Could be due to using the dummy allocator, or needing to bump allocate a + * new free list. + */ + template + SNMALLOC_SLOW_PATH void* + small_alloc_rare(sizeclass_t sizeclass, size_t size) + { + if (likely(!NeedsInitialisation(this))) { - slab = alloc_slab(sizeclass); - - if ((allow_reserve == NoReserve) && (slab == nullptr)) - return nullptr; - - if (slab == nullptr) - return nullptr; - - sl.insert_back(slab->get_link()); + stats().alloc_request(size); + stats().sizeclass_alloc(sizeclass); + return small_alloc_new_free_list(sizeclass); } + return small_alloc_first_alloc(sizeclass, size); + } + + /** + * Called on first allocation to set up the thread local allocator, + * then directs the allocation request to the newly created allocator. + */ + template + SNMALLOC_SLOW_PATH void* + small_alloc_first_alloc(sizeclass_t sizeclass, size_t size) + { + auto replacement = InitThreadAllocator(); + return reinterpret_cast(replacement) + ->template small_alloc_inner(sizeclass, size); + } + + /** + * Called to create a new free list, and service the request from that new + * list. + */ + template + SNMALLOC_FAST_PATH void* small_alloc_new_free_list(sizeclass_t sizeclass) + { + auto& bp = bump_ptrs[sizeclass]; + if (likely(pointer_align_up(bp, SLAB_SIZE) != bp)) + { + return small_alloc_build_free_list(sizeclass); + } + // Fetch new slab + return small_alloc_new_slab(sizeclass); + } + + /** + * Creates a new free list from the thread local bump allocator and service + * the request from that new list. + */ + template + SNMALLOC_FAST_PATH void* small_alloc_build_free_list(sizeclass_t sizeclass) + { + auto& bp = bump_ptrs[sizeclass]; + auto rsize = sizeclass_to_size(sizeclass); auto& ffl = small_fast_free_lists[sizeclass]; - return slab->alloc( - sl, ffl, rsize, large_allocator.memory_provider); + SNMALLOC_ASSERT(ffl.value == nullptr); + Slab::alloc_new_list(bp, ffl, rsize); + + void* p = remove_cache_friendly_offset(ffl.value, sizeclass); + ffl.value = Metaslab::follow_next(p); + + if constexpr (zero_mem == YesZero) + { + large_allocator.memory_provider.zero(p, sizeclass_to_size(sizeclass)); + } + return p; + } + + /** + * Allocates a new slab to allocate from, set it to be the bump allocator + * for this size class, and then builds a new free list from the thread + * local bump allocator and service the request from that new list. + */ + template + SNMALLOC_SLOW_PATH void* small_alloc_new_slab(sizeclass_t sizeclass) + { + auto& bp = bump_ptrs[sizeclass]; + // Fetch new slab + Slab* slab = alloc_slab(sizeclass); + if (slab == nullptr) + return nullptr; + bp = + pointer_offset(slab, get_initial_offset(sizeclass, slab->is_short())); + + return small_alloc_build_free_list(sizeclass); } SNMALLOC_FAST_PATH void @@ -1149,8 +1283,9 @@ namespace snmalloc } else { - if (void* replacement = Replacement(this)) + if (NeedsInitialisation(this)) { + void* replacement = InitThreadAllocator(); return reinterpret_cast(replacement) ->template medium_alloc( sizeclass, rsize, size); @@ -1170,6 +1305,7 @@ namespace snmalloc sc->insert(slab); } + stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); return p; } @@ -1221,8 +1357,9 @@ namespace snmalloc zero_mem == YesZero ? "zeromem" : "nozeromem", allow_reserve == NoReserve ? "noreserve" : "reserve")); - if (void* replacement = Replacement(this)) + if (NeedsInitialisation(this)) { + void* replacement = InitThreadAllocator(); return reinterpret_cast(replacement) ->template large_alloc(size); } @@ -1237,6 +1374,7 @@ namespace snmalloc { chunkmap().set_large_size(p, size); + stats().alloc_request(size); stats().large_alloc(large_class); } return p; @@ -1246,6 +1384,13 @@ namespace snmalloc { MEASURE_TIME(large_dealloc, 4, 16); + if (NeedsInitialisation(this)) + { + void* replacement = InitThreadAllocator(); + return reinterpret_cast(replacement) + ->large_dealloc(p, size); + } + size_t size_bits = bits::next_pow2_bits(size); SNMALLOC_ASSERT(bits::one_at_bit(size_bits) >= SUPERSLAB_SIZE); size_t large_class = size_bits - SUPERSLAB_BITS; @@ -1260,11 +1405,10 @@ namespace snmalloc large_allocator.dealloc(slab, large_class); } - // Note that this is on the slow path as it lead to better code. - // As it is tail, not inlining means that it is jumped to, so has no perf - // impact on the producer consumer scenarios, and doesn't require register - // spills in the fast path for local deallocation. - SNMALLOC_SLOW_PATH + // This is still considered the fast path as all the complex code is tail + // called in its slow path. This leads to one fewer unconditional jump in + // Clang. + SNMALLOC_FAST_PATH void remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) { MEASURE_TIME(remote_dealloc, 4, 16); @@ -1293,8 +1437,9 @@ namespace snmalloc // Now that we've established that we're in the slow path (if we're a // real allocator, we will have to empty our cache now), check if we are // a real allocator and construct one if we aren't. - if (void* replacement = Replacement(this)) + if (NeedsInitialisation(this)) { + void* replacement = InitThreadAllocator(); // We have to do a dealloc, not a remote_dealloc here because this may // have been allocated with the allocator that we've just had returned. reinterpret_cast(replacement)->dealloc(p); diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h index 444fc1e..1f6a4ac 100644 --- a/src/mem/allocstats.h +++ b/src/mem/allocstats.h @@ -125,7 +125,9 @@ namespace snmalloc bits::one_at_bit(bits::ADDRESS_BITS - 1)); Stats sizeclass[N]; - Stats large[LARGE_N]; + + size_t large_pop_count[LARGE_N] = {0}; + size_t large_push_count[LARGE_N] = {0}; size_t remote_freed = 0; size_t remote_posted = 0; @@ -159,7 +161,7 @@ namespace snmalloc for (size_t i = 0; i < LARGE_N; i++) { - if (!large[i].is_empty()) + if (large_push_count[i] != large_pop_count[i]) return false; } @@ -194,7 +196,7 @@ namespace snmalloc UNUSED(sc); #ifdef USE_SNMALLOC_STATS - large[sc].count.inc(); + large_pop_count[sc]++; #endif } @@ -223,7 +225,7 @@ namespace snmalloc UNUSED(sc); #ifdef USE_SNMALLOC_STATS - large[sc].count.dec(); + large_push_count[sc]++; #endif } @@ -289,7 +291,10 @@ namespace snmalloc sizeclass[i].add(that.sizeclass[i]); for (size_t i = 0; i < LARGE_N; i++) - large[i].add(that.large[i]); + { + large_push_count[i] += that.large_push_count[i]; + large_pop_count[i] += that.large_pop_count[i]; + } for (size_t i = 0; i < TOTAL_BUCKETS; i++) bucketed_requests[i] += that.bucketed_requests[i]; @@ -343,6 +348,14 @@ namespace snmalloc << "Average Slab Usage" << "Average wasted space" << csv.endl; + csv << "LargeBucketedStats" + << "DumpID" + << "AllocatorID" + << "Size group" + << "Size" + << "Push count" + << "Pop count" << csv.endl; + csv << "AllocSizes" << "DumpID" << "AllocatorID" @@ -367,13 +380,12 @@ namespace snmalloc for (uint8_t i = 0; i < LARGE_N; i++) { - if (large[i].count.is_unused()) + if ((large_push_count[i] == 0) && (large_pop_count[i] == 0)) continue; - csv << "BucketedStats" << dumpid << allocatorid << (i + N) - << large_sizeclass_to_size(i); - - large[i].print(csv, large_sizeclass_to_size(i)); + csv << "LargeBucketedStats" << dumpid << allocatorid << (i + N) + << large_sizeclass_to_size(i) << large_push_count[i] + << large_pop_count[i] << csv.endl; } size_t low = 0; diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index c790026..7c33ed7 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -6,24 +6,32 @@ namespace snmalloc { - inline void* lazy_replacement(void*); - using Alloc = - Allocator; + inline bool needs_initialisation(void*); + void* init_thread_allocator(); + + using Alloc = Allocator< + needs_initialisation, + init_thread_allocator, + GlobalVirtual, + SNMALLOC_DEFAULT_CHUNKMAP, + true>; template class AllocPool : Pool< Allocator< + needs_initialisation, + init_thread_allocator, MemoryProvider, SNMALLOC_DEFAULT_CHUNKMAP, - true, - lazy_replacement>, + true>, MemoryProvider> { using Alloc = Allocator< + needs_initialisation, + init_thread_allocator, MemoryProvider, SNMALLOC_DEFAULT_CHUNKMAP, - true, - lazy_replacement>; + true>; using Parent = Pool; public: diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index ed0fd36..916eaed 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -60,23 +60,23 @@ namespace snmalloc { /** * Flag to protect the bump allocator - **/ + */ std::atomic_flag lock = ATOMIC_FLAG_INIT; /** * Pointer to block being bump allocated - **/ + */ void* bump = nullptr; /** * Space remaining in this block being bump allocated - **/ + */ size_t remaining = 0; /** * Simple flag for checking if another instance of lazy-decommit is * running - **/ + */ std::atomic_flag lazy_decommit_guard = {}; public: @@ -87,7 +87,7 @@ namespace snmalloc /** * Make a new memory provide for this PAL. - **/ + */ static MemoryProviderStateMixin* make() noexcept { // Temporary stack-based storage to start the allocator in. @@ -203,7 +203,7 @@ namespace snmalloc /*** * Method for callback object to perform lazy decommit. - **/ + */ static void process(PalNotificationObject* p) { // Unsafe downcast here. Don't want vtable and RTTI. diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 837c265..4d53b77 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -1,5 +1,6 @@ #pragma once +#include "../ds/cdllist.h" #include "../ds/dllist.h" #include "../ds/helpers.h" #include "sizeclass.h" @@ -8,18 +9,13 @@ namespace snmalloc { class Slab; - struct SlabLink + using SlabList = CDLLNode; + using SlabLink = CDLLNode; + + SNMALLOC_FAST_PATH Slab* get_slab(SlabLink* sl) { - SlabLink* prev; - SlabLink* next; - - Slab* get_slab() - { - return pointer_align_down(this); - } - }; - - using SlabList = DLList; + return pointer_align_down(sl); + } static_assert( sizeof(SlabLink) <= MIN_ALLOC_SIZE, @@ -68,7 +64,7 @@ namespace snmalloc * - empty adding the entry to the free list, or * - was full before the subtraction * this returns true, otherwise returns false. - **/ + */ bool return_object() { return (--needed) == 0; @@ -86,7 +82,7 @@ namespace snmalloc return result; } - void set_full() + SNMALLOC_FAST_PATH void set_full() { SNMALLOC_ASSERT(head == nullptr); SNMALLOC_ASSERT(link != 1); @@ -161,7 +157,7 @@ namespace snmalloc * https://en.wikipedia.org/wiki/Cycle_detection#Floyd's_Tortoise_and_Hare * We don't expect a cycle, so worst case is only followed by a crash, so * slow doesn't mater. - **/ + */ size_t debug_slab_acyclic_free_list(Slab* slab) { #ifndef NDEBUG diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index 6d2c944..fc3da85 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -319,7 +319,7 @@ namespace snmalloc /** * Simple pagemap that for each GRANULARITY_BITS of the address range * stores a T. - **/ + */ template class alignas(OS_PAGE_SIZE) FlatPagemap { diff --git a/src/mem/pool.h b/src/mem/pool.h index 2736c56..366e687 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -15,7 +15,7 @@ namespace snmalloc * concurrency safe. * * This is used to bootstrap the allocation of allocators. - **/ + */ template class Pool { diff --git a/src/mem/slab.h b/src/mem/slab.h index 3274b05..548d9ff 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -31,8 +31,13 @@ namespace snmalloc return get_meta().get_link(this); } + /** + * Takes a free list out of a slabs meta data. + * Returns the link as the allocation, and places the free list into the + * `fast_free_list` for further allocations. + */ template - inline void* alloc( + SNMALLOC_FAST_PATH void* alloc( SlabList& sl, FreeListHead& fast_free_list, size_t rsize, @@ -40,90 +45,26 @@ namespace snmalloc { // Read the head from the metadata stored in the superslab. Metaslab& meta = get_meta(); - void* head = meta.head; + SNMALLOC_ASSERT(meta.link != 1); SNMALLOC_ASSERT(rsize == sizeclass_to_size(meta.sizeclass)); SNMALLOC_ASSERT( - sl.get_head() == (SlabLink*)pointer_offset(this, meta.link)); + sl.get_next() == (SlabLink*)pointer_offset(this, meta.link)); SNMALLOC_ASSERT(!meta.is_full()); meta.debug_slab_invariant(this); - void* p = nullptr; - bool p_has_value = false; + // Put everything in allocators small_class free list. + fast_free_list.value = meta.head; + meta.head = nullptr; - if (head == nullptr) - { - size_t bumpptr = get_initial_offset(meta.sizeclass, is_short()); - bumpptr += meta.allocated * rsize; - if (bumpptr == SLAB_SIZE) - { - // Everything is in use, so we need all entries to be - // return before we can reclaim this slab. - meta.needed = meta.allocated; + // Return the link as the node for this allocation. + void* link = pointer_offset(this, meta.link); + void* p = remove_cache_friendly_offset(link, meta.sizeclass); - void* link = pointer_offset(this, meta.link); - p = remove_cache_friendly_offset(link, meta.sizeclass); - - meta.set_full(); - sl.pop(); - p_has_value = true; - } - else - { - // Allocate the last object on the current page if there is one, - // and then thread the next free list worth of allocations. - bool crossed_page_boundary = false; - void* curr = nullptr; - while (true) - { - size_t newbumpptr = bumpptr + rsize; - auto alignedbumpptr = bits::align_up(bumpptr - 1, OS_PAGE_SIZE); - auto alignednewbumpptr = bits::align_up(newbumpptr, OS_PAGE_SIZE); - - if (alignedbumpptr != alignednewbumpptr) - { - // We have crossed a page boundary already, so - // lets stop building our free list. - if (crossed_page_boundary) - break; - - crossed_page_boundary = true; - } - - if (curr == nullptr) - { - meta.head = pointer_offset(this, bumpptr); - } - else - { - Metaslab::store_next( - curr, (bumpptr == 1) ? nullptr : pointer_offset(this, bumpptr)); - } - curr = pointer_offset(this, bumpptr); - bumpptr = newbumpptr; - meta.allocated = meta.allocated + 1; - } - - SNMALLOC_ASSERT(curr != nullptr); - Metaslab::store_next(curr, nullptr); - } - } - - if (!p_has_value) - { - p = meta.head; - - // Read the next slot from the memory that's about to be allocated. - void* next = Metaslab::follow_next(p); - // Put everything in allocators small_class free list. - meta.head = nullptr; - fast_free_list.value = next; - // Treat stealing the free list as allocating it all. - // Link is not in use, i.e. - 1 is required. - meta.needed = meta.allocated - 1; - - p = remove_cache_friendly_offset(p, meta.sizeclass); - } + // Treat stealing the free list as allocating it all. + meta.needed = meta.allocated; + meta.set_full(); + sl.get_next()->remove(); SNMALLOC_ASSERT(is_start_of_object(Superslab::get(p), p)); @@ -136,10 +77,61 @@ namespace snmalloc else memory_provider.template zero(p, rsize); } + else + { + UNUSED(rsize); + } return p; } + /** + * Given a bumpptr and a fast_free_list head reference, builds a new free + * list, and stores it in the fast_free_list. It will only create a page + * worth of allocations, or one if the allocation size is larger than a + * page. + */ + static SNMALLOC_FAST_PATH void + alloc_new_list(void*& bumpptr, FreeListHead& fast_free_list, size_t rsize) + { + // Allocate the last object on the current page if there is one, + // and then thread the next free list worth of allocations. + bool crossed_page_boundary = false; + void* curr = nullptr; + while (true) + { + void* newbumpptr = pointer_offset(bumpptr, rsize); + auto alignedbumpptr = + bits::align_up(address_cast(bumpptr) - 1, OS_PAGE_SIZE); + auto alignednewbumpptr = + bits::align_up(address_cast(newbumpptr), OS_PAGE_SIZE); + + if (alignedbumpptr != alignednewbumpptr) + { + // We have crossed a page boundary already, so + // lets stop building our free list. + if (crossed_page_boundary) + break; + + crossed_page_boundary = true; + } + + if (curr == nullptr) + { + fast_free_list.value = bumpptr; + } + else + { + Metaslab::store_next(curr, bumpptr); + } + curr = bumpptr; + bumpptr = newbumpptr; + } + + SNMALLOC_ASSERT(curr != nullptr); + Metaslab::store_next(curr, nullptr); + } + bool is_start_of_object(Superslab* super, void* p) { Metaslab& meta = super->get_meta(this); @@ -204,13 +196,13 @@ namespace snmalloc meta.needed = meta.allocated - 1; // Push on the list of slabs for this sizeclass. - sl->insert_back(meta.get_link(this)); + sl->insert_prev(meta.get_link(this)); meta.debug_slab_invariant(this); return Superslab::NoSlabReturn; } // Remove from the sizeclass list and dealloc on the superslab. - sl->remove(meta.get_link(this)); + meta.get_link(this)->remove(); if (is_short()) return super->dealloc_short_slab(); diff --git a/src/mem/superslab.h b/src/mem/superslab.h index e236534..443e33f 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -160,10 +160,18 @@ namespace snmalloc if ((used & 1) == 1) return alloc_slab(sizeclass); - meta[0].allocated = 1; meta[0].head = nullptr; + // Set up meta data as if the entire slab has been turned into a free + // list. This means we don't have to check for special cases where we have + // returned all the elements, but this is a slab that is still being bump + // allocated from. Hence, the bump allocator slab will never be returned + // for use in another size class. + meta[0].allocated = static_cast( + (SLAB_SIZE - get_initial_offset(sizeclass, true)) / + sizeclass_to_size(sizeclass)); + meta[0].link = 1; + meta[0].needed = 1; meta[0].sizeclass = static_cast(sizeclass); - meta[0].link = get_initial_offset(sizeclass, true); used++; return reinterpret_cast(this); @@ -178,9 +186,17 @@ namespace snmalloc uint8_t n = meta[h].next; meta[h].head = nullptr; - meta[h].allocated = 1; + // Set up meta data as if the entire slab has been turned into a free + // list. This means we don't have to check for special cases where we have + // returned all the elements, but this is a slab that is still being bump + // allocated from. Hence, the bump allocator slab will never be returned + // for use in another size class. + meta[h].allocated = static_cast( + (SLAB_SIZE - get_initial_offset(sizeclass, false)) / + sizeclass_to_size(sizeclass)); + meta[h].needed = 1; + meta[h].link = 1; meta[h].sizeclass = static_cast(sizeclass); - meta[h].link = get_initial_offset(sizeclass, false); head = h + n + 1; used += 2; diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 79f8e4d..969d1dc 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -40,13 +40,23 @@ namespace snmalloc /** * Function passed as a template parameter to `Allocator` to allow lazy - * replacement. In this case we are assuming the underlying external thread - * alloc is performing initialization, so this is not required, and just - * always returns nullptr to specify no new allocator is required. + * replacement. This function returns true, if the allocator passed in + * requires initialisation. As the TLS state is managed externally, + * this will always return false. */ - SNMALLOC_FAST_PATH void* lazy_replacement(void* existing) + SNMALLOC_FAST_PATH bool needs_initialisation(void* existing) { UNUSED(existing); + return false; + } + + /** + * Function passed as a tempalte parameter to `Allocator` to allow lazy + * replacement. There is nothing to initialise in this case, so we expect + * this to never be called. + */ + SNMALLOC_FAST_PATH void* init_thread_allocator() + { return nullptr; } @@ -58,10 +68,21 @@ namespace snmalloc * slabs to allocate from, it will discover that it is the placeholder and * replace itself with the thread-local allocator, allocating one if * required. This avoids a branch on the fast path. + * + * The fake allocator is a zero initialised area of memory of the correct + * size. All data structures used potentially before initialisation must be + * okay with zero init to move to the slow path, that is, zero must signify + * empty. */ - inline GlobalVirtual dummy_memory_provider; - inline Alloc GlobalPlaceHolder( - dummy_memory_provider, SNMALLOC_DEFAULT_CHUNKMAP(), nullptr, true); + inline const char GlobalPlaceHolder[sizeof(Alloc)] = {0}; + inline Alloc* get_GlobalPlaceHolder() + { + // This cast is not legal. Effectively, we want a minimal constructor + // for the global allocator as zero, and then a second constructor for + // the rest. This is UB. + auto a = reinterpret_cast(&GlobalPlaceHolder); + return const_cast(a); + } /** * Common aspects of thread local allocator. Subclasses handle how releasing @@ -69,22 +90,22 @@ namespace snmalloc */ class ThreadAllocCommon { - friend void* lazy_replacement_slow(); + friend void* init_thread_allocator(); protected: static inline void inner_release() { auto& per_thread = get_reference(); - if (per_thread != &GlobalPlaceHolder) + if (per_thread != get_GlobalPlaceHolder()) { current_alloc_pool()->release(per_thread); - per_thread = &GlobalPlaceHolder; + per_thread = get_GlobalPlaceHolder(); } } /** * Default clean up does nothing except print statistics if enabled. - **/ + */ static void register_cleanup() { # ifdef USE_SNMALLOC_STATS @@ -113,7 +134,7 @@ namespace snmalloc */ static inline Alloc*& get_reference() { - static thread_local Alloc* alloc = &GlobalPlaceHolder; + static thread_local Alloc* alloc = get_GlobalPlaceHolder(); return alloc; } @@ -147,10 +168,11 @@ namespace snmalloc return get_reference(); # else auto alloc = get_reference(); - auto new_alloc = lazy_replacement(alloc); - return (likely(new_alloc == nullptr)) ? - alloc : - reinterpret_cast(new_alloc); + if (unlikely(needs_initialisation(alloc))) + { + alloc = reinterpret_cast(init_thread_allocator()); + } + return alloc; # endif } }; @@ -215,35 +237,38 @@ namespace snmalloc # endif /** - * Slow path for the placeholder replacement. The simple check that this is - * the global placeholder is inlined, the rest of it is only hit in a very - * unusual case and so should go off the fast path. + * Slow path for the placeholder replacement. + * Function passed as a tempalte parameter to `Allocator` to allow lazy + * replacement. This function initialises the thread local state if requried. + * The simple check that this is the global placeholder is inlined, the rest + * of it is only hit in a very unusual case and so should go off the fast + * path. */ - SNMALLOC_SLOW_PATH inline void* lazy_replacement_slow() + SNMALLOC_SLOW_PATH inline void* init_thread_allocator() { auto*& local_alloc = ThreadAlloc::get_reference(); - SNMALLOC_ASSERT(local_alloc == &GlobalPlaceHolder); + if (local_alloc != get_GlobalPlaceHolder()) + { + // If someone reuses a noncachable call, then we can end up here. + // The allocator has already been initialised. Could either error + // to say stop doing this, or just give them the initialised version. + return local_alloc; + } local_alloc = current_alloc_pool()->acquire(); - SNMALLOC_ASSERT(local_alloc != &GlobalPlaceHolder); + SNMALLOC_ASSERT(local_alloc != get_GlobalPlaceHolder()); ThreadAlloc::register_cleanup(); return local_alloc; } /** * Function passed as a template parameter to `Allocator` to allow lazy - * replacement. This is called on all of the slow paths in `Allocator`. If - * the caller is the global placeholder allocator then this function will - * check if we've already allocated a per-thread allocator, returning it if - * so. If we have not allocated a per-thread allocator yet, then this - * function will allocate one. + * replacement. This function returns true, if the allocated passed in, + * is the placeholder allocator. If it returns true, then + * `init_thread_allocator` should be called. */ - SNMALLOC_FAST_PATH void* lazy_replacement(void* existing) + SNMALLOC_FAST_PATH bool needs_initialisation(void* existing) { - if (existing != &GlobalPlaceHolder) - { - return nullptr; - } - return lazy_replacement_slow(); + return existing == get_GlobalPlaceHolder(); } #endif } // namespace snmalloc diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index a86feb5..538f90d 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -62,7 +62,7 @@ namespace snmalloc * This struct is used to represent callbacks for notification from the * platform. It contains a next pointer as client is responsible for * allocation as we cannot assume an allocator at this point. - **/ + */ struct PalNotificationObject { std::atomic pal_next; @@ -72,12 +72,12 @@ namespace snmalloc /*** * Wrapper for managing notifications for PAL events - **/ + */ class PalNotifier { /** * List of callbacks to notify - **/ + */ std::atomic callbacks = nullptr; public: @@ -86,7 +86,7 @@ namespace snmalloc * * The object should never be deallocated by the client after calling * this. - **/ + */ void register_notification(PalNotificationObject* callback) { callback->pal_next = nullptr; @@ -105,7 +105,7 @@ namespace snmalloc /** * Calls the pal_notify of all the registered objects. - **/ + */ void notify_all() { PalNotificationObject* curr = callbacks; diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 8c1fc27..ba7fcea 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -34,7 +34,7 @@ namespace snmalloc /** * List of callbacks for low-memory notification - **/ + */ static inline PalNotifier low_memory_callbacks; /** @@ -98,7 +98,7 @@ namespace snmalloc * Register callback object for low-memory notifications. * Client is responsible for allocation, and ensuring the object is live * for the duration of the program. - **/ + */ static void register_for_low_memory_callback(PalNotificationObject* callback) { diff --git a/src/test/func/first_operation/first_operation.cc b/src/test/func/first_operation/first_operation.cc new file mode 100644 index 0000000..6af9954 --- /dev/null +++ b/src/test/func/first_operation/first_operation.cc @@ -0,0 +1,112 @@ +/** + * The first operation a thread performs takes a different path to every + * subsequent operation as it must lazily initialise the thread local allocator. + * This tests performs all sizes of allocation, and deallocation as the first + * operation. + */ + +#include "test/setup.h" + +#include +#include + +void alloc1(size_t size) +{ + void* r = snmalloc::ThreadAlloc::get_noncachable()->alloc(size); + snmalloc::ThreadAlloc::get_noncachable()->dealloc(r); +} + +void alloc2(size_t size) +{ + auto a = snmalloc::ThreadAlloc::get_noncachable(); + void* r = a->alloc(size); + a->dealloc(r); +} + +void alloc3(size_t size) +{ + auto a = snmalloc::ThreadAlloc::get_noncachable(); + void* r = a->alloc(size); + a->dealloc(r, size); +} + +void alloc4(size_t size) +{ + auto a = snmalloc::ThreadAlloc::get(); + void* r = a->alloc(size); + a->dealloc(r); +} + +void dealloc1(void* p, size_t) +{ + snmalloc::ThreadAlloc::get_noncachable()->dealloc(p); +} + +void dealloc2(void* p, size_t size) +{ + snmalloc::ThreadAlloc::get_noncachable()->dealloc(p, size); +} + +void dealloc3(void* p, size_t) +{ + snmalloc::ThreadAlloc::get()->dealloc(p); +} + +void dealloc4(void* p, size_t size) +{ + snmalloc::ThreadAlloc::get()->dealloc(p, size); +} + +void f(size_t size) +{ + auto t1 = std::thread(alloc1, size); + auto t2 = std::thread(alloc2, size); + auto t3 = std::thread(alloc3, size); + auto t4 = std::thread(alloc4, size); + + auto a = snmalloc::ThreadAlloc::get(); + auto p1 = a->alloc(size); + auto p2 = a->alloc(size); + auto p3 = a->alloc(size); + auto p4 = a->alloc(size); + + auto t5 = std::thread(dealloc1, p1, size); + auto t6 = std::thread(dealloc2, p2, size); + auto t7 = std::thread(dealloc3, p3, size); + auto t8 = std::thread(dealloc4, p4, size); + + t1.join(); + t2.join(); + t3.join(); + t4.join(); + t5.join(); + t6.join(); + t7.join(); + t8.join(); +} + +int main(int, char**) +{ + setup(); + + f(0); + f(1); + f(3); + f(5); + f(7); + for (size_t exp = 1; exp < snmalloc::SUPERSLAB_BITS; exp++) + { + f(1ULL << exp); + f(3ULL << exp); + f(5ULL << exp); + f(7ULL << exp); + f((1ULL << exp) + 1); + f((3ULL << exp) + 1); + f((5ULL << exp) + 1); + f((7ULL << exp) + 1); + f((1ULL << exp) - 1); + f((3ULL << exp) - 1); + f((5ULL << exp) - 1); + f((7ULL << exp) - 1); + } +} diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index 046a2cc..68007b4 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -97,7 +97,7 @@ void reduce_pressure(Queue& allocations) * Wrapper to handle Pals that don't have the method. * Template parameter required to handle `if constexpr` always evaluating both * sides. - **/ + */ template void register_for_pal_notifications() {