From bc365e0abb9c798f3bab3ede3d4d11e32039c713 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Fri, 8 Oct 2021 13:57:48 +0100 Subject: [PATCH] Default template args for FreeObject & friends Now that explicit annotations have gotten us through the refactoring, it's time for the scaffolding to disappear. src/mem/freelist.h is left generic for any future machinations, but `FreeObject::T<>`, the several `FreeObject::...Ptr<>`s, `FreeListIter<>`, and `FreeListBuilder<>` are given default parameters and all uses are shortened to use defaults where possible. --- src/mem/corealloc.h | 73 ++++++++++---------------- src/mem/freelist.h | 105 +++++++++++++++++++++----------------- src/mem/localalloc.h | 57 ++++++++++----------- src/mem/localcache.h | 24 ++++----- src/mem/metaslab.h | 13 ++--- src/mem/remoteallocator.h | 37 +++++--------- src/mem/remotecache.h | 21 +++----- 7 files changed, 143 insertions(+), 187 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 8d84c91..611827c 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -161,9 +161,8 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - auto dummy = - capptr::Alloc(small_alloc_one(MIN_ALLOC_SIZE)) - .template as_static>(); + auto dummy = capptr::Alloc(small_alloc_one(MIN_ALLOC_SIZE)) + .template as_static>(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -181,18 +180,12 @@ namespace snmalloc { SNMALLOC_ASSERT(attached_cache != nullptr); auto domesticate = - [this](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate( - backend_state_ptr(), p); - }; + [this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(backend_state_ptr(), p); + }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( - domesticate, - size, - [&]( - sizeclass_t sizeclass, - FreeListIter* fl) { + domesticate, size, [&](sizeclass_t sizeclass, FreeListIter<>* fl) { return small_alloc(sizeclass, *fl); }); } @@ -284,15 +277,14 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter fl; + FreeListIter<> fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); #ifdef SNMALLOC_CHECK_CLIENT @@ -423,10 +415,9 @@ namespace snmalloc bool need_post = false; auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; for (size_t i = 0; i < REMOTE_BATCH; i++) { auto p = message_queue().peek(); @@ -617,7 +608,7 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = p.as_static>(); + auto cp = p.as_static>(); auto& key = entropy.get_free_list_key(); @@ -628,10 +619,8 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH void* small_alloc( - sizeclass_t sizeclass, - FreeListIter& - fast_free_list) + SNMALLOC_SLOW_PATH void* + small_alloc(sizeclass_t sizeclass, FreeListIter<>& fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -655,12 +644,10 @@ namespace snmalloc if (meta->needed() == 0) alloc_classes[sizeclass].unused--; - auto domesticate = - [this](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate( - backend_state_ptr(), p); - }; + auto domesticate = [this]( + FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(backend_state_ptr(), p); + }; auto [p, still_active] = Metaslab::alloc_free_list( domesticate, meta, fast_free_list, entropy, sizeclass); @@ -695,10 +682,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc_slow( - sizeclass_t sizeclass, - FreeListIter& - fast_free_list, - size_t rsize) + sizeclass_t sizeclass, FreeListIter<>& fast_free_list, size_t rsize) { // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); @@ -729,11 +713,9 @@ namespace snmalloc alloc_new_list(slab, meta, rsize, slab_size, entropy); auto domesticate = - [this](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate( - backend_state_ptr(), p); - }; + [this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(backend_state_ptr(), p); + }; auto [p, still_active] = Metaslab::alloc_free_list( domesticate, meta, fast_free_list, entropy, sizeclass); @@ -756,10 +738,9 @@ namespace snmalloc SNMALLOC_ASSERT(attached_cache != nullptr); auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; if (destroy_queue) { diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 8d6ad07..a002c00 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -55,22 +55,31 @@ namespace snmalloc class FreeObject { public: - template + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> class T; /** * This "inductive step" type -- a queue-annotated pointer to a FreeObject * containing a queue-annotated pointer -- shows up all over the place. - * Give it a shorter name (FreeObject::QueuePtr) for convenience. + * Give it a shorter name (FreeObject::BQueuePtr) for convenience. */ template - using QueuePtr = CapPtr, BQueue>; + using BQueuePtr = CapPtr, BQueue>; /** - * As with QueuePtr, but atomic. + * In particular, external code almost always uses AllocWild as the queue + * annotation. Make their lives easier. + */ + using QueuePtr = BQueuePtr; + + /** + * As with BQueuePtr, but atomic. */ template - using AtomicQueuePtr = AtomicCapPtr, BQueue>; + using BAtomicQueuePtr = AtomicCapPtr, BQueue>; + + using AtomicQueuePtr = BAtomicQueuePtr; /** * This is the "base case" of that induction. While we can't get rid of the @@ -81,7 +90,9 @@ namespace snmalloc template< SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - using HeadPtr = CapPtr, BView>; + using BHeadPtr = CapPtr, BView>; + + using HeadPtr = BHeadPtr; /** * As with HeadPtr, but atomic. @@ -115,9 +126,9 @@ namespace snmalloc union { - QueuePtr next_object; + BQueuePtr next_object; // TODO: Should really use C++20 atomic_ref rather than a union. - AtomicQueuePtr atomic_next_object; + BAtomicQueuePtr atomic_next_object; }; #ifdef SNMALLOC_CHECK_CLIENT // Encoded representation of a back pointer. @@ -131,7 +142,7 @@ namespace snmalloc SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: template with_wildness, typename Domesticator> - HeadPtr + BHeadPtr atomic_read_next(const FreeListKey& key, Domesticator domesticate) { auto n_wild = FreeObject::decode_next( @@ -156,7 +167,7 @@ namespace snmalloc SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: template with_wildness, typename Domesticator> - HeadPtr + BHeadPtr read_next(const FreeListKey& key, Domesticator domesticate) { return domesticate(FreeObject::decode_next( @@ -179,7 +190,7 @@ namespace snmalloc template< SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, SNMALLOC_CONCEPT(capptr::ConceptBound) BView> - static HeadPtr make(CapPtr p) + static BHeadPtr make(CapPtr p) { return p.template as_static>(); } @@ -241,10 +252,10 @@ namespace snmalloc template< SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - inline static QueuePtr encode_next( - address_t curr, HeadPtr next, const FreeListKey& key) + inline static BQueuePtr encode_next( + address_t curr, BHeadPtr next, const FreeListKey& key) { - return QueuePtr(code_next(curr, next.unsafe_ptr(), key)); + return BQueuePtr(code_next(curr, next.unsafe_ptr(), key)); } /** @@ -264,10 +275,10 @@ namespace snmalloc template< SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - inline static HeadPtr decode_next( - address_t curr, HeadPtr next, const FreeListKey& key) + inline static BHeadPtr decode_next( + address_t curr, BHeadPtr next, const FreeListKey& key) { - return HeadPtr(code_next(curr, next.unsafe_ptr(), key)); + return BHeadPtr(code_next(curr, next.unsafe_ptr(), key)); } template< @@ -299,9 +310,9 @@ namespace snmalloc template< SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - static QueuePtr* store_next( - QueuePtr* curr, - HeadPtr next, + static BQueuePtr* store_next( + BQueuePtr* curr, + BHeadPtr next, const FreeListKey& key) { assert_view_queue_bounds(); @@ -317,9 +328,9 @@ namespace snmalloc } template - static void store_null(QueuePtr* curr, const FreeListKey& key) + static void store_null(BQueuePtr* curr, const FreeListKey& key) { - *curr = encode_next(address_cast(curr), QueuePtr(nullptr), key); + *curr = encode_next(address_cast(curr), BQueuePtr(nullptr), key); } /** @@ -331,8 +342,8 @@ namespace snmalloc SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> static void atomic_store_next( - HeadPtr curr, - HeadPtr next, + BHeadPtr curr, + BHeadPtr next, const FreeListKey& key) { static_assert(BView::wildness == capptr::dimension::Wildness::Tame); @@ -354,13 +365,13 @@ namespace snmalloc SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> static void - atomic_store_null(HeadPtr curr, const FreeListKey& key) + atomic_store_null(BHeadPtr curr, const FreeListKey& key) { static_assert(BView::wildness == capptr::dimension::Wildness::Tame); curr->atomic_next_object.store( encode_next( - address_cast(&curr->next_object), QueuePtr(nullptr), key), + address_cast(&curr->next_object), BQueuePtr(nullptr), key), std::memory_order_relaxed); } }; @@ -375,18 +386,18 @@ namespace snmalloc * Checks signing of pointers */ template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> class FreeListIter { - FreeObject::HeadPtr curr{nullptr}; + FreeObject::BHeadPtr curr{nullptr}; #ifdef SNMALLOC_CHECK_CLIENT address_t prev{0}; #endif public: constexpr FreeListIter( - FreeObject::HeadPtr head, address_t prev_value) + FreeObject::BHeadPtr head, address_t prev_value) : curr(head) { #ifdef SNMALLOC_CHECK_CLIENT @@ -408,7 +419,7 @@ namespace snmalloc /** * Returns current head without affecting the iterator. */ - FreeObject::HeadPtr peek() + FreeObject::BHeadPtr peek() { return curr; } @@ -417,7 +428,7 @@ namespace snmalloc * Moves the iterator on, and returns the current value. */ template - FreeObject::HeadPtr + FreeObject::BHeadPtr take(const FreeListKey& key, Domesticator domesticate) { auto c = curr; @@ -455,9 +466,9 @@ namespace snmalloc */ template< bool RANDOM, - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, - bool INIT = true> + bool INIT = true, + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> class FreeListBuilder { static constexpr size_t LENGTH = RANDOM ? 2 : 1; @@ -466,7 +477,7 @@ namespace snmalloc * We use native pointers below so that we don't run afoul of strict * aliasing rules. head is a FreeObject::HeadPtr -- that * is, a known-domesticated pointer to a queue of wild pointers -- and - * it's usually the case that end is a FreeObject::QueuePtr* -- + * it's usually the case that end is a FreeObject::BQueuePtr* -- * that is, a known-domesticated pointer to a wild pointer to a queue of * wild pointers. However, in order to do branchless inserts, we set end * = &head, which breaks strict aliasing rules with the types as given. @@ -481,19 +492,19 @@ namespace snmalloc // This enables branch free enqueuing. std::array end{nullptr}; - FreeObject::QueuePtr* cast_end(uint32_t ix) + FreeObject::BQueuePtr* cast_end(uint32_t ix) { - return reinterpret_cast*>(end[ix]); + return reinterpret_cast*>(end[ix]); } - void set_end(uint32_t ix, FreeObject::QueuePtr* p) + void set_end(uint32_t ix, FreeObject::BQueuePtr* p) { end[ix] = reinterpret_cast(p); } - FreeObject::HeadPtr cast_head(uint32_t ix) + FreeObject::BHeadPtr cast_head(uint32_t ix) { - return FreeObject::HeadPtr( + return FreeObject::BHeadPtr( static_cast*>(head[ix])); } @@ -527,7 +538,7 @@ namespace snmalloc * Adds an element to the builder */ void add( - FreeObject::HeadPtr n, + FreeObject::BHeadPtr n, const FreeListKey& key, LocalEntropy& entropy) { @@ -554,7 +565,7 @@ namespace snmalloc */ template std::enable_if_t - add(FreeObject::HeadPtr n, const FreeListKey& key) + add(FreeObject::BHeadPtr n, const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); set_end(0, FreeObject::store_next(cast_end(0), n, key)); @@ -578,7 +589,7 @@ namespace snmalloc * and is thus subject to encoding if the next_object pointers * encoded. */ - FreeObject::HeadPtr + FreeObject::BHeadPtr read_head(uint32_t index, const FreeListKey& key) { return FreeObject::decode_next( @@ -652,8 +663,8 @@ namespace snmalloc std::enable_if_t< !RANDOM_, std::pair< - FreeObject::HeadPtr, - FreeObject::HeadPtr>> + FreeObject::BHeadPtr, + FreeObject::BHeadPtr>> extract_segment(const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); @@ -664,7 +675,7 @@ namespace snmalloc // this is doing a CONTAINING_RECORD like cast to get back // to the actual object. This isn't true if the builder is // empty, but you are not allowed to call this in the empty case. - auto last = FreeObject::HeadPtr( + auto last = FreeObject::BHeadPtr( FreeObject::from_next_ptr(cast_end(0))); init(); return {first, last}; diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 83c78c6..23a6987 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,38 +211,33 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); - auto domesticate = - [this](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate( - core_alloc->backend_state_ptr(), p); - }; - auto slowpath = - [&]( - sizeclass_t sizeclass, - FreeListIter* - fl) SNMALLOC_FAST_PATH_LAMBDA { - if (likely(core_alloc != nullptr)) - { - return core_alloc->handle_message_queue( - []( - CoreAlloc* core_alloc, - sizeclass_t sizeclass, - FreeListIter* - fl) { - return core_alloc->template small_alloc( - sizeclass, *fl); - }, - core_alloc, - sizeclass, - fl); - } - return lazy_init( - [&](CoreAlloc*, sizeclass_t sizeclass) { - return small_alloc(sizeclass_to_size(sizeclass)); + auto domesticate = [this](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate( + core_alloc->backend_state_ptr(), p); + }; + auto slowpath = [&]( + sizeclass_t sizeclass, + FreeListIter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { + if (likely(core_alloc != nullptr)) + { + return core_alloc->handle_message_queue( + []( + CoreAlloc* core_alloc, + sizeclass_t sizeclass, + FreeListIter<>* fl) { + return core_alloc->template small_alloc(sizeclass, *fl); }, - sizeclass); - }; + core_alloc, + sizeclass, + fl); + } + return lazy_init( + [&](CoreAlloc*, sizeclass_t sizeclass) { + return small_alloc(sizeclass_to_size(sizeclass)); + }, + sizeclass); + }; return local_cache.template alloc( domesticate, size, slowpath); diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 4d3d55a..593fc32 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -12,9 +12,8 @@ namespace snmalloc { using Stats = AllocStats; - template - inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero( - FreeObject::HeadPtr p, sizeclass_t sizeclass) + inline static SNMALLOC_FAST_PATH void* + finish_alloc_no_zero(FreeObject::HeadPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -24,12 +23,9 @@ namespace snmalloc return r; } - template< - ZeroMem zero_mem, - typename SharedStateHandle, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - inline static SNMALLOC_FAST_PATH void* finish_alloc( - FreeObject::HeadPtr p, sizeclass_t sizeclass) + template + inline static SNMALLOC_FAST_PATH void* + finish_alloc(FreeObject::HeadPtr p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); @@ -50,8 +46,7 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - FreeListIter - small_fast_free_lists[NUM_SIZECLASSES] = {}; + FreeListIter<> small_fast_free_lists[NUM_SIZECLASSES] = {}; // This is the entropy for a particular thread. LocalEntropy entropy; @@ -85,10 +80,9 @@ namespace snmalloc { auto& key = entropy.get_free_list_key(); auto domesticate = - [local_state](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; for (size_t i = 0; i < NUM_SIZECLASSES; i++) { diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 4be0499..3b31fa4 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -25,11 +25,9 @@ namespace snmalloc * Data-structure for building the free list for this slab. */ #ifdef SNMALLOC_CHECK_CLIENT - FreeListBuilder - free_queue; + FreeListBuilder free_queue; #else - FreeListBuilder - free_queue; + FreeListBuilder free_queue; #endif /** @@ -155,14 +153,11 @@ namespace snmalloc * available objects for this metaslab. */ template - static SNMALLOC_FAST_PATH std::pair< - FreeObject::HeadPtr, - bool> + static SNMALLOC_FAST_PATH std::pair alloc_free_list( Domesticator domesticate, Metaslab* meta, - FreeListIter& - fast_free_list, + FreeListIter<>& fast_free_list, LocalEntropy& entropy, sizeclass_t sizeclass) { diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index c658c2e..6e3b0a3 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -35,12 +35,10 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) - FreeObject::AtomicQueuePtr back{nullptr}; + alignas(CACHELINE_SIZE) FreeObject::AtomicQueuePtr back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) - FreeObject::QueuePtr front{nullptr}; + alignas(CACHELINE_SIZE) FreeObject::QueuePtr front{nullptr}; constexpr RemoteAllocator() = default; @@ -50,9 +48,7 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void - init(FreeObject::HeadPtr - stub) + void init(FreeObject::HeadPtr stub) { FreeObject::atomic_store_null(stub, key_global); front = capptr_rewild(stub); @@ -60,9 +56,9 @@ namespace snmalloc invariant(); } - FreeObject::QueuePtr destroy() + FreeObject::QueuePtr destroy() { - FreeObject::QueuePtr fnt = front; + FreeObject::QueuePtr fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -70,8 +66,7 @@ namespace snmalloc inline bool is_empty() { - FreeObject::QueuePtr bk = - back.load(std::memory_order_relaxed); + FreeObject::QueuePtr bk = back.load(std::memory_order_relaxed); return bk == front; } @@ -82,10 +77,8 @@ namespace snmalloc */ template void enqueue( - FreeObject::HeadPtr - first, - FreeObject::HeadPtr - last, + FreeObject::HeadPtr first, + FreeObject::HeadPtr last, const FreeListKey& key, Domesticator domesticate) { @@ -93,13 +86,13 @@ namespace snmalloc FreeObject::atomic_store_null(last, key); // exchange needs to be a release, so nullptr in next is visible. - FreeObject::QueuePtr prev = + FreeObject::QueuePtr prev = back.exchange(capptr_rewild(last), std::memory_order_release); FreeObject::atomic_store_next(domesticate(prev), first, key); } - FreeObject::QueuePtr peek() + FreeObject::QueuePtr peek() { return front; } @@ -108,16 +101,12 @@ namespace snmalloc * Returns the front message, or null if not possible to return a message. */ template - std::pair< - FreeObject::HeadPtr, - bool> + std::pair dequeue(const FreeListKey& key, Domesticator domesticate) { invariant(); - FreeObject::HeadPtr - first = domesticate(front); - FreeObject::HeadPtr - next = first->atomic_read_next(key, domesticate); + FreeObject::HeadPtr first = domesticate(front); + FreeObject::HeadPtr next = first->atomic_read_next(key, domesticate); if (next != nullptr) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 0d20d89..5a739aa 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -16,14 +16,7 @@ namespace snmalloc */ struct RemoteDeallocCache { - std::array< - FreeListBuilder< - false, - capptr::bounds::Alloc, - capptr::bounds::AllocWild, - false>, - REMOTE_SLOTS> - list; + std::array, REMOTE_SLOTS> list; /** * The total amount of memory we are waiting for before we will dispatch @@ -77,8 +70,7 @@ namespace snmalloc const FreeListKey& key) { SNMALLOC_ASSERT(initialised); - auto r = - p.template as_reinterpret>(); + auto r = p.template as_reinterpret>(); list[get_slot(target_id, 0)].add(r, key); } @@ -93,10 +85,9 @@ namespace snmalloc size_t post_round = 0; bool sent_something = false; auto domesticate = - [local_state](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; while (true) { @@ -123,7 +114,7 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - FreeListIter resend; + FreeListIter<> resend; list[my_slot].close(resend, key); post_round++;