From b501da69db887a01a9a71d4290a54e921ca363ce Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 14 Jul 2021 19:49:21 +0100 Subject: [PATCH] Implements protection on remote messages queues This extends the freelist protection to the remote message queues. They effectively perform doubly linked list entries for the message queue with the enqueue operation first linking in the previous pointer, and then then atomically setting the next. This ensures that the visible states always satisfy the invariant that the forward and backward pointers are correct for any visisble object. There is a key_global that is used for all remote deallocations. The remote cache uses the same protection to build the temporary lists before forwarding to the next allocator. The mpscq is integrated into the remoteallocator as it is no longer a reusable datastructure, but a special purpose implementation. --- src/ds/mpscq.h | 91 -------------------- src/mem/commonconfig.h | 2 +- src/mem/corealloc.h | 39 +++++---- src/mem/freelist.h | 171 ++++++++++++++++++++++++++++---------- src/mem/globalconfig.h | 3 + src/mem/localalloc.h | 8 +- src/mem/localcache.h | 10 ++- src/mem/metaslab.h | 63 +++++++------- src/mem/remoteallocator.h | 120 +++++++++++++++++--------- src/mem/remotecache.h | 83 +++++------------- 10 files changed, 295 insertions(+), 295 deletions(-) delete mode 100644 src/ds/mpscq.h diff --git a/src/ds/mpscq.h b/src/ds/mpscq.h deleted file mode 100644 index c3e8fb9..0000000 --- a/src/ds/mpscq.h +++ /dev/null @@ -1,91 +0,0 @@ -#pragma once - -#include "bits.h" -#include "helpers.h" - -#include -namespace snmalloc -{ - template< - class T, - template typename Ptr = Pointer, - template typename AtomicPtr = AtomicPointer> - class MPSCQ - { - private: - static_assert( - std::is_same>::value, - "T->next must be an AtomicPtr"); - - AtomicPtr back{nullptr}; - Ptr front{nullptr}; - - public: - constexpr MPSCQ() = default; - - void invariant() - { - SNMALLOC_ASSERT(back != nullptr); - SNMALLOC_ASSERT(front != nullptr); - } - - void init(Ptr stub) - { - stub->next.store(nullptr, std::memory_order_relaxed); - front = stub; - back.store(stub, std::memory_order_relaxed); - invariant(); - } - - Ptr destroy() - { - Ptr fnt = front; - back.store(nullptr, std::memory_order_relaxed); - front = nullptr; - return fnt; - } - - inline bool is_empty() - { - Ptr bk = back.load(std::memory_order_relaxed); - - return bk == front; - } - - void enqueue(Ptr first, Ptr last) - { - // Pushes a list of messages to the queue. Each message from first to - // last should be linked together through their next pointers. - invariant(); - last->next.store(nullptr, std::memory_order_relaxed); - std::atomic_thread_fence(std::memory_order_release); - Ptr prev = back.exchange(last, std::memory_order_relaxed); - prev->next.store(first, std::memory_order_relaxed); - } - - Ptr peek() - { - return front; - } - - std::pair, bool> dequeue() - { - // Returns the front message, or null if not possible to return a message. - invariant(); - Ptr first = front; - Ptr next = first->next.load(std::memory_order_relaxed); - - Aal::prefetch(&(next->next)); - if (next != nullptr) - { - front = next; - SNMALLOC_ASSERT(front != nullptr); - std::atomic_thread_fence(std::memory_order_acquire); - invariant(); - return {first, true}; - } - - return {nullptr, false}; - } - }; -} // namespace snmalloc diff --git a/src/mem/commonconfig.h b/src/mem/commonconfig.h index adfdb33..9c24371 100644 --- a/src/mem/commonconfig.h +++ b/src/mem/commonconfig.h @@ -1,7 +1,7 @@ #pragma once #include "../ds/defines.h" -#include "remoteallocator.h" +#include "remotecache.h" namespace snmalloc { diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index e307fb1..4df9ca8 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -95,7 +95,7 @@ namespace snmalloc */ auto& message_queue() { - return public_state()->message_queue; + return *public_state(); } /** @@ -108,7 +108,7 @@ namespace snmalloc // Using an actual allocation removes a conditional from a critical path. auto dummy = CapPtr(small_alloc_one(sizeof(MIN_ALLOC_SIZE))) - .template as_static(); + .template as_static(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -141,6 +141,8 @@ namespace snmalloc { auto slab_end = pointer_offset(bumpptr, slab_size + 1 - rsize); + FreeListKey key(entropy.get_constant_key()); + FreeListBuilder b; SNMALLOC_ASSERT(b.empty()); @@ -187,14 +189,14 @@ namespace snmalloc auto curr_ptr = start_ptr; do { - b.add(FreeObject::make(curr_ptr.as_void()), entropy); + b.add(FreeObject::make(curr_ptr.as_void()), key); curr_ptr = curr_ptr->next; } while (curr_ptr != start_ptr); #else auto p = bumpptr; do { - b.add(Aal::capptr_bound(p, rsize), entropy); + b.add(Aal::capptr_bound(p, rsize), key); p = pointer_offset(p, rsize); } while (p < slab_end); #endif @@ -202,14 +204,15 @@ namespace snmalloc bumpptr = slab_end; SNMALLOC_ASSERT(!b.empty()); - b.close(fast_free_list, entropy); + b.close(fast_free_list, key); } ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { + FreeListKey key(entropy.get_constant_key()); FreeListIter fl; - meta->free_queue.close(fl, entropy); - void* p = finish_alloc_no_zero(fl.take(entropy), sizeclass); + meta->free_queue.close(fl, key); + void* p = finish_alloc_no_zero(fl.take(key), sizeclass); #ifdef CHECK_CLIENT // Check free list is well-formed on platforms with @@ -217,7 +220,7 @@ namespace snmalloc size_t count = 1; // Already taken one above. while (!fl.empty()) { - fl.take(entropy); + fl.take(key); count++; } // Check the list contains all the elements @@ -335,7 +338,7 @@ namespace snmalloc auto& entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), snmalloc::address_cast(p)); - auto r = message_queue().dequeue(); + auto r = message_queue().dequeue(key_global); if (unlikely(!r.second)) break; @@ -360,7 +363,7 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaEntry& entry, CapPtr p, bool& need_post) + const MetaEntry& entry, CapPtr p, bool& need_post) { // TODO this needs to not double count stats // TODO this needs to not double revoke if using MTE @@ -381,7 +384,7 @@ namespace snmalloc need_post = true; attached_cache->remote_dealloc_cache .template dealloc( - entry.get_remote()->trunc_id(), p.as_void()); + entry.get_remote()->trunc_id(), p.as_void(), key_global); } } @@ -431,7 +434,7 @@ namespace snmalloc // stats().remote_post(); // TODO queue not in line! bool sent_something = attached_cache->remote_dealloc_cache.post( - handle, public_state()->trunc_id()); + handle, public_state()->trunc_id(), key_global); return sent_something; } @@ -472,8 +475,10 @@ namespace snmalloc auto cp = CapPtr(reinterpret_cast(p)); + FreeListKey key(entropy.get_constant_key()); + // Update the head and the next pointer in the free list. - meta->free_queue.add(cp, entropy); + meta->free_queue.add(cp, key, entropy); return likely(!meta->return_object()); } @@ -533,8 +538,10 @@ namespace snmalloc // Set meta slab to empty. meta->initialise(sizeclass); + FreeListKey key(entropy.get_constant_key()); + // take an allocation from the free list - auto p = fast_free_list.take(entropy); + auto p = fast_free_list.take(key); return finish_alloc(p, sizeclass); } @@ -550,12 +557,12 @@ namespace snmalloc if (destroy_queue) { - CapPtr p = message_queue().destroy(); + auto p = message_queue().destroy(); while (p != nullptr) { bool need_post = true; // Always going to post, so ignore. - auto n = p->non_atomic_next; + auto n = p->atomic_read_next(key_global); auto& entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 4813791..50dbc04 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -41,7 +41,18 @@ namespace snmalloc { - struct Remote; + struct FreeListKey + { + address_t key; + + FreeListKey(uint64_t key_) + { + if constexpr (bits::BITS == 64) + key = static_cast(key_); + else + key = key_ & 0xffff'ffff; + } + }; /** * This function is used to sign back pointers in the free list. @@ -53,11 +64,11 @@ namespace snmalloc * list. */ inline static uintptr_t - signed_prev(address_t curr, address_t next, LocalEntropy& entropy) + signed_prev(address_t curr, address_t next, FreeListKey& key) { auto c = curr; auto n = next; - auto k = entropy.get_constant_key(); + auto k = key.key; return (c + k) * (n - k); } @@ -66,10 +77,19 @@ namespace snmalloc * There is an optional second field that is effectively a * back pointer in a doubly linked list, however, it is encoded * to prevent corruption. + * + * TODO: Consider put prev_encoded at the end of the object, would + * require size to be threaded through, but would provide more OOB + * detection. */ class FreeObject { - CapPtr next_object; + union + { + CapPtr next_object; + // TODO: Should really use C++20 atomic_ref rather than a union. + AtomicCapPtr atomic_next_object; + }; #ifdef CHECK_CLIENT // Encoded representation of a back pointer. // Hard to fake, and provides consistency on @@ -83,39 +103,66 @@ namespace snmalloc return p.template as_static(); } - /** - * Construct a FreeObject for local slabs from a Remote message. - */ - static CapPtr make(CapPtr p) - { - // TODO: Zero the difference between a FreeObject and a Remote - return p.template as_reinterpret(); - } - /** * Assign next_object and update its prev_encoded if CHECK_CLIENT. - * * Static so that it can be used on reference to a FreeObject. * * Returns a pointer to the next_object field of the next parameter as an * optimization for repeated snoc operations (in which * next->next_object is nullptr). */ - static CapPtr* store( + static CapPtr* store_next( CapPtr* curr, CapPtr next, - LocalEntropy& entropy) + FreeListKey& key) { - *curr = next; #ifdef CHECK_CLIENT next->prev_encoded = - signed_prev(address_cast(curr), address_cast(next), entropy); + signed_prev(address_cast(curr), address_cast(next), key); #else - UNUSED(entropy); + UNUSED(key); #endif + *curr = next; return &(next->next_object); } + /** + * Assign next_object and update its prev_encoded if CHECK_CLIENT + * + * Uses the atomic view of next, so can be used in the message queues. + */ + void atomic_store_next(CapPtr next, FreeListKey& key) + { +#ifdef CHECK_CLIENT + next->prev_encoded = + signed_prev(address_cast(this), address_cast(next), key); +#else + UNUSED(key); +#endif + // Signature needs to be visible before item is linked in + // so requires release semantics. + atomic_next_object.store(next, std::memory_order_release); + } + + void atomic_store_null() + { + atomic_next_object.store(nullptr, std::memory_order_relaxed); + } + + CapPtr atomic_read_next(FreeListKey& key) + { + auto n = atomic_next_object.load(std::memory_order_acquire); +#ifdef CHECK_CLIENT + if (n != nullptr) + { + n->check_prev(signed_prev(address_cast(this), address_cast(n), key)); + } +#else + UNUSED(key); +#endif + return n; + } + /** * Check the signature of this FreeObject */ @@ -135,6 +182,10 @@ namespace snmalloc } }; + static_assert( + sizeof(FreeObject) <= MIN_ALLOC_SIZE, + "Needs to be able to fit in smallest allocation."); + /** * Used to iterate a free list in object space. * @@ -179,7 +230,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - CapPtr take(LocalEntropy& entropy) + CapPtr take(FreeListKey& key) { auto c = curr; auto next = curr->read_next(); @@ -188,9 +239,9 @@ namespace snmalloc curr = next; #ifdef CHECK_CLIENT c->check_prev(prev); - prev = signed_prev(address_cast(c), address_cast(next), entropy); + prev = signed_prev(address_cast(c), address_cast(next), key); #else - UNUSED(entropy); + UNUSED(key); #endif return c; @@ -220,25 +271,23 @@ namespace snmalloc * If RANDOM is set to false, then the code does not perform any * randomisation. */ - template + template class FreeListBuilder { static constexpr size_t LENGTH = RANDOM ? 2 : 1; // Pointer to the first element. - CapPtr head[LENGTH]; + std::array, LENGTH> head; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - CapPtr* end[LENGTH]; - - public: - S s; + std::array*, LENGTH> end{nullptr}; public: constexpr FreeListBuilder() { - init(); + if (INIT) + init(); } /** @@ -257,7 +306,8 @@ namespace snmalloc /** * Adds an element to the builder */ - void add(CapPtr n, LocalEntropy& entropy) + void + add(CapPtr n, FreeListKey& key, LocalEntropy& entropy) { uint32_t index; if constexpr (RANDOM) @@ -265,33 +315,45 @@ namespace snmalloc else index = 0; - end[index] = FreeObject::store(end[index], n, entropy); + end[index] = FreeObject::store_next(end[index], n, key); + } + + /** + * Adds an element to the builder, if we are guaranteed that + * RANDOM is false. This is useful in certain construction + * cases that do not need to introduce randomness, such as + * during the initialation construction of a free list, which + * uses its own algorithm, or during building remote deallocation + * lists, which will be randomised at the other end. + */ + template + std::enable_if_t + add(CapPtr n, FreeListKey& key) + { + static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); + end[0] = FreeObject::store_next(end[0], n, key); } /** * Makes a terminator to a free list. - * - * Termination uses the bottom bit, this allows the next pointer - * to always be to the same slab. */ - SNMALLOC_FAST_PATH void - terminate_list(uint32_t index, LocalEntropy& entropy) + SNMALLOC_FAST_PATH void terminate_list(uint32_t index, FreeListKey& key) { - UNUSED(entropy); + UNUSED(key); *end[index] = nullptr; } - address_t get_fake_signed_prev(uint32_t index, LocalEntropy& entropy) + address_t get_fake_signed_prev(uint32_t index, FreeListKey& key) { return signed_prev( - address_cast(&head[index]), address_cast(head[index]), entropy); + address_cast(&head[index]), address_cast(head[index]), key); } /** * Close a free list, and set the iterator parameter * to iterate it. */ - SNMALLOC_FAST_PATH void close(FreeListIter& fl, LocalEntropy& entropy) + SNMALLOC_FAST_PATH void close(FreeListIter& fl, FreeListKey& key) { if constexpr (RANDOM) { @@ -303,27 +365,27 @@ namespace snmalloc { // The start token has been corrupted. // TOCTTOU issue, but small window here. - head[1]->check_prev(get_fake_signed_prev(1, entropy)); + head[1]->check_prev(get_fake_signed_prev(1, key)); - terminate_list(1, entropy); + terminate_list(1, key); // Append 1 to 0 - FreeObject::store(end[0], head[1], entropy); + FreeObject::store_next(end[0], head[1], key); SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); } else { - terminate_list(0, entropy); + terminate_list(0, key); } } else { - terminate_list(0, entropy); + terminate_list(0, key); } - fl = {head[0], get_fake_signed_prev(0, entropy)}; + fl = {head[0], get_fake_signed_prev(0, key)}; init(); } @@ -337,5 +399,22 @@ namespace snmalloc end[i] = &head[i]; } } + + std::pair, CapPtr> + extract_segment() + { + SNMALLOC_ASSERT(!empty()); + SNMALLOC_ASSERT(!RANDOM); // TODO: Turn this into a static failure. + + auto first = head[0]; + // end[0] is pointing to the first field in the object, + // this is doing a CONTAINING_RECORD like cast to get back + // to the actual object. This isn't true if the builder is + // empty, but you are not allowed to call this in the empty case. + auto last = + CapPtr(reinterpret_cast(end[0])); + init(); + return {first, last}; + } }; } // namespace snmalloc diff --git a/src/mem/globalconfig.h b/src/mem/globalconfig.h index d87e75a..981f09a 100644 --- a/src/mem/globalconfig.h +++ b/src/mem/globalconfig.h @@ -73,6 +73,9 @@ namespace snmalloc if (initialised) return; + // Initialise key for remote deallocation lists + key_global = FreeListKey(get_entropy64()); + // Need to initialise pagemap. backend_state.init(); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 9a67491..c42685e 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -228,7 +228,7 @@ namespace snmalloc MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), CapPtr(p)); + entry.get_remote()->trunc_id(), CapPtr(p), key_global); post_remote_cache(); return; } @@ -249,7 +249,7 @@ namespace snmalloc */ auto& message_queue() { - return local_cache.remote_allocator->message_queue; + return local_cache.remote_allocator; } public: @@ -382,7 +382,9 @@ namespace snmalloc if (local_cache.remote_dealloc_cache.reserve_space(entry)) { local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), CapPtr(p)); + entry.get_remote()->trunc_id(), + CapPtr(p), + key_global); #ifdef SNMALLOC_TRACING std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p) << std::endl; diff --git a/src/mem/localcache.h b/src/mem/localcache.h index ba364ad..2070966 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -76,6 +76,8 @@ namespace snmalloc typename SharedStateHandle> bool flush(DeallocFun dealloc, SharedStateHandle handle) { + FreeListKey key(entropy.get_constant_key()); + // Return all the free lists to the allocator. // Used during thread teardown for (size_t i = 0; i < NUM_SIZECLASSES; i++) @@ -84,25 +86,27 @@ namespace snmalloc // call. while (!small_fast_free_lists[i].empty()) { - auto p = small_fast_free_lists[i].take(entropy); + auto p = small_fast_free_lists[i].take(key); dealloc(finish_alloc_no_zero(p, i)); } } return remote_dealloc_cache.post( - handle, remote_allocator->trunc_id()); + handle, remote_allocator->trunc_id(), key_global); } template SNMALLOC_FAST_PATH void* alloc(size_t size, Slowpath slowpath) { + FreeListKey key(entropy.get_constant_key()); + sizeclass_t sizeclass = size_to_sizeclass(size); stats.alloc_request(size); stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; if (likely(!fl.empty())) { - auto p = fl.take(entropy); + auto p = fl.take(key); return finish_alloc(p, sizeclass); } return slowpath(sizeclass, &fl); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index f341064..9d774e2 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -18,34 +18,6 @@ namespace snmalloc sizeof(SlabLink) <= MIN_ALLOC_SIZE, "Need to be able to pack a SlabLink into any free small alloc"); - /** - * This struct is used inside FreeListBuilder to account for the - * alignment space that is wasted in sizeof. - * - * This is part of Metaslab abstraction. - */ - struct MetaslabEnd - { - /** - * The number of deallocation required until we hit a slow path. This - * counts down in two different ways that are handled the same on the - * fast path. The first is - * - deallocations until the slab has sufficient entries to be considered - * useful to allocate from. This could be as low as 1, or when we have - * a requirement for entropy then it could be much higher. - * - deallocations until the slab is completely unused. This is needed - * to be detected, so that the statistics can be kept up to date, and - * potentially return memory to the a global pool of slabs/chunks. - */ - uint16_t needed = 0; - - /** - * Flag that is used to indicate that the slab is currently not active. - * I.e. it is not in a CoreAllocator cache for the appropriate sizeclass. - */ - bool sleeping = false; - }; - // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. class alignas(CACHELINE_SIZE) Metaslab : public SlabLink @@ -59,19 +31,38 @@ namespace snmalloc * Spare 32bits are used for the fields in MetaslabEnd. */ #ifdef CHECK_CLIENT - FreeListBuilder free_queue; + FreeListBuilder free_queue; #else - FreeListBuilder free_queue; + FreeListBuilder free_queue; #endif + /** + * The number of deallocation required until we hit a slow path. This + * counts down in two different ways that are handled the same on the + * fast path. The first is + * - deallocations until the slab has sufficient entries to be considered + * useful to allocate from. This could be as low as 1, or when we have + * a requirement for entropy then it could be much higher. + * - deallocations until the slab is completely unused. This is needed + * to be detected, so that the statistics can be kept up to date, and + * potentially return memory to the a global pool of slabs/chunks. + */ + uint16_t needed_ = 0; + + /** + * Flag that is used to indicate that the slab is currently not active. + * I.e. it is not in a CoreAllocator cache for the appropriate sizeclass. + */ + bool sleeping_ = false; + uint16_t& needed() { - return free_queue.s.needed; + return needed_; } bool& sleeping() { - return free_queue.s.sleeping; + return sleeping_; } /** @@ -151,13 +142,17 @@ namespace snmalloc LocalEntropy& entropy, sizeclass_t sizeclass) { + FreeListKey key(entropy.get_constant_key()); + FreeListIter tmp_fl; - meta->free_queue.close(tmp_fl, entropy); - auto p = tmp_fl.take(entropy); + meta->free_queue.close(tmp_fl, key); + auto p = tmp_fl.take(key); fast_free_list = tmp_fl; #ifdef CHECK_CLIENT entropy.refresh_bits(); +#else + UNUSED(entropy); #endif // Treat stealing the free list as allocating it all. diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index c639e5c..d3c1a73 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/mpscq.h" #include "../mem/allocconfig.h" #include "../mem/freelist.h" #include "../mem/metaslab.h" @@ -9,59 +8,102 @@ #include #include -#ifdef CHECK_CLIENT -# define SNMALLOC_DONT_CACHE_ALLOCATOR_PTR -#endif - namespace snmalloc { - /* - * A region of memory destined for a remote allocator's dealloc() via the - * message passing system. This structure is placed at the beginning of - * the allocation itself when it is queued for sending. - */ - struct Remote - { - using alloc_id_t = size_t; - union - { - CapPtr non_atomic_next; - AtomicCapPtr next{nullptr}; - }; - - constexpr Remote() : next(nullptr) {} - - /** Zero out a Remote tracking structure, return pointer to object base */ - template - SNMALLOC_FAST_PATH static CapPtr clear(CapPtr self) - { - pal_zero(self, sizeof(Remote)); - return self.as_void(); - } - }; - - static_assert( - sizeof(Remote) <= MIN_ALLOC_SIZE, - "Needs to be able to fit in smallest allocation."); - // Remotes need to be aligned enough that all the // small size classes can fit in the bottom bits. static constexpr size_t REMOTE_MIN_ALIGN = bits::min( CACHELINE_SIZE, bits::next_pow2_const(NUM_SIZECLASSES + 1)); + /** + * Global key for all remote lists. + */ + inline static FreeListKey key_global(0xdeadbeef); + struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator { - using alloc_id_t = Remote::alloc_id_t; + using alloc_id_t = address_t; + // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. + alignas(CACHELINE_SIZE) AtomicCapPtr back{nullptr}; + // Store the two ends on different cache lines as access by different + // threads. + alignas(CACHELINE_SIZE) CapPtr front{nullptr}; - MPSCQ message_queue; + constexpr RemoteAllocator() = default; + + void invariant() + { + SNMALLOC_ASSERT(back != nullptr); + SNMALLOC_ASSERT(front != nullptr); + } + + void init(CapPtr stub) + { + stub->atomic_store_null(); + front = stub; + back.store(stub, std::memory_order_relaxed); + invariant(); + } + + CapPtr destroy() + { + CapPtr fnt = front; + back.store(nullptr, std::memory_order_relaxed); + front = nullptr; + return fnt; + } + + inline bool is_empty() + { + CapPtr bk = back.load(std::memory_order_relaxed); + + return bk == front; + } + + void enqueue( + CapPtr first, + CapPtr last, + FreeListKey& key) + { + // Pushes a list of messages to the queue. Each message from first to + // last should be linked together through their next pointers. + invariant(); + last->atomic_store_null(); + + // exchange needs to be a release, so nullptr in next is visible. + CapPtr prev = + back.exchange(last, std::memory_order_release); + + prev->atomic_store_next(first, key); + } + + CapPtr peek() + { + return front; + } + + std::pair, bool> dequeue(FreeListKey& key) + { + // Returns the front message, or null if not possible to return a message. + invariant(); + CapPtr first = front; + CapPtr next = first->atomic_read_next(key); + + if (next != nullptr) + { + front = next; + invariant(); + return {first, true}; + } + + return {nullptr, false}; + } alloc_id_t trunc_id() { - return static_cast( - reinterpret_cast(&message_queue)) & - ~SIZECLASS_MASK; + return address_cast(this); } }; } // namespace snmalloc diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index a017b67..af71c22 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/mpscq.h" #include "../mem/allocconfig.h" #include "../mem/freelist.h" #include "../mem/metaslab.h" @@ -17,40 +16,7 @@ namespace snmalloc */ struct RemoteDeallocCache { - /* - * A singly-linked list of Remote objects, supporting append and - * take-all operations. Intended only for the private use of this - * allocator; the Remote objects here will later be taken and pushed - * to the inter-thread message queues. - */ - struct RemoteList - { - /* - * A stub Remote object that will always be the head of this list; - * never taken for further processing. - */ - Remote head{}; - - /** - * Initially is null ptr, and needs to be non-null before anything runs on - * this. - */ - CapPtr last{nullptr}; - - void clear() - { - last = CapPtr(&head); - } - - bool empty() - { - return address_cast(last) == address_cast(&head); - } - - constexpr RemoteList() = default; - }; - - std::array list{}; + std::array, REMOTE_SLOTS> list; /** * The total amount of memory we are waiting for before we will dispatch @@ -98,19 +64,22 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH void - dealloc(Remote::alloc_id_t target_id, CapPtr p) + SNMALLOC_FAST_PATH void dealloc( + RemoteAllocator::alloc_id_t target_id, + CapPtr p, + FreeListKey& key) { SNMALLOC_ASSERT(initialised); - auto r = p.template as_reinterpret(); + auto r = p.template as_reinterpret(); - RemoteList* l = &list[get_slot(target_id, 0)]; - l->last->non_atomic_next = r; - l->last = r; + list[get_slot(target_id, 0)].add(r, key); } template - bool post(SharedStateHandle handle, Remote::alloc_id_t id) + bool post( + SharedStateHandle handle, + RemoteAllocator::alloc_id_t id, + FreeListKey& key) { SNMALLOC_ASSERT(initialised); size_t post_round = 0; @@ -125,46 +94,37 @@ namespace snmalloc if (i == my_slot) continue; - RemoteList* l = &list[i]; - CapPtr first = l->head.non_atomic_next; - - if (!l->empty()) + if (!list[i].empty()) { + auto [first, last] = list[i].extract_segment(); MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(first)); - entry.get_remote()->message_queue.enqueue(first, l->last); - l->clear(); + entry.get_remote()->enqueue(first, last, key); sent_something = true; } } - RemoteList* resend = &list[my_slot]; - if (resend->empty()) + if (list[my_slot].empty()) break; // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - CapPtr r = resend->head.non_atomic_next; - resend->last->non_atomic_next = nullptr; - resend->clear(); + FreeListIter resend; + list[my_slot].close(resend, key); post_round++; - while (r != nullptr) + while (!resend.empty()) { // Use the next N bits to spread out remote deallocs in our own // slot. + auto r = resend.take(key); MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(r)); auto i = entry.get_remote()->trunc_id(); - // TODO correct size for slot offset size_t slot = get_slot(i, post_round); - RemoteList* l = &list[slot]; - l->last->non_atomic_next = r; - l->last = r; - - r = r->non_atomic_next; + list[slot].add(r, key); } } @@ -190,8 +150,7 @@ namespace snmalloc #endif for (auto& l : list) { - SNMALLOC_ASSERT(l.last == nullptr || l.empty()); - l.clear(); + l.init(); } capacity = REMOTE_CACHE; }