diff --git a/src/ds/mpscq.h b/src/ds/mpscq.h deleted file mode 100644 index c3e8fb9..0000000 --- a/src/ds/mpscq.h +++ /dev/null @@ -1,91 +0,0 @@ -#pragma once - -#include "bits.h" -#include "helpers.h" - -#include -namespace snmalloc -{ - template< - class T, - template typename Ptr = Pointer, - template typename AtomicPtr = AtomicPointer> - class MPSCQ - { - private: - static_assert( - std::is_same>::value, - "T->next must be an AtomicPtr"); - - AtomicPtr back{nullptr}; - Ptr front{nullptr}; - - public: - constexpr MPSCQ() = default; - - void invariant() - { - SNMALLOC_ASSERT(back != nullptr); - SNMALLOC_ASSERT(front != nullptr); - } - - void init(Ptr stub) - { - stub->next.store(nullptr, std::memory_order_relaxed); - front = stub; - back.store(stub, std::memory_order_relaxed); - invariant(); - } - - Ptr destroy() - { - Ptr fnt = front; - back.store(nullptr, std::memory_order_relaxed); - front = nullptr; - return fnt; - } - - inline bool is_empty() - { - Ptr bk = back.load(std::memory_order_relaxed); - - return bk == front; - } - - void enqueue(Ptr first, Ptr last) - { - // Pushes a list of messages to the queue. Each message from first to - // last should be linked together through their next pointers. - invariant(); - last->next.store(nullptr, std::memory_order_relaxed); - std::atomic_thread_fence(std::memory_order_release); - Ptr prev = back.exchange(last, std::memory_order_relaxed); - prev->next.store(first, std::memory_order_relaxed); - } - - Ptr peek() - { - return front; - } - - std::pair, bool> dequeue() - { - // Returns the front message, or null if not possible to return a message. - invariant(); - Ptr first = front; - Ptr next = first->next.load(std::memory_order_relaxed); - - Aal::prefetch(&(next->next)); - if (next != nullptr) - { - front = next; - SNMALLOC_ASSERT(front != nullptr); - std::atomic_thread_fence(std::memory_order_acquire); - invariant(); - return {first, true}; - } - - return {nullptr, false}; - } - }; -} // namespace snmalloc diff --git a/src/mem/commonconfig.h b/src/mem/commonconfig.h index adfdb33..9c24371 100644 --- a/src/mem/commonconfig.h +++ b/src/mem/commonconfig.h @@ -1,7 +1,7 @@ #pragma once #include "../ds/defines.h" -#include "remoteallocator.h" +#include "remotecache.h" namespace snmalloc { diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index e307fb1..4df9ca8 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -95,7 +95,7 @@ namespace snmalloc */ auto& message_queue() { - return public_state()->message_queue; + return *public_state(); } /** @@ -108,7 +108,7 @@ namespace snmalloc // Using an actual allocation removes a conditional from a critical path. auto dummy = CapPtr(small_alloc_one(sizeof(MIN_ALLOC_SIZE))) - .template as_static(); + .template as_static(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -141,6 +141,8 @@ namespace snmalloc { auto slab_end = pointer_offset(bumpptr, slab_size + 1 - rsize); + FreeListKey key(entropy.get_constant_key()); + FreeListBuilder b; SNMALLOC_ASSERT(b.empty()); @@ -187,14 +189,14 @@ namespace snmalloc auto curr_ptr = start_ptr; do { - b.add(FreeObject::make(curr_ptr.as_void()), entropy); + b.add(FreeObject::make(curr_ptr.as_void()), key); curr_ptr = curr_ptr->next; } while (curr_ptr != start_ptr); #else auto p = bumpptr; do { - b.add(Aal::capptr_bound(p, rsize), entropy); + b.add(Aal::capptr_bound(p, rsize), key); p = pointer_offset(p, rsize); } while (p < slab_end); #endif @@ -202,14 +204,15 @@ namespace snmalloc bumpptr = slab_end; SNMALLOC_ASSERT(!b.empty()); - b.close(fast_free_list, entropy); + b.close(fast_free_list, key); } ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { + FreeListKey key(entropy.get_constant_key()); FreeListIter fl; - meta->free_queue.close(fl, entropy); - void* p = finish_alloc_no_zero(fl.take(entropy), sizeclass); + meta->free_queue.close(fl, key); + void* p = finish_alloc_no_zero(fl.take(key), sizeclass); #ifdef CHECK_CLIENT // Check free list is well-formed on platforms with @@ -217,7 +220,7 @@ namespace snmalloc size_t count = 1; // Already taken one above. while (!fl.empty()) { - fl.take(entropy); + fl.take(key); count++; } // Check the list contains all the elements @@ -335,7 +338,7 @@ namespace snmalloc auto& entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), snmalloc::address_cast(p)); - auto r = message_queue().dequeue(); + auto r = message_queue().dequeue(key_global); if (unlikely(!r.second)) break; @@ -360,7 +363,7 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaEntry& entry, CapPtr p, bool& need_post) + const MetaEntry& entry, CapPtr p, bool& need_post) { // TODO this needs to not double count stats // TODO this needs to not double revoke if using MTE @@ -381,7 +384,7 @@ namespace snmalloc need_post = true; attached_cache->remote_dealloc_cache .template dealloc( - entry.get_remote()->trunc_id(), p.as_void()); + entry.get_remote()->trunc_id(), p.as_void(), key_global); } } @@ -431,7 +434,7 @@ namespace snmalloc // stats().remote_post(); // TODO queue not in line! bool sent_something = attached_cache->remote_dealloc_cache.post( - handle, public_state()->trunc_id()); + handle, public_state()->trunc_id(), key_global); return sent_something; } @@ -472,8 +475,10 @@ namespace snmalloc auto cp = CapPtr(reinterpret_cast(p)); + FreeListKey key(entropy.get_constant_key()); + // Update the head and the next pointer in the free list. - meta->free_queue.add(cp, entropy); + meta->free_queue.add(cp, key, entropy); return likely(!meta->return_object()); } @@ -533,8 +538,10 @@ namespace snmalloc // Set meta slab to empty. meta->initialise(sizeclass); + FreeListKey key(entropy.get_constant_key()); + // take an allocation from the free list - auto p = fast_free_list.take(entropy); + auto p = fast_free_list.take(key); return finish_alloc(p, sizeclass); } @@ -550,12 +557,12 @@ namespace snmalloc if (destroy_queue) { - CapPtr p = message_queue().destroy(); + auto p = message_queue().destroy(); while (p != nullptr) { bool need_post = true; // Always going to post, so ignore. - auto n = p->non_atomic_next; + auto n = p->atomic_read_next(key_global); auto& entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 4813791..50dbc04 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -41,7 +41,18 @@ namespace snmalloc { - struct Remote; + struct FreeListKey + { + address_t key; + + FreeListKey(uint64_t key_) + { + if constexpr (bits::BITS == 64) + key = static_cast(key_); + else + key = key_ & 0xffff'ffff; + } + }; /** * This function is used to sign back pointers in the free list. @@ -53,11 +64,11 @@ namespace snmalloc * list. */ inline static uintptr_t - signed_prev(address_t curr, address_t next, LocalEntropy& entropy) + signed_prev(address_t curr, address_t next, FreeListKey& key) { auto c = curr; auto n = next; - auto k = entropy.get_constant_key(); + auto k = key.key; return (c + k) * (n - k); } @@ -66,10 +77,19 @@ namespace snmalloc * There is an optional second field that is effectively a * back pointer in a doubly linked list, however, it is encoded * to prevent corruption. + * + * TODO: Consider put prev_encoded at the end of the object, would + * require size to be threaded through, but would provide more OOB + * detection. */ class FreeObject { - CapPtr next_object; + union + { + CapPtr next_object; + // TODO: Should really use C++20 atomic_ref rather than a union. + AtomicCapPtr atomic_next_object; + }; #ifdef CHECK_CLIENT // Encoded representation of a back pointer. // Hard to fake, and provides consistency on @@ -83,39 +103,66 @@ namespace snmalloc return p.template as_static(); } - /** - * Construct a FreeObject for local slabs from a Remote message. - */ - static CapPtr make(CapPtr p) - { - // TODO: Zero the difference between a FreeObject and a Remote - return p.template as_reinterpret(); - } - /** * Assign next_object and update its prev_encoded if CHECK_CLIENT. - * * Static so that it can be used on reference to a FreeObject. * * Returns a pointer to the next_object field of the next parameter as an * optimization for repeated snoc operations (in which * next->next_object is nullptr). */ - static CapPtr* store( + static CapPtr* store_next( CapPtr* curr, CapPtr next, - LocalEntropy& entropy) + FreeListKey& key) { - *curr = next; #ifdef CHECK_CLIENT next->prev_encoded = - signed_prev(address_cast(curr), address_cast(next), entropy); + signed_prev(address_cast(curr), address_cast(next), key); #else - UNUSED(entropy); + UNUSED(key); #endif + *curr = next; return &(next->next_object); } + /** + * Assign next_object and update its prev_encoded if CHECK_CLIENT + * + * Uses the atomic view of next, so can be used in the message queues. + */ + void atomic_store_next(CapPtr next, FreeListKey& key) + { +#ifdef CHECK_CLIENT + next->prev_encoded = + signed_prev(address_cast(this), address_cast(next), key); +#else + UNUSED(key); +#endif + // Signature needs to be visible before item is linked in + // so requires release semantics. + atomic_next_object.store(next, std::memory_order_release); + } + + void atomic_store_null() + { + atomic_next_object.store(nullptr, std::memory_order_relaxed); + } + + CapPtr atomic_read_next(FreeListKey& key) + { + auto n = atomic_next_object.load(std::memory_order_acquire); +#ifdef CHECK_CLIENT + if (n != nullptr) + { + n->check_prev(signed_prev(address_cast(this), address_cast(n), key)); + } +#else + UNUSED(key); +#endif + return n; + } + /** * Check the signature of this FreeObject */ @@ -135,6 +182,10 @@ namespace snmalloc } }; + static_assert( + sizeof(FreeObject) <= MIN_ALLOC_SIZE, + "Needs to be able to fit in smallest allocation."); + /** * Used to iterate a free list in object space. * @@ -179,7 +230,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - CapPtr take(LocalEntropy& entropy) + CapPtr take(FreeListKey& key) { auto c = curr; auto next = curr->read_next(); @@ -188,9 +239,9 @@ namespace snmalloc curr = next; #ifdef CHECK_CLIENT c->check_prev(prev); - prev = signed_prev(address_cast(c), address_cast(next), entropy); + prev = signed_prev(address_cast(c), address_cast(next), key); #else - UNUSED(entropy); + UNUSED(key); #endif return c; @@ -220,25 +271,23 @@ namespace snmalloc * If RANDOM is set to false, then the code does not perform any * randomisation. */ - template + template class FreeListBuilder { static constexpr size_t LENGTH = RANDOM ? 2 : 1; // Pointer to the first element. - CapPtr head[LENGTH]; + std::array, LENGTH> head; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - CapPtr* end[LENGTH]; - - public: - S s; + std::array*, LENGTH> end{nullptr}; public: constexpr FreeListBuilder() { - init(); + if (INIT) + init(); } /** @@ -257,7 +306,8 @@ namespace snmalloc /** * Adds an element to the builder */ - void add(CapPtr n, LocalEntropy& entropy) + void + add(CapPtr n, FreeListKey& key, LocalEntropy& entropy) { uint32_t index; if constexpr (RANDOM) @@ -265,33 +315,45 @@ namespace snmalloc else index = 0; - end[index] = FreeObject::store(end[index], n, entropy); + end[index] = FreeObject::store_next(end[index], n, key); + } + + /** + * Adds an element to the builder, if we are guaranteed that + * RANDOM is false. This is useful in certain construction + * cases that do not need to introduce randomness, such as + * during the initialation construction of a free list, which + * uses its own algorithm, or during building remote deallocation + * lists, which will be randomised at the other end. + */ + template + std::enable_if_t + add(CapPtr n, FreeListKey& key) + { + static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); + end[0] = FreeObject::store_next(end[0], n, key); } /** * Makes a terminator to a free list. - * - * Termination uses the bottom bit, this allows the next pointer - * to always be to the same slab. */ - SNMALLOC_FAST_PATH void - terminate_list(uint32_t index, LocalEntropy& entropy) + SNMALLOC_FAST_PATH void terminate_list(uint32_t index, FreeListKey& key) { - UNUSED(entropy); + UNUSED(key); *end[index] = nullptr; } - address_t get_fake_signed_prev(uint32_t index, LocalEntropy& entropy) + address_t get_fake_signed_prev(uint32_t index, FreeListKey& key) { return signed_prev( - address_cast(&head[index]), address_cast(head[index]), entropy); + address_cast(&head[index]), address_cast(head[index]), key); } /** * Close a free list, and set the iterator parameter * to iterate it. */ - SNMALLOC_FAST_PATH void close(FreeListIter& fl, LocalEntropy& entropy) + SNMALLOC_FAST_PATH void close(FreeListIter& fl, FreeListKey& key) { if constexpr (RANDOM) { @@ -303,27 +365,27 @@ namespace snmalloc { // The start token has been corrupted. // TOCTTOU issue, but small window here. - head[1]->check_prev(get_fake_signed_prev(1, entropy)); + head[1]->check_prev(get_fake_signed_prev(1, key)); - terminate_list(1, entropy); + terminate_list(1, key); // Append 1 to 0 - FreeObject::store(end[0], head[1], entropy); + FreeObject::store_next(end[0], head[1], key); SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); } else { - terminate_list(0, entropy); + terminate_list(0, key); } } else { - terminate_list(0, entropy); + terminate_list(0, key); } - fl = {head[0], get_fake_signed_prev(0, entropy)}; + fl = {head[0], get_fake_signed_prev(0, key)}; init(); } @@ -337,5 +399,22 @@ namespace snmalloc end[i] = &head[i]; } } + + std::pair, CapPtr> + extract_segment() + { + SNMALLOC_ASSERT(!empty()); + SNMALLOC_ASSERT(!RANDOM); // TODO: Turn this into a static failure. + + auto first = head[0]; + // end[0] is pointing to the first field in the object, + // this is doing a CONTAINING_RECORD like cast to get back + // to the actual object. This isn't true if the builder is + // empty, but you are not allowed to call this in the empty case. + auto last = + CapPtr(reinterpret_cast(end[0])); + init(); + return {first, last}; + } }; } // namespace snmalloc diff --git a/src/mem/globalconfig.h b/src/mem/globalconfig.h index d87e75a..981f09a 100644 --- a/src/mem/globalconfig.h +++ b/src/mem/globalconfig.h @@ -73,6 +73,9 @@ namespace snmalloc if (initialised) return; + // Initialise key for remote deallocation lists + key_global = FreeListKey(get_entropy64()); + // Need to initialise pagemap. backend_state.init(); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 9a67491..c42685e 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -228,7 +228,7 @@ namespace snmalloc MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), CapPtr(p)); + entry.get_remote()->trunc_id(), CapPtr(p), key_global); post_remote_cache(); return; } @@ -249,7 +249,7 @@ namespace snmalloc */ auto& message_queue() { - return local_cache.remote_allocator->message_queue; + return local_cache.remote_allocator; } public: @@ -382,7 +382,9 @@ namespace snmalloc if (local_cache.remote_dealloc_cache.reserve_space(entry)) { local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), CapPtr(p)); + entry.get_remote()->trunc_id(), + CapPtr(p), + key_global); #ifdef SNMALLOC_TRACING std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p) << std::endl; diff --git a/src/mem/localcache.h b/src/mem/localcache.h index ba364ad..2070966 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -76,6 +76,8 @@ namespace snmalloc typename SharedStateHandle> bool flush(DeallocFun dealloc, SharedStateHandle handle) { + FreeListKey key(entropy.get_constant_key()); + // Return all the free lists to the allocator. // Used during thread teardown for (size_t i = 0; i < NUM_SIZECLASSES; i++) @@ -84,25 +86,27 @@ namespace snmalloc // call. while (!small_fast_free_lists[i].empty()) { - auto p = small_fast_free_lists[i].take(entropy); + auto p = small_fast_free_lists[i].take(key); dealloc(finish_alloc_no_zero(p, i)); } } return remote_dealloc_cache.post( - handle, remote_allocator->trunc_id()); + handle, remote_allocator->trunc_id(), key_global); } template SNMALLOC_FAST_PATH void* alloc(size_t size, Slowpath slowpath) { + FreeListKey key(entropy.get_constant_key()); + sizeclass_t sizeclass = size_to_sizeclass(size); stats.alloc_request(size); stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; if (likely(!fl.empty())) { - auto p = fl.take(entropy); + auto p = fl.take(key); return finish_alloc(p, sizeclass); } return slowpath(sizeclass, &fl); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index f341064..9d774e2 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -18,34 +18,6 @@ namespace snmalloc sizeof(SlabLink) <= MIN_ALLOC_SIZE, "Need to be able to pack a SlabLink into any free small alloc"); - /** - * This struct is used inside FreeListBuilder to account for the - * alignment space that is wasted in sizeof. - * - * This is part of Metaslab abstraction. - */ - struct MetaslabEnd - { - /** - * The number of deallocation required until we hit a slow path. This - * counts down in two different ways that are handled the same on the - * fast path. The first is - * - deallocations until the slab has sufficient entries to be considered - * useful to allocate from. This could be as low as 1, or when we have - * a requirement for entropy then it could be much higher. - * - deallocations until the slab is completely unused. This is needed - * to be detected, so that the statistics can be kept up to date, and - * potentially return memory to the a global pool of slabs/chunks. - */ - uint16_t needed = 0; - - /** - * Flag that is used to indicate that the slab is currently not active. - * I.e. it is not in a CoreAllocator cache for the appropriate sizeclass. - */ - bool sleeping = false; - }; - // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. class alignas(CACHELINE_SIZE) Metaslab : public SlabLink @@ -59,19 +31,38 @@ namespace snmalloc * Spare 32bits are used for the fields in MetaslabEnd. */ #ifdef CHECK_CLIENT - FreeListBuilder free_queue; + FreeListBuilder free_queue; #else - FreeListBuilder free_queue; + FreeListBuilder free_queue; #endif + /** + * The number of deallocation required until we hit a slow path. This + * counts down in two different ways that are handled the same on the + * fast path. The first is + * - deallocations until the slab has sufficient entries to be considered + * useful to allocate from. This could be as low as 1, or when we have + * a requirement for entropy then it could be much higher. + * - deallocations until the slab is completely unused. This is needed + * to be detected, so that the statistics can be kept up to date, and + * potentially return memory to the a global pool of slabs/chunks. + */ + uint16_t needed_ = 0; + + /** + * Flag that is used to indicate that the slab is currently not active. + * I.e. it is not in a CoreAllocator cache for the appropriate sizeclass. + */ + bool sleeping_ = false; + uint16_t& needed() { - return free_queue.s.needed; + return needed_; } bool& sleeping() { - return free_queue.s.sleeping; + return sleeping_; } /** @@ -151,13 +142,17 @@ namespace snmalloc LocalEntropy& entropy, sizeclass_t sizeclass) { + FreeListKey key(entropy.get_constant_key()); + FreeListIter tmp_fl; - meta->free_queue.close(tmp_fl, entropy); - auto p = tmp_fl.take(entropy); + meta->free_queue.close(tmp_fl, key); + auto p = tmp_fl.take(key); fast_free_list = tmp_fl; #ifdef CHECK_CLIENT entropy.refresh_bits(); +#else + UNUSED(entropy); #endif // Treat stealing the free list as allocating it all. diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index c639e5c..d3c1a73 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/mpscq.h" #include "../mem/allocconfig.h" #include "../mem/freelist.h" #include "../mem/metaslab.h" @@ -9,59 +8,102 @@ #include #include -#ifdef CHECK_CLIENT -# define SNMALLOC_DONT_CACHE_ALLOCATOR_PTR -#endif - namespace snmalloc { - /* - * A region of memory destined for a remote allocator's dealloc() via the - * message passing system. This structure is placed at the beginning of - * the allocation itself when it is queued for sending. - */ - struct Remote - { - using alloc_id_t = size_t; - union - { - CapPtr non_atomic_next; - AtomicCapPtr next{nullptr}; - }; - - constexpr Remote() : next(nullptr) {} - - /** Zero out a Remote tracking structure, return pointer to object base */ - template - SNMALLOC_FAST_PATH static CapPtr clear(CapPtr self) - { - pal_zero(self, sizeof(Remote)); - return self.as_void(); - } - }; - - static_assert( - sizeof(Remote) <= MIN_ALLOC_SIZE, - "Needs to be able to fit in smallest allocation."); - // Remotes need to be aligned enough that all the // small size classes can fit in the bottom bits. static constexpr size_t REMOTE_MIN_ALIGN = bits::min( CACHELINE_SIZE, bits::next_pow2_const(NUM_SIZECLASSES + 1)); + /** + * Global key for all remote lists. + */ + inline static FreeListKey key_global(0xdeadbeef); + struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator { - using alloc_id_t = Remote::alloc_id_t; + using alloc_id_t = address_t; + // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. + alignas(CACHELINE_SIZE) AtomicCapPtr back{nullptr}; + // Store the two ends on different cache lines as access by different + // threads. + alignas(CACHELINE_SIZE) CapPtr front{nullptr}; - MPSCQ message_queue; + constexpr RemoteAllocator() = default; + + void invariant() + { + SNMALLOC_ASSERT(back != nullptr); + SNMALLOC_ASSERT(front != nullptr); + } + + void init(CapPtr stub) + { + stub->atomic_store_null(); + front = stub; + back.store(stub, std::memory_order_relaxed); + invariant(); + } + + CapPtr destroy() + { + CapPtr fnt = front; + back.store(nullptr, std::memory_order_relaxed); + front = nullptr; + return fnt; + } + + inline bool is_empty() + { + CapPtr bk = back.load(std::memory_order_relaxed); + + return bk == front; + } + + void enqueue( + CapPtr first, + CapPtr last, + FreeListKey& key) + { + // Pushes a list of messages to the queue. Each message from first to + // last should be linked together through their next pointers. + invariant(); + last->atomic_store_null(); + + // exchange needs to be a release, so nullptr in next is visible. + CapPtr prev = + back.exchange(last, std::memory_order_release); + + prev->atomic_store_next(first, key); + } + + CapPtr peek() + { + return front; + } + + std::pair, bool> dequeue(FreeListKey& key) + { + // Returns the front message, or null if not possible to return a message. + invariant(); + CapPtr first = front; + CapPtr next = first->atomic_read_next(key); + + if (next != nullptr) + { + front = next; + invariant(); + return {first, true}; + } + + return {nullptr, false}; + } alloc_id_t trunc_id() { - return static_cast( - reinterpret_cast(&message_queue)) & - ~SIZECLASS_MASK; + return address_cast(this); } }; } // namespace snmalloc diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index a017b67..af71c22 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/mpscq.h" #include "../mem/allocconfig.h" #include "../mem/freelist.h" #include "../mem/metaslab.h" @@ -17,40 +16,7 @@ namespace snmalloc */ struct RemoteDeallocCache { - /* - * A singly-linked list of Remote objects, supporting append and - * take-all operations. Intended only for the private use of this - * allocator; the Remote objects here will later be taken and pushed - * to the inter-thread message queues. - */ - struct RemoteList - { - /* - * A stub Remote object that will always be the head of this list; - * never taken for further processing. - */ - Remote head{}; - - /** - * Initially is null ptr, and needs to be non-null before anything runs on - * this. - */ - CapPtr last{nullptr}; - - void clear() - { - last = CapPtr(&head); - } - - bool empty() - { - return address_cast(last) == address_cast(&head); - } - - constexpr RemoteList() = default; - }; - - std::array list{}; + std::array, REMOTE_SLOTS> list; /** * The total amount of memory we are waiting for before we will dispatch @@ -98,19 +64,22 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH void - dealloc(Remote::alloc_id_t target_id, CapPtr p) + SNMALLOC_FAST_PATH void dealloc( + RemoteAllocator::alloc_id_t target_id, + CapPtr p, + FreeListKey& key) { SNMALLOC_ASSERT(initialised); - auto r = p.template as_reinterpret(); + auto r = p.template as_reinterpret(); - RemoteList* l = &list[get_slot(target_id, 0)]; - l->last->non_atomic_next = r; - l->last = r; + list[get_slot(target_id, 0)].add(r, key); } template - bool post(SharedStateHandle handle, Remote::alloc_id_t id) + bool post( + SharedStateHandle handle, + RemoteAllocator::alloc_id_t id, + FreeListKey& key) { SNMALLOC_ASSERT(initialised); size_t post_round = 0; @@ -125,46 +94,37 @@ namespace snmalloc if (i == my_slot) continue; - RemoteList* l = &list[i]; - CapPtr first = l->head.non_atomic_next; - - if (!l->empty()) + if (!list[i].empty()) { + auto [first, last] = list[i].extract_segment(); MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(first)); - entry.get_remote()->message_queue.enqueue(first, l->last); - l->clear(); + entry.get_remote()->enqueue(first, last, key); sent_something = true; } } - RemoteList* resend = &list[my_slot]; - if (resend->empty()) + if (list[my_slot].empty()) break; // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - CapPtr r = resend->head.non_atomic_next; - resend->last->non_atomic_next = nullptr; - resend->clear(); + FreeListIter resend; + list[my_slot].close(resend, key); post_round++; - while (r != nullptr) + while (!resend.empty()) { // Use the next N bits to spread out remote deallocs in our own // slot. + auto r = resend.take(key); MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(r)); auto i = entry.get_remote()->trunc_id(); - // TODO correct size for slot offset size_t slot = get_slot(i, post_round); - RemoteList* l = &list[slot]; - l->last->non_atomic_next = r; - l->last = r; - - r = r->non_atomic_next; + list[slot].add(r, key); } } @@ -190,8 +150,7 @@ namespace snmalloc #endif for (auto& l : list) { - SNMALLOC_ASSERT(l.last == nullptr || l.empty()); - l.clear(); + l.init(); } capacity = REMOTE_CACHE; }