diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 881ce10..7870d73 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -161,8 +161,9 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - auto dummy = capptr::AllocFull(small_alloc_one(MIN_ALLOC_SIZE)) - .template as_static(); + auto dummy = + capptr::AllocFull(small_alloc_one(MIN_ALLOC_SIZE)) + .template as_static>(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -181,9 +182,11 @@ namespace snmalloc SNMALLOC_ASSERT(attached_cache != nullptr); // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( - size, [&](sizeclass_t sizeclass, FreeListIter* fl) { - return small_alloc(sizeclass, *fl); - }); + size, + [&]( + sizeclass_t sizeclass, + FreeListIter* + fl) { return small_alloc(sizeclass, *fl); }); } static SNMALLOC_FAST_PATH void alloc_new_list( @@ -244,7 +247,10 @@ namespace snmalloc auto curr_ptr = start_ptr; do { - b.add(FreeObject::make(curr_ptr.as_void()), key, entropy); + b.add( + FreeObject::make(curr_ptr.as_void()), + key, + entropy); curr_ptr = curr_ptr->next; } while (curr_ptr != start_ptr); #else @@ -252,7 +258,9 @@ namespace snmalloc do { b.add( - Aal::capptr_bound(p, rsize), + Aal::capptr_bound< + FreeObject::T, + capptr::bounds::AllocFull>(p, rsize), key); p = pointer_offset(p, rsize); } while (p < slab_end); @@ -264,7 +272,7 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter fl; + FreeListIter fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); void* p = finish_alloc_no_zero(fl.take(key), sizeclass); @@ -426,7 +434,9 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaEntry& entry, capptr::AllocFull p, bool& need_post) + const MetaEntry& entry, + FreeObject::QueuePtr p, + bool& need_post) { // TODO this needs to not double count stats // TODO this needs to not double revoke if using MTE @@ -580,7 +590,8 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = capptr::AllocFull(reinterpret_cast(p)); + auto cp = FreeObject::QueuePtr( + reinterpret_cast*>(p)); auto& key = entropy.get_free_list_key(); @@ -591,8 +602,10 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH void* - small_alloc(sizeclass_t sizeclass, FreeListIter& fast_free_list) + SNMALLOC_SLOW_PATH void* small_alloc( + sizeclass_t sizeclass, + FreeListIter& + fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -650,7 +663,10 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc_slow( - sizeclass_t sizeclass, FreeListIter& fast_free_list, size_t rsize) + sizeclass_t sizeclass, + FreeListIter& + fast_free_list, + size_t rsize) { // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); @@ -712,6 +728,7 @@ namespace snmalloc auto& entry = SharedStateHandle::Pagemap::get_metaentry( backend_state_ptr(), snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); + // XXX n is not known to be domesticated p = n; } } diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 09db138..dd15f10 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -52,44 +52,136 @@ namespace snmalloc return (c + key.key1) * (n + key.key2); } - /** - * Free objects within each slab point directly to the next. - * There is an optional second field that is effectively a - * back pointer in a doubly linked list, however, it is encoded - * to prevent corruption. - * - * TODO: Consider putting prev_encoded at the end of the object, would - * require size to be threaded through, but would provide more OOB - * detection. - */ class FreeObject { - union - { - capptr::AllocFull next_object; - // TODO: Should really use C++20 atomic_ref rather than a union. - capptr::AtomicAllocFull atomic_next_object; - }; -#ifdef SNMALLOC_CHECK_CLIENT - // Encoded representation of a back pointer. - // Hard to fake, and provides consistency on - // the next pointers. - address_t prev_encoded; -#endif - public: - static capptr::AllocFull make(capptr::AllocFull p) - { - return p.template as_static(); - } + template + class T; /** - * Encode next + * This "inductive step" type -- a queue-annotated pointer to a FreeObject + * containing a queue-annotated pointer -- shows up all over the place. + * Give it a shorter name (FreeObject::QueuePtr) for convenience. */ - inline static capptr::AllocFull encode_next( - address_t curr, - capptr::AllocFull next, - const FreeListKey& key) + template + using QueuePtr = CapPtr, BQueue>; + + /** + * As with QueuePtr, but atomic. + */ + template + using AtomicQueuePtr = AtomicCapPtr, BQueue>; + + /** + * This is the "base case" of that induction. While we can't get rid of the + * two different type parameters (in general), we can at least get rid of a + * bit of the clutter. "FreeObject::HeadPtr" looks a little + * nicer than "CapPtr, BView>". + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + using HeadPtr = CapPtr, BView>; + + /** + * As with HeadPtr, but atomic. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + using AtomicHeadPtr = AtomicCapPtr, BView>; + + /** + * Free objects within each slab point directly to the next. + * There is an optional second field that is effectively a + * back pointer in a doubly linked list, however, it is encoded + * to prevent corruption. + * + * This is an inner class to avoid the need to specify BQueue when calling + * static methods. + * + * Raw C++ pointers to this type are *assumed to be domesticated*. In some + * cases we still explicitly annotate domesticated FreeObject*-s as + * CapPtr<>, but more often CapPtr,B> will have B = A. + * + * TODO: Consider putting prev_encoded at the end of the object, would + * require size to be threaded through, but would provide more OOB + * detection. + */ + template + class T + { + friend class FreeObject; + + union + { + QueuePtr next_object; + // TODO: Should really use C++20 atomic_ref rather than a union. + AtomicQueuePtr atomic_next_object; + }; +#ifdef SNMALLOC_CHECK_CLIENT + // Encoded representation of a back pointer. + // Hard to fake, and provides consistency on + // the next pointers. + address_t prev_encoded; +#endif + + public: + QueuePtr atomic_read_next(const FreeListKey& key) + { + auto n = FreeObject::decode_next( + address_cast(&this->next_object), + this->atomic_next_object.load(std::memory_order_acquire), + key); +#ifdef SNMALLOC_CHECK_CLIENT + // XXX n is not known to be domesticated here + if (n != nullptr) + { + n->check_prev(signed_prev(address_cast(this), address_cast(n), key)); + } +#else + UNUSED(key); +#endif + return n; + } + + /** + * Read the next pointer + */ + QueuePtr read_next(const FreeListKey& key) + { + return FreeObject::decode_next( + address_cast(&this->next_object), this->next_object, key); + } + + /** + * Check the signature of this FreeObject + */ + void check_prev(address_t signed_prev) + { + UNUSED(signed_prev); + check_client( + signed_prev == this->prev_encoded, + "Heap corruption - free list corrupted!"); + } + }; + + // Note the inverted template argument order, since BView is inferable. + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, + SNMALLOC_CONCEPT(capptr::ConceptBound) BView> + static HeadPtr make(CapPtr p) + { + return p.template as_static>(); + } + + private: + /** + * Involutive encryption with raw pointers + */ + template + inline static FreeObject::T* code_next( + address_t curr, FreeObject::T* next, const FreeListKey& key) { // Note we can consider other encoding schemes here. // * XORing curr and next. This doesn't require any key material @@ -100,8 +192,8 @@ namespace snmalloc if constexpr (CHECK_CLIENT && !aal_supports) { - return capptr::AllocFull(reinterpret_cast( - reinterpret_cast(next.unsafe_ptr()) ^ key.key_next)); + return reinterpret_cast*>( + reinterpret_cast(next) ^ key.key_next); } else { @@ -110,19 +202,82 @@ namespace snmalloc } } + public: /** - * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT. - * Static so that it can be used on reference to a FreeObject. + * Encode next. We perform two convenient little bits of type-level + * sleight of hand here: + * + * 1) We convert the provided HeadPtr to a QueuePtr, forgetting BView in + * the result; all the callers write the result through a pointer to a + * QueuePtr, though, strictly, the result itself is no less domesticated + * than the input (even if it is obfuscated). + * + * 2) Speaking of obfuscation, we continue to use a CapPtr<> type even + * though the result is likely not safe to dereference, being an + * obfuscated bundle of bits (on non-CHERI architectures, anyway). That's + * additional motivation to consider the result BQueue-bounded, as that is + * likely (but not necessarily) Wild. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + inline static QueuePtr encode_next( + address_t curr, HeadPtr next, const FreeListKey& key) + { + return QueuePtr(code_next(curr, next.unsafe_ptr(), key)); + } + + /** + * Decode next. While traversing a queue, BView and BQueue here will + * often be equal (i.e., CBAllocExportWild) rather than dichotomous. + * However, we do occasionally decode an actual head pointer, so be + * polymorphic here. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + inline static HeadPtr decode_next( + address_t curr, HeadPtr next, const FreeListKey& key) + { + return HeadPtr(code_next(curr, next.unsafe_ptr(), key)); + } + + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void assert_view_queue_bounds() + { + static_assert( + BView::wildness == capptr::dimension::Wildness::Tame, + "FreeObject View must be domesticated, justifying raw pointers"); + + static_assert( + std::is_same_v< + typename BQueue::template with_wildness< + capptr::dimension::Wildness::Tame>, + BView>, + "FreeObject Queue bounds must match View bounds (but may be Wild)"); + } + + /** + * Assign next_object and update its prev_encoded if + * SNMALLOC_CHECK_CLIENT. Static so that it can be used on reference to a + * FreeObject. * * Returns a pointer to the next_object field of the next parameter as an * optimization for repeated snoc operations (in which * next->next_object is nullptr). */ - static capptr::AllocFull* store_next( - capptr::AllocFull* curr, - capptr::AllocFull next, + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static QueuePtr* store_next( + QueuePtr* curr, + HeadPtr next, const FreeListKey& key) { + assert_view_queue_bounds(); + #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = signed_prev(address_cast(curr), address_cast(next), key); @@ -133,10 +288,10 @@ namespace snmalloc return &(next->next_object); } - static void - store_null(capptr::AllocFull* curr, const FreeListKey& key) + template + static void store_null(QueuePtr* curr, const FreeListKey& key) { - *curr = encode_next(address_cast(curr), nullptr, key); + *curr = encode_next(address_cast(curr), QueuePtr(nullptr), key); } /** @@ -144,63 +299,42 @@ namespace snmalloc * * Uses the atomic view of next, so can be used in the message queues. */ - void atomic_store_next( - capptr::AllocFull next, const FreeListKey& key) + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void atomic_store_next( + HeadPtr curr, + HeadPtr next, + const FreeListKey& key) { + static_assert(BView::wildness == capptr::dimension::Wildness::Tame); + #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = - signed_prev(address_cast(this), address_cast(next), key); + signed_prev(address_cast(curr), address_cast(next), key); #else UNUSED(key); #endif // Signature needs to be visible before item is linked in // so requires release semantics. - atomic_next_object.store( - encode_next(address_cast(&next_object), next, key), + curr->atomic_next_object.store( + encode_next(address_cast(&curr->next_object), next, key), std::memory_order_release); } - void atomic_store_null(const FreeListKey& key) + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void + atomic_store_null(HeadPtr curr, const FreeListKey& key) { - atomic_next_object.store( - encode_next(address_cast(&next_object), nullptr, key), + static_assert(BView::wildness == capptr::dimension::Wildness::Tame); + + curr->atomic_next_object.store( + encode_next( + address_cast(&curr->next_object), QueuePtr(nullptr), key), std::memory_order_relaxed); } - - capptr::AllocFull atomic_read_next(const FreeListKey& key) - { - auto n = encode_next( - address_cast(&next_object), - atomic_next_object.load(std::memory_order_acquire), - key); -#ifdef SNMALLOC_CHECK_CLIENT - if (n != nullptr) - { - n->check_prev(signed_prev(address_cast(this), address_cast(n), key)); - } -#else - UNUSED(key); -#endif - return n; - } - - /** - * Check the signature of this FreeObject - */ - void check_prev(address_t signed_prev) - { - UNUSED(signed_prev); - check_client( - signed_prev == prev_encoded, "Heap corruption - free list corrupted!"); - } - - /** - * Read the next pointer - */ - capptr::AllocFull read_next(const FreeListKey& key) - { - return encode_next(address_cast(&next_object), next_object, key); - } }; static_assert( @@ -212,16 +346,19 @@ namespace snmalloc * * Checks signing of pointers */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> class FreeListIter { - capptr::AllocFull curr{nullptr}; + FreeObject::HeadPtr curr{nullptr}; #ifdef SNMALLOC_CHECK_CLIENT address_t prev{0}; #endif public: constexpr FreeListIter( - capptr::AllocFull head, address_t prev_value) + FreeObject::HeadPtr head, address_t prev_value) : curr(head) { #ifdef SNMALLOC_CHECK_CLIENT @@ -243,7 +380,7 @@ namespace snmalloc /** * Returns current head without affecting the iterator. */ - capptr::AllocFull peek() + FreeObject::HeadPtr peek() { return curr; } @@ -251,7 +388,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - capptr::AllocFull take(const FreeListKey& key) + FreeObject::HeadPtr take(const FreeListKey& key) { auto c = curr; auto next = curr->read_next(key); @@ -286,17 +423,49 @@ namespace snmalloc * If RANDOM is set to false, then the code does not perform any * randomisation. */ - template + template< + bool RANDOM, + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, + bool INIT = true> class FreeListBuilder { static constexpr size_t LENGTH = RANDOM ? 2 : 1; + /* + * We use native pointers below so that we don't run afoul of strict + * aliasing rules. head is a FreeObject::HeadPtr -- that + * is, a known-domesticated pointer to a queue of wild pointers -- and + * it's usually the case that end is a FreeObject::QueuePtr* -- + * that is, a known-domesticated pointer to a wild pointer to a queue of + * wild pointers. However, in order to do branchless inserts, we set end + * = &head, which breaks strict aliasing rules with the types as given. + * Fortunately, these are private members and so we can use native + * pointers and just expose a more strongly typed interface. + */ + // Pointer to the first element. - std::array, LENGTH> head; + std::array head{nullptr}; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - std::array*, LENGTH> end{nullptr}; + std::array end{nullptr}; + + FreeObject::QueuePtr* cast_end(uint32_t ix) + { + return reinterpret_cast*>(end[ix]); + } + + void set_end(uint32_t ix, FreeObject::QueuePtr* p) + { + end[ix] = reinterpret_cast(p); + } + + FreeObject::HeadPtr cast_head(uint32_t ix) + { + return FreeObject::HeadPtr( + static_cast*>(head[ix])); + } std::array length{}; @@ -304,7 +473,9 @@ namespace snmalloc constexpr FreeListBuilder() { if (INIT) + { init(); + } } /** @@ -314,8 +485,10 @@ namespace snmalloc { for (size_t i = 0; i < LENGTH; i++) { - if (address_cast(end[i]) != address_cast(&head[i])) + if (end[i] != &head[i]) + { return false; + } } return true; } @@ -324,7 +497,7 @@ namespace snmalloc * Adds an element to the builder */ void add( - capptr::AllocFull n, + FreeObject::HeadPtr n, const FreeListKey& key, LocalEntropy& entropy) { @@ -334,7 +507,7 @@ namespace snmalloc else index = 0; - end[index] = FreeObject::store_next(end[index], n, key); + set_end(index, FreeObject::store_next(cast_end(index), n, key)); if constexpr (RANDOM) { length[index]++; @@ -351,10 +524,10 @@ namespace snmalloc */ template std::enable_if_t - add(capptr::AllocFull n, const FreeListKey& key) + add(FreeObject::HeadPtr n, const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); - end[0] = FreeObject::store_next(end[0], n, key); + set_end(0, FreeObject::store_next(cast_end(0), n, key)); } /** @@ -363,7 +536,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void terminate_list(uint32_t index, const FreeListKey& key) { - FreeObject::store_null(end[index], key); + FreeObject::store_null(cast_end(index), key); } /** @@ -375,11 +548,11 @@ namespace snmalloc * and is thus subject to encoding if the next_object pointers * encoded. */ - capptr::AllocFull + FreeObject::HeadPtr read_head(uint32_t index, const FreeListKey& key) { - return FreeObject::encode_next( - address_cast(&head[index]), head[index], key); + return FreeObject::decode_next( + address_cast(&head[index]), cast_head(index), key); } address_t get_fake_signed_prev(uint32_t index, const FreeListKey& key) @@ -396,7 +569,8 @@ namespace snmalloc * * The return value is how many entries are still contained in the builder. */ - SNMALLOC_FAST_PATH uint16_t close(FreeListIter& fl, const FreeListKey& key) + SNMALLOC_FAST_PATH uint16_t + close(FreeListIter& fl, const FreeListKey& key) { uint32_t i; if constexpr (RANDOM) @@ -447,7 +621,9 @@ namespace snmalloc template std::enable_if_t< !RANDOM_, - std::pair, capptr::AllocFull>> + std::pair< + FreeObject::HeadPtr, + FreeObject::HeadPtr>> extract_segment(const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); @@ -458,8 +634,8 @@ namespace snmalloc // this is doing a CONTAINING_RECORD like cast to get back // to the actual object. This isn't true if the builder is // empty, but you are not allowed to call this in the empty case. - auto last = - capptr::AllocFull(reinterpret_cast(end[0])); + auto last = FreeObject::HeadPtr( + reinterpret_cast*>(end[0])); init(); return {first, last}; } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index b20b3d3..ac183f0 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,25 +211,33 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); - auto slowpath = [&]( - sizeclass_t sizeclass, - FreeListIter* fl) SNMALLOC_FAST_PATH_LAMBDA { - if (likely(core_alloc != nullptr)) - { - return core_alloc->handle_message_queue( - [](CoreAlloc* core_alloc, sizeclass_t sizeclass, FreeListIter* fl) { - return core_alloc->template small_alloc(sizeclass, *fl); + auto slowpath = + [&]( + sizeclass_t sizeclass, + FreeListIter* + fl) SNMALLOC_FAST_PATH_LAMBDA { + if (likely(core_alloc != nullptr)) + { + return core_alloc->handle_message_queue( + []( + CoreAlloc* core_alloc, + sizeclass_t sizeclass, + FreeListIter< + capptr::bounds::AllocFull, + capptr::bounds::AllocFull>* fl) { + return core_alloc->template small_alloc( + sizeclass, *fl); + }, + core_alloc, + sizeclass, + fl); + } + return lazy_init( + [&](CoreAlloc*, sizeclass_t sizeclass) { + return small_alloc(sizeclass_to_size(sizeclass)); }, - core_alloc, - sizeclass, - fl); - } - return lazy_init( - [&](CoreAlloc*, sizeclass_t sizeclass) { - return small_alloc(sizeclass_to_size(sizeclass)); - }, - sizeclass); - }; + sizeclass); + }; return local_cache.template alloc( size, slowpath); diff --git a/src/mem/localcache.h b/src/mem/localcache.h index a655cfb..8184141 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -12,8 +12,9 @@ namespace snmalloc { using Stats = AllocStats; - inline static SNMALLOC_FAST_PATH void* - finish_alloc_no_zero(capptr::AllocFull p, sizeclass_t sizeclass) + inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero( + FreeObject::HeadPtr p, + sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -24,8 +25,9 @@ namespace snmalloc } template - inline static SNMALLOC_FAST_PATH void* - finish_alloc(capptr::AllocFull p, sizeclass_t sizeclass) + inline static SNMALLOC_FAST_PATH void* finish_alloc( + FreeObject::HeadPtr p, + sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); @@ -46,7 +48,8 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - FreeListIter small_fast_free_lists[NUM_SIZECLASSES]; + FreeListIter + small_fast_free_lists[NUM_SIZECLASSES]; // This is the entropy for a particular thread. LocalEntropy entropy; diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index a6015b2..9a7165b 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -25,9 +25,11 @@ namespace snmalloc * Data-structure for building the free list for this slab. */ #ifdef SNMALLOC_CHECK_CLIENT - FreeListBuilder free_queue; + FreeListBuilder + free_queue; #else - FreeListBuilder free_queue; + FreeListBuilder + free_queue; #endif /** @@ -152,16 +154,18 @@ namespace snmalloc * component, but with randomisation, it may only return part of the * available objects for this metaslab. */ - static SNMALLOC_FAST_PATH std::pair, bool> - alloc_free_list( - Metaslab* meta, - FreeListIter& fast_free_list, - LocalEntropy& entropy, - sizeclass_t sizeclass) + static SNMALLOC_FAST_PATH + std::pair, bool> + alloc_free_list( + Metaslab* meta, + FreeListIter& + fast_free_list, + LocalEntropy& entropy, + sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter tmp_fl; + std::remove_reference_t tmp_fl; auto remaining = meta->free_queue.close(tmp_fl, key); auto p = tmp_fl.take(key); fast_free_list = tmp_fl; diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index f9596c7..2e4e735 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -35,10 +35,12 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) capptr::AtomicAllocFull back{nullptr}; + alignas(CACHELINE_SIZE) + FreeObject::AtomicQueuePtr back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) capptr::AllocFull front{nullptr}; + alignas(CACHELINE_SIZE) + FreeObject::QueuePtr front{nullptr}; constexpr RemoteAllocator() = default; @@ -48,17 +50,17 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void init(capptr::AllocFull stub) + void init(FreeObject::QueuePtr stub) { - stub->atomic_store_null(key_global); + FreeObject::atomic_store_null(stub, key_global); front = stub; back.store(stub, std::memory_order_relaxed); invariant(); } - capptr::AllocFull destroy() + FreeObject::QueuePtr destroy() { - capptr::AllocFull fnt = front; + FreeObject::QueuePtr fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -66,7 +68,8 @@ namespace snmalloc inline bool is_empty() { - capptr::AllocFull bk = back.load(std::memory_order_relaxed); + FreeObject::QueuePtr bk = + back.load(std::memory_order_relaxed); return bk == front; } @@ -76,21 +79,22 @@ namespace snmalloc * last should be linked together through their next pointers. */ void enqueue( - capptr::AllocFull first, - capptr::AllocFull last, + FreeObject::QueuePtr first, + FreeObject::QueuePtr last, const FreeListKey& key) { invariant(); - last->atomic_store_null(key); + FreeObject::atomic_store_null(last, key); // exchange needs to be a release, so nullptr in next is visible. - capptr::AllocFull prev = + FreeObject::QueuePtr prev = back.exchange(last, std::memory_order_release); - prev->atomic_store_next(first, key); + // XXX prev is not known to be domesticated + FreeObject::atomic_store_next(prev, first, key); } - capptr::AllocFull peek() + FreeObject::QueuePtr peek() { return front; } @@ -98,12 +102,13 @@ namespace snmalloc /** * Returns the front message, or null if not possible to return a message. */ - std::pair, bool> + std::pair, bool> dequeue(const FreeListKey& key) { invariant(); - capptr::AllocFull first = front; - capptr::AllocFull next = first->atomic_read_next(key); + FreeObject::QueuePtr first = front; + FreeObject::QueuePtr next = + first->atomic_read_next(key); if (next != nullptr) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 0bf11ed..7bcc57a 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -16,7 +16,14 @@ namespace snmalloc */ struct RemoteDeallocCache { - std::array, REMOTE_SLOTS> list; + std::array< + FreeListBuilder< + false, + capptr::bounds::AllocFull, + capptr::bounds::AllocFull, + false>, + REMOTE_SLOTS> + list; /** * The total amount of memory we are waiting for before we will dispatch @@ -70,7 +77,8 @@ namespace snmalloc const FreeListKey& key) { SNMALLOC_ASSERT(initialised); - auto r = p.template as_reinterpret(); + auto r = + p.template as_reinterpret>(); list[get_slot(target_id, 0)].add(r, key); } @@ -110,7 +118,8 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - FreeListIter resend; + FreeListIter + resend; list[my_slot].close(resend, key); post_round++;