From 96155db64041171c43f34d5798d1f57612249f9e Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Fri, 8 Oct 2021 16:45:00 +0100 Subject: [PATCH] Collect freelist things in a namespace Motivated by renaming `FreeObject::{Head,Queue,AtomicQueue}Ptr` to `freelist::...Ptr`, in fact go further, moving `FreeObject` itself to `freelist::Object` and `FreeListBuilder` to `freelist::Builder` and `FreeListIter` to `freelist::Iter` --- src/mem/corealloc.h | 28 +- src/mem/freelist.h | 1162 +++++++++++++++++++------------------ src/mem/localalloc.h | 6 +- src/mem/localcache.h | 10 +- src/mem/metaslab.h | 8 +- src/mem/remoteallocator.h | 32 +- src/mem/remotecache.h | 8 +- 7 files changed, 637 insertions(+), 617 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 611827c..0188e91 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -162,7 +162,7 @@ namespace snmalloc // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. auto dummy = capptr::Alloc(small_alloc_one(MIN_ALLOC_SIZE)) - .template as_static>(); + .template as_static>(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -180,12 +180,12 @@ namespace snmalloc { SNMALLOC_ASSERT(attached_cache != nullptr); auto domesticate = - [this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(backend_state_ptr(), p); }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( - domesticate, size, [&](sizeclass_t sizeclass, FreeListIter<>* fl) { + domesticate, size, [&](sizeclass_t sizeclass, freelist::Iter<>* fl) { return small_alloc(sizeclass, *fl); }); } @@ -250,7 +250,7 @@ namespace snmalloc { b.add( // Here begins our treatment of the heap as containing Wild pointers - FreeObject::make( + freelist::Object::make( capptr_to_user_address_control(curr_ptr.as_void())), key, entropy); @@ -262,7 +262,7 @@ namespace snmalloc { b.add( // Here begins our treatment of the heap as containing Wild pointers - FreeObject::make( + freelist::Object::make( capptr_to_user_address_control( Aal::capptr_bound( p.as_void(), rsize))), @@ -277,12 +277,12 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter<> fl; + freelist::Iter<> fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); @@ -415,7 +415,7 @@ namespace snmalloc bool need_post = false; auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; for (size_t i = 0; i < REMOTE_BATCH; i++) @@ -608,7 +608,7 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = p.as_static>(); + auto cp = p.as_static>(); auto& key = entropy.get_free_list_key(); @@ -620,7 +620,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* - small_alloc(sizeclass_t sizeclass, FreeListIter<>& fast_free_list) + small_alloc(sizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -645,7 +645,7 @@ namespace snmalloc alloc_classes[sizeclass].unused--; auto domesticate = [this]( - FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(backend_state_ptr(), p); }; auto [p, still_active] = Metaslab::alloc_free_list( @@ -682,7 +682,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc_slow( - sizeclass_t sizeclass, FreeListIter<>& fast_free_list, size_t rsize) + sizeclass_t sizeclass, freelist::Iter<>& fast_free_list, size_t rsize) { // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); @@ -713,7 +713,7 @@ namespace snmalloc alloc_new_list(slab, meta, rsize, slab_size, entropy); auto domesticate = - [this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(backend_state_ptr(), p); }; auto [p, still_active] = Metaslab::alloc_free_list( @@ -738,7 +738,7 @@ namespace snmalloc SNMALLOC_ASSERT(attached_cache != nullptr); auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; diff --git a/src/mem/freelist.h b/src/mem/freelist.h index a002c00..b58978a 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -9,7 +9,7 @@ * * The corruption detection works as follows * - * FreeObject + * free Object * ----------------------------- * | next | prev_encoded | ... | * ----------------------------- @@ -52,633 +52,653 @@ namespace snmalloc return (c + key.key1) * (n + key.key2); } - class FreeObject + namespace freelist { - public: - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> - class T; - - /** - * This "inductive step" type -- a queue-annotated pointer to a FreeObject - * containing a queue-annotated pointer -- shows up all over the place. - * Give it a shorter name (FreeObject::BQueuePtr) for convenience. - */ - template - using BQueuePtr = CapPtr, BQueue>; - - /** - * In particular, external code almost always uses AllocWild as the queue - * annotation. Make their lives easier. - */ - using QueuePtr = BQueuePtr; - - /** - * As with BQueuePtr, but atomic. - */ - template - using BAtomicQueuePtr = AtomicCapPtr, BQueue>; - - using AtomicQueuePtr = BAtomicQueuePtr; - - /** - * This is the "base case" of that induction. While we can't get rid of the - * two different type parameters (in general), we can at least get rid of a - * bit of the clutter. "FreeObject::HeadPtr" looks a little - * nicer than "CapPtr, BView>". - */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - using BHeadPtr = CapPtr, BView>; - - using HeadPtr = BHeadPtr; - - /** - * As with HeadPtr, but atomic. - */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - using AtomicHeadPtr = AtomicCapPtr, BView>; - - /** - * Free objects within each slab point directly to the next. - * There is an optional second field that is effectively a - * back pointer in a doubly linked list, however, it is encoded - * to prevent corruption. - * - * This is an inner class to avoid the need to specify BQueue when calling - * static methods. - * - * Raw C++ pointers to this type are *assumed to be domesticated*. In some - * cases we still explicitly annotate domesticated FreeObject*-s as - * CapPtr<>, but more often CapPtr,B> will have B = A. - * - * TODO: Consider putting prev_encoded at the end of the object, would - * require size to be threaded through, but would provide more OOB - * detection. - */ - template - class T + class Object { - friend class FreeObject; - - union - { - BQueuePtr next_object; - // TODO: Should really use C++20 atomic_ref rather than a union. - BAtomicQueuePtr atomic_next_object; - }; -#ifdef SNMALLOC_CHECK_CLIENT - // Encoded representation of a back pointer. - // Hard to fake, and provides consistency on - // the next pointers. - address_t prev_encoded; -#endif - public: template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: - template with_wildness, - typename Domesticator> - BHeadPtr - atomic_read_next(const FreeListKey& key, Domesticator domesticate) - { - auto n_wild = FreeObject::decode_next( - address_cast(&this->next_object), - this->atomic_next_object.load(std::memory_order_acquire), - key); - auto n_tame = domesticate(n_wild); -#ifdef SNMALLOC_CHECK_CLIENT - if (n_tame != nullptr) - { - n_tame->check_prev( - signed_prev(address_cast(this), address_cast(n_tame), key)); - } -#endif - return n_tame; - } + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = + capptr::bounds::AllocWild> + class T; /** - * Read the next pointer + * This "inductive step" type -- a queue-annotated pointer to a free + * Object containing a queue-annotated pointer -- shows up all over the + * place. Give it a shorter name (Object::BQueuePtr) for + * convenience. + */ + template + using BQueuePtr = CapPtr, BQueue>; + + /** + * As with BQueuePtr, but atomic. + */ + template + using BAtomicQueuePtr = AtomicCapPtr, BQueue>; + + /** + * This is the "base case" of that induction. While we can't get rid of + * the two different type parameters (in general), we can at least get rid + * of a bit of the clutter. "freelist::Object::HeadPtr" + * looks a little nicer than "CapPtr, BView>". */ template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: - template with_wildness, - typename Domesticator> - BHeadPtr - read_next(const FreeListKey& key, Domesticator domesticate) + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + using BHeadPtr = CapPtr, BView>; + + /** + * As with BHeadPtr, but atomic. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + using BAtomicHeadPtr = AtomicCapPtr, BView>; + + /** + * Free objects within each slab point directly to the next. + * There is an optional second field that is effectively a + * back pointer in a doubly linked list, however, it is encoded + * to prevent corruption. + * + * This is an inner class to avoid the need to specify BQueue when calling + * static methods. + * + * Raw C++ pointers to this type are *assumed to be domesticated*. In + * some cases we still explicitly annotate domesticated free Object*-s as + * CapPtr<>, but more often CapPtr,B> will have B = A. + * + * TODO: Consider putting prev_encoded at the end of the object, would + * require size to be threaded through, but would provide more OOB + * detection. + */ + template + class T { - return domesticate(FreeObject::decode_next( - address_cast(&this->next_object), this->next_object, key)); + friend class Object; + + union + { + BQueuePtr next_object; + // TODO: Should really use C++20 atomic_ref rather than a union. + BAtomicQueuePtr atomic_next_object; + }; +#ifdef SNMALLOC_CHECK_CLIENT + // Encoded representation of a back pointer. + // Hard to fake, and provides consistency on + // the next pointers. + address_t prev_encoded; +#endif + + public: + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: + template with_wildness, + typename Domesticator> + BHeadPtr + atomic_read_next(const FreeListKey& key, Domesticator domesticate) + { + auto n_wild = Object::decode_next( + address_cast(&this->next_object), + this->atomic_next_object.load(std::memory_order_acquire), + key); + auto n_tame = domesticate(n_wild); +#ifdef SNMALLOC_CHECK_CLIENT + if (n_tame != nullptr) + { + n_tame->check_prev( + signed_prev(address_cast(this), address_cast(n_tame), key)); + } +#endif + return n_tame; + } + + /** + * Read the next pointer + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: + template with_wildness, + typename Domesticator> + BHeadPtr + read_next(const FreeListKey& key, Domesticator domesticate) + { + return domesticate(Object::decode_next( + address_cast(&this->next_object), this->next_object, key)); + } + + /** + * Check the signature of this free Object + */ + void check_prev(address_t signed_prev) + { + UNUSED(signed_prev); + check_client( + signed_prev == this->prev_encoded, + "Heap corruption - free list corrupted!"); + } + }; + + // Note the inverted template argument order, since BView is inferable. + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, + SNMALLOC_CONCEPT(capptr::ConceptBound) BView> + static BHeadPtr make(CapPtr p) + { + return p.template as_static>(); } /** - * Check the signature of this FreeObject + * A container-of operation to convert &f->next_object to f */ - void check_prev(address_t signed_prev) + template + static Object::T* + from_next_ptr(CapPtr, BQueue>* ptr) { - UNUSED(signed_prev); - check_client( - signed_prev == this->prev_encoded, - "Heap corruption - free list corrupted!"); + static_assert(offsetof(Object::T, next_object) == 0); + return reinterpret_cast*>(ptr); + } + + private: + /** + * Involutive encryption with raw pointers + */ + template + inline static Object::T* + code_next(address_t curr, Object::T* next, const FreeListKey& key) + { + // Note we can consider other encoding schemes here. + // * XORing curr and next. This doesn't require any key material + // * XORing (curr * key). This makes it harder to guess the underlying + // key, as each location effectively has its own key. + // Curr is not used in the current encoding scheme. + UNUSED(curr); + + if constexpr (CHECK_CLIENT && !aal_supports) + { + return reinterpret_cast*>( + reinterpret_cast(next) ^ key.key_next); + } + else + { + UNUSED(key); + return next; + } + } + + public: + /** + * Encode next. We perform two convenient little bits of type-level + * sleight of hand here: + * + * 1) We convert the provided HeadPtr to a QueuePtr, forgetting BView in + * the result; all the callers write the result through a pointer to a + * QueuePtr, though, strictly, the result itself is no less domesticated + * than the input (even if it is obfuscated). + * + * 2) Speaking of obfuscation, we continue to use a CapPtr<> type even + * though the result is likely not safe to dereference, being an + * obfuscated bundle of bits (on non-CHERI architectures, anyway). That's + * additional motivation to consider the result BQueue-bounded, as that + * is likely (but not necessarily) Wild. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + inline static BQueuePtr encode_next( + address_t curr, BHeadPtr next, const FreeListKey& key) + { + return BQueuePtr(code_next(curr, next.unsafe_ptr(), key)); + } + + /** + * Decode next. While traversing a queue, BView and BQueue here will + * often be equal (i.e., AllocUserWild) rather than dichotomous. However, + * we do occasionally decode an actual head pointer, so be polymorphic + * here. + * + * TODO: We'd like, in some sense, to more tightly couple or integrate + * this into to the domestication process. We could introduce an + * additional state in the capptr_bounds::wild taxonomy (e.g, Obfuscated) + * so that the Domesticator-s below have to call through this function to + * get the Wild pointer they can then make Tame. It's not yet entirely + * clear what that would look like and whether/how the encode_next side of + * things should be exposed. For the moment, obfuscation is left + * encapsulated within Object and we do not capture any of it statically. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + inline static BHeadPtr decode_next( + address_t curr, BHeadPtr next, const FreeListKey& key) + { + return BHeadPtr(code_next(curr, next.unsafe_ptr(), key)); + } + + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void assert_view_queue_bounds() + { + static_assert( + BView::wildness == capptr::dimension::Wildness::Tame, + "Free Object View must be domesticated, justifying raw pointers"); + + static_assert( + std::is_same_v< + typename BQueue::template with_wildness< + capptr::dimension::Wildness::Tame>, + BView>, + "Free Object Queue bounds must match View bounds (but may be Wild)"); + } + + /** + * Assign next_object and update its prev_encoded if + * SNMALLOC_CHECK_CLIENT. Static so that it can be used on reference to a + * free Object. + * + * Returns a pointer to the next_object field of the next parameter as an + * optimization for repeated snoc operations (in which + * next->next_object is nullptr). + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static BQueuePtr* store_next( + BQueuePtr* curr, + BHeadPtr next, + const FreeListKey& key) + { + assert_view_queue_bounds(); + +#ifdef SNMALLOC_CHECK_CLIENT + next->prev_encoded = + signed_prev(address_cast(curr), address_cast(next), key); +#else + UNUSED(key); +#endif + *curr = encode_next(address_cast(curr), next, key); + return &(next->next_object); + } + + template + static void store_null(BQueuePtr* curr, const FreeListKey& key) + { + *curr = + encode_next(address_cast(curr), BQueuePtr(nullptr), key); + } + + /** + * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT + * + * Uses the atomic view of next, so can be used in the message queues. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void atomic_store_next( + BHeadPtr curr, + BHeadPtr next, + const FreeListKey& key) + { + static_assert(BView::wildness == capptr::dimension::Wildness::Tame); + +#ifdef SNMALLOC_CHECK_CLIENT + next->prev_encoded = + signed_prev(address_cast(curr), address_cast(next), key); +#else + UNUSED(key); +#endif + // Signature needs to be visible before item is linked in + // so requires release semantics. + curr->atomic_next_object.store( + encode_next(address_cast(&curr->next_object), next, key), + std::memory_order_release); + } + + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void + atomic_store_null(BHeadPtr curr, const FreeListKey& key) + { + static_assert(BView::wildness == capptr::dimension::Wildness::Tame); + + curr->atomic_next_object.store( + encode_next( + address_cast(&curr->next_object), BQueuePtr(nullptr), key), + std::memory_order_relaxed); } }; - // Note the inverted template argument order, since BView is inferable. + static_assert( + sizeof(Object) <= MIN_ALLOC_SIZE, + "Needs to be able to fit in smallest allocation."); + + /** + * External code almost always uses Alloc and AllocWild for its free lists. + * Give them a convenient alias. + */ + using HeadPtr = + Object::BHeadPtr; + + /** + * Like HeadPtr, but atomic + */ + using AtomicHeadPtr = + Object::BAtomicHeadPtr; + + /** + * External code's inductive cases almost always use AllocWild. + */ + using QueuePtr = Object::BQueuePtr; + + /** + * Like QueuePtr, but atomic + */ + using AtomicQueuePtr = Object::BAtomicQueuePtr; + + /** + * Used to iterate a free list in object space. + * + * Checks signing of pointers + */ template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, - SNMALLOC_CONCEPT(capptr::ConceptBound) BView> - static BHeadPtr make(CapPtr p) + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> + class Iter { - return p.template as_static>(); - } + Object::BHeadPtr curr{nullptr}; +#ifdef SNMALLOC_CHECK_CLIENT + address_t prev{0}; +#endif - /** - * A container-of operation to convert &f->next_object to f - */ - template - static FreeObject::T* - from_next_ptr(CapPtr, BQueue>* ptr) - { - static_assert(offsetof(FreeObject::T, next_object) == 0); - return reinterpret_cast*>(ptr); - } - - private: - /** - * Involutive encryption with raw pointers - */ - template - inline static FreeObject::T* code_next( - address_t curr, FreeObject::T* next, const FreeListKey& key) - { - // Note we can consider other encoding schemes here. - // * XORing curr and next. This doesn't require any key material - // * XORing (curr * key). This makes it harder to guess the underlying - // key, as each location effectively has its own key. - // Curr is not used in the current encoding scheme. - UNUSED(curr); - - if constexpr (CHECK_CLIENT && !aal_supports) + public: + constexpr Iter(Object::BHeadPtr head, address_t prev_value) + : curr(head) { - return reinterpret_cast*>( - reinterpret_cast(next) ^ key.key_next); +#ifdef SNMALLOC_CHECK_CLIENT + prev = prev_value; +#endif + UNUSED(prev_value); } - else + + constexpr Iter() = default; + + /** + * Checks if there are any more values to iterate. + */ + bool empty() { + return curr == nullptr; + } + + /** + * Returns current head without affecting the iterator. + */ + Object::BHeadPtr peek() + { + return curr; + } + + /** + * Moves the iterator on, and returns the current value. + */ + template + Object::BHeadPtr + take(const FreeListKey& key, Domesticator domesticate) + { + auto c = curr; + auto next = curr->read_next(key, domesticate); + + Aal::prefetch(next.unsafe_ptr()); + curr = next; +#ifdef SNMALLOC_CHECK_CLIENT + c->check_prev(prev); + prev = signed_prev(address_cast(c), address_cast(next), key); +#else UNUSED(key); - return next; +#endif + + return c; } - } + }; - public: /** - * Encode next. We perform two convenient little bits of type-level - * sleight of hand here: + * Used to build a free list in object space. * - * 1) We convert the provided HeadPtr to a QueuePtr, forgetting BView in - * the result; all the callers write the result through a pointer to a - * QueuePtr, though, strictly, the result itself is no less domesticated - * than the input (even if it is obfuscated). + * Adds signing of pointers in the SNMALLOC_CHECK_CLIENT mode * - * 2) Speaking of obfuscation, we continue to use a CapPtr<> type even - * though the result is likely not safe to dereference, being an - * obfuscated bundle of bits (on non-CHERI architectures, anyway). That's - * additional motivation to consider the result BQueue-bounded, as that is - * likely (but not necessarily) Wild. - */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - inline static BQueuePtr encode_next( - address_t curr, BHeadPtr next, const FreeListKey& key) - { - return BQueuePtr(code_next(curr, next.unsafe_ptr(), key)); - } - - /** - * Decode next. While traversing a queue, BView and BQueue here will often - * be equal (i.e., AllocUserWild) rather than dichotomous. However, we do - * occasionally decode an actual head pointer, so be polymorphic here. + * If RANDOM is enabled, the builder uses two queues, and + * "randomly" decides to add to one of the two queues. This + * means that we will maintain a randomisation of the order + * between allocations. * - * TODO: We'd like, in some sense, to more tightly couple or integrate this - * into to the domestication process. We could introduce an additional - * state in the capptr_bounds::wild taxonomy (e.g, Obfuscated) so that the - * Domesticator-s below have to call through this function to get the Wild - * pointer they can then make Tame. It's not yet entirely clear what that - * would look like and whether/how the encode_next side of things should be - * exposed. For the moment, obfuscation is left encapsulated within - * FreeObject and we do not capture any of it statically. - */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - inline static BHeadPtr decode_next( - address_t curr, BHeadPtr next, const FreeListKey& key) - { - return BHeadPtr(code_next(curr, next.unsafe_ptr(), key)); - } - - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - static void assert_view_queue_bounds() - { - static_assert( - BView::wildness == capptr::dimension::Wildness::Tame, - "FreeObject View must be domesticated, justifying raw pointers"); - - static_assert( - std::is_same_v< - typename BQueue::template with_wildness< - capptr::dimension::Wildness::Tame>, - BView>, - "FreeObject Queue bounds must match View bounds (but may be Wild)"); - } - - /** - * Assign next_object and update its prev_encoded if - * SNMALLOC_CHECK_CLIENT. Static so that it can be used on reference to a - * FreeObject. + * The fields are paired up to give better codegen as then they are offset + * by a power of 2, and the bit extract from the interleaving seed can + * be shifted to calculate the relevant offset to index the fields. * - * Returns a pointer to the next_object field of the next parameter as an - * optimization for repeated snoc operations (in which - * next->next_object is nullptr). + * If RANDOM is set to false, then the code does not perform any + * randomisation. */ template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - static BQueuePtr* store_next( - BQueuePtr* curr, - BHeadPtr next, - const FreeListKey& key) + bool RANDOM, + bool INIT = true, + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> + class Builder { - assert_view_queue_bounds(); + static constexpr size_t LENGTH = RANDOM ? 2 : 1; -#ifdef SNMALLOC_CHECK_CLIENT - next->prev_encoded = - signed_prev(address_cast(curr), address_cast(next), key); -#else - UNUSED(key); -#endif - *curr = encode_next(address_cast(curr), next, key); - return &(next->next_object); - } + /* + * We use native pointers below so that we don't run afoul of strict + * aliasing rules. head is a Object::HeadPtr -- that is, a + * known-domesticated pointer to a queue of wild pointers -- and it's + * usually the case that end is a Object::BQueuePtr* -- that is, a + * known-domesticated pointer to a wild pointer to a queue of wild + * pointers. However, in order to do branchless inserts, we set end = + * &head, which breaks strict aliasing rules with the types as given. + * Fortunately, these are private members and so we can use native + * pointers and just expose a more strongly typed interface. + */ - template - static void store_null(BQueuePtr* curr, const FreeListKey& key) - { - *curr = encode_next(address_cast(curr), BQueuePtr(nullptr), key); - } + // Pointer to the first element. + std::array head{nullptr}; + // Pointer to the reference to the last element. + // In the empty case end[i] == &head[i] + // This enables branch free enqueuing. + std::array end{nullptr}; - /** - * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT - * - * Uses the atomic view of next, so can be used in the message queues. - */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - static void atomic_store_next( - BHeadPtr curr, - BHeadPtr next, - const FreeListKey& key) - { - static_assert(BView::wildness == capptr::dimension::Wildness::Tame); - -#ifdef SNMALLOC_CHECK_CLIENT - next->prev_encoded = - signed_prev(address_cast(curr), address_cast(next), key); -#else - UNUSED(key); -#endif - // Signature needs to be visible before item is linked in - // so requires release semantics. - curr->atomic_next_object.store( - encode_next(address_cast(&curr->next_object), next, key), - std::memory_order_release); - } - - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - static void - atomic_store_null(BHeadPtr curr, const FreeListKey& key) - { - static_assert(BView::wildness == capptr::dimension::Wildness::Tame); - - curr->atomic_next_object.store( - encode_next( - address_cast(&curr->next_object), BQueuePtr(nullptr), key), - std::memory_order_relaxed); - } - }; - - static_assert( - sizeof(FreeObject) <= MIN_ALLOC_SIZE, - "Needs to be able to fit in smallest allocation."); - - /** - * Used to iterate a free list in object space. - * - * Checks signing of pointers - */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> - class FreeListIter - { - FreeObject::BHeadPtr curr{nullptr}; -#ifdef SNMALLOC_CHECK_CLIENT - address_t prev{0}; -#endif - - public: - constexpr FreeListIter( - FreeObject::BHeadPtr head, address_t prev_value) - : curr(head) - { -#ifdef SNMALLOC_CHECK_CLIENT - prev = prev_value; -#endif - UNUSED(prev_value); - } - - constexpr FreeListIter() = default; - - /** - * Checks if there are any more values to iterate. - */ - bool empty() - { - return curr == nullptr; - } - - /** - * Returns current head without affecting the iterator. - */ - FreeObject::BHeadPtr peek() - { - return curr; - } - - /** - * Moves the iterator on, and returns the current value. - */ - template - FreeObject::BHeadPtr - take(const FreeListKey& key, Domesticator domesticate) - { - auto c = curr; - auto next = curr->read_next(key, domesticate); - - Aal::prefetch(next.unsafe_ptr()); - curr = next; -#ifdef SNMALLOC_CHECK_CLIENT - c->check_prev(prev); - prev = signed_prev(address_cast(c), address_cast(next), key); -#else - UNUSED(key); -#endif - - return c; - } - }; - - /** - * Used to build a free list in object space. - * - * Adds signing of pointers in the SNMALLOC_CHECK_CLIENT mode - * - * If RANDOM is enabled, the builder uses two queues, and - * "randomly" decides to add to one of the two queues. This - * means that we will maintain a randomisation of the order - * between allocations. - * - * The fields are paired up to give better codegen as then they are offset - * by a power of 2, and the bit extract from the interleaving seed can - * be shifted to calculate the relevant offset to index the fields. - * - * If RANDOM is set to false, then the code does not perform any - * randomisation. - */ - template< - bool RANDOM, - bool INIT = true, - SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> - class FreeListBuilder - { - static constexpr size_t LENGTH = RANDOM ? 2 : 1; - - /* - * We use native pointers below so that we don't run afoul of strict - * aliasing rules. head is a FreeObject::HeadPtr -- that - * is, a known-domesticated pointer to a queue of wild pointers -- and - * it's usually the case that end is a FreeObject::BQueuePtr* -- - * that is, a known-domesticated pointer to a wild pointer to a queue of - * wild pointers. However, in order to do branchless inserts, we set end - * = &head, which breaks strict aliasing rules with the types as given. - * Fortunately, these are private members and so we can use native - * pointers and just expose a more strongly typed interface. - */ - - // Pointer to the first element. - std::array head{nullptr}; - // Pointer to the reference to the last element. - // In the empty case end[i] == &head[i] - // This enables branch free enqueuing. - std::array end{nullptr}; - - FreeObject::BQueuePtr* cast_end(uint32_t ix) - { - return reinterpret_cast*>(end[ix]); - } - - void set_end(uint32_t ix, FreeObject::BQueuePtr* p) - { - end[ix] = reinterpret_cast(p); - } - - FreeObject::BHeadPtr cast_head(uint32_t ix) - { - return FreeObject::BHeadPtr( - static_cast*>(head[ix])); - } - - std::array length{}; - - public: - constexpr FreeListBuilder() - { - if (INIT) + Object::BQueuePtr* cast_end(uint32_t ix) { - init(); + return reinterpret_cast*>(end[ix]); } - } - /** - * Checks if the builder contains any elements. - */ - bool empty() - { - for (size_t i = 0; i < LENGTH; i++) + void set_end(uint32_t ix, Object::BQueuePtr* p) { - if (end[i] != &head[i]) + end[ix] = reinterpret_cast(p); + } + + Object::BHeadPtr cast_head(uint32_t ix) + { + return Object::BHeadPtr( + static_cast*>(head[ix])); + } + + std::array length{}; + + public: + constexpr Builder() + { + if (INIT) { - return false; + init(); } } - return true; - } - /** - * Adds an element to the builder - */ - void add( - FreeObject::BHeadPtr n, - const FreeListKey& key, - LocalEntropy& entropy) - { - uint32_t index; - if constexpr (RANDOM) - index = entropy.next_bit(); - else - index = 0; - - set_end(index, FreeObject::store_next(cast_end(index), n, key)); - if constexpr (RANDOM) + /** + * Checks if the builder contains any elements. + */ + bool empty() { - length[index]++; - } - } - - /** - * Adds an element to the builder, if we are guaranteed that - * RANDOM is false. This is useful in certain construction - * cases that do not need to introduce randomness, such as - * during the initialisation construction of a free list, which - * uses its own algorithm, or during building remote deallocation - * lists, which will be randomised at the other end. - */ - template - std::enable_if_t - add(FreeObject::BHeadPtr n, const FreeListKey& key) - { - static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); - set_end(0, FreeObject::store_next(cast_end(0), n, key)); - } - - /** - * Makes a terminator to a free list. - */ - SNMALLOC_FAST_PATH void - terminate_list(uint32_t index, const FreeListKey& key) - { - FreeObject::store_null(cast_end(index), key); - } - - /** - * Read head removing potential encoding - * - * Although, head does not require meta-data protection - * as it is not stored in an object allocation. For uniformity - * it is treated like the next_object field in a FreeObject - * and is thus subject to encoding if the next_object pointers - * encoded. - */ - FreeObject::BHeadPtr - read_head(uint32_t index, const FreeListKey& key) - { - return FreeObject::decode_next( - address_cast(&head[index]), cast_head(index), key); - } - - address_t get_fake_signed_prev(uint32_t index, const FreeListKey& key) - { - return signed_prev( - address_cast(&head[index]), address_cast(read_head(index, key)), key); - } - - /** - * Close a free list, and set the iterator parameter - * to iterate it. - * - * In the RANDOM case, it may return only part of the freelist. - * - * The return value is how many entries are still contained in the builder. - */ - SNMALLOC_FAST_PATH uint16_t - close(FreeListIter& fl, const FreeListKey& key) - { - uint32_t i; - if constexpr (RANDOM) - { - SNMALLOC_ASSERT(end[1] != &head[0]); - SNMALLOC_ASSERT(end[0] != &head[1]); - - // Select longest list. - i = length[0] > length[1] ? 0 : 1; - } - else - { - i = 0; + for (size_t i = 0; i < LENGTH; i++) + { + if (end[i] != &head[i]) + { + return false; + } + } + return true; } - terminate_list(i, key); - - fl = {read_head(i, key), get_fake_signed_prev(i, key)}; - - end[i] = &head[i]; - - if constexpr (RANDOM) + /** + * Adds an element to the builder + */ + void add( + Object::BHeadPtr n, + const FreeListKey& key, + LocalEntropy& entropy) { - length[i] = 0; - return length[1 - i]; + uint32_t index; + if constexpr (RANDOM) + index = entropy.next_bit(); + else + index = 0; + + set_end(index, Object::store_next(cast_end(index), n, key)); + if constexpr (RANDOM) + { + length[index]++; + } } - else - { - return 0; - } - } - /** - * Set the builder to a not building state. - */ - constexpr void init() - { - for (size_t i = 0; i < LENGTH; i++) + /** + * Adds an element to the builder, if we are guaranteed that + * RANDOM is false. This is useful in certain construction + * cases that do not need to introduce randomness, such as + * during the initialisation construction of a free list, which + * uses its own algorithm, or during building remote deallocation + * lists, which will be randomised at the other end. + */ + template + std::enable_if_t + add(Object::BHeadPtr n, const FreeListKey& key) { + static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); + set_end(0, Object::store_next(cast_end(0), n, key)); + } + + /** + * Makes a terminator to a free list. + */ + SNMALLOC_FAST_PATH void + terminate_list(uint32_t index, const FreeListKey& key) + { + Object::store_null(cast_end(index), key); + } + + /** + * Read head removing potential encoding + * + * Although, head does not require meta-data protection + * as it is not stored in an object allocation. For uniformity + * it is treated like the next_object field in a free Object + * and is thus subject to encoding if the next_object pointers + * encoded. + */ + Object::BHeadPtr + read_head(uint32_t index, const FreeListKey& key) + { + return Object::decode_next( + address_cast(&head[index]), cast_head(index), key); + } + + address_t get_fake_signed_prev(uint32_t index, const FreeListKey& key) + { + return signed_prev( + address_cast(&head[index]), address_cast(read_head(index, key)), key); + } + + /** + * Close a free list, and set the iterator parameter + * to iterate it. + * + * In the RANDOM case, it may return only part of the freelist. + * + * The return value is how many entries are still contained in the + * builder. + */ + SNMALLOC_FAST_PATH uint16_t + close(Iter& fl, const FreeListKey& key) + { + uint32_t i; + if constexpr (RANDOM) + { + SNMALLOC_ASSERT(end[1] != &head[0]); + SNMALLOC_ASSERT(end[0] != &head[1]); + + // Select longest list. + i = length[0] > length[1] ? 0 : 1; + } + else + { + i = 0; + } + + terminate_list(i, key); + + fl = {read_head(i, key), get_fake_signed_prev(i, key)}; + end[i] = &head[i]; - if (RANDOM) + + if constexpr (RANDOM) { length[i] = 0; + return length[1 - i]; + } + else + { + return 0; } } - } - template - std::enable_if_t< - !RANDOM_, - std::pair< - FreeObject::BHeadPtr, - FreeObject::BHeadPtr>> - extract_segment(const FreeListKey& key) - { - static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); - SNMALLOC_ASSERT(!empty()); + /** + * Set the builder to a not building state. + */ + constexpr void init() + { + for (size_t i = 0; i < LENGTH; i++) + { + end[i] = &head[i]; + if (RANDOM) + { + length[i] = 0; + } + } + } - auto first = read_head(0, key); - // end[0] is pointing to the first field in the object, - // this is doing a CONTAINING_RECORD like cast to get back - // to the actual object. This isn't true if the builder is - // empty, but you are not allowed to call this in the empty case. - auto last = FreeObject::BHeadPtr( - FreeObject::from_next_ptr(cast_end(0))); - init(); - return {first, last}; - } - }; + template + std::enable_if_t< + !RANDOM_, + std::pair< + Object::BHeadPtr, + Object::BHeadPtr>> + extract_segment(const FreeListKey& key) + { + static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); + SNMALLOC_ASSERT(!empty()); + + auto first = read_head(0, key); + // end[0] is pointing to the first field in the object, + // this is doing a CONTAINING_RECORD like cast to get back + // to the actual object. This isn't true if the builder is + // empty, but you are not allowed to call this in the empty case. + auto last = + Object::BHeadPtr(Object::from_next_ptr(cast_end(0))); + init(); + return {first, last}; + } + }; + } // namespace freelist } // namespace snmalloc diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 23a6987..b90cca2 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,21 +211,21 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); - auto domesticate = [this](FreeObject::QueuePtr p) + auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate( core_alloc->backend_state_ptr(), p); }; auto slowpath = [&]( sizeclass_t sizeclass, - FreeListIter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { + freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { if (likely(core_alloc != nullptr)) { return core_alloc->handle_message_queue( []( CoreAlloc* core_alloc, sizeclass_t sizeclass, - FreeListIter<>* fl) { + freelist::Iter<>* fl) { return core_alloc->template small_alloc(sizeclass, *fl); }, core_alloc, diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 593fc32..1ee55a8 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -13,7 +13,7 @@ namespace snmalloc using Stats = AllocStats; inline static SNMALLOC_FAST_PATH void* - finish_alloc_no_zero(FreeObject::HeadPtr p, sizeclass_t sizeclass) + finish_alloc_no_zero(freelist::HeadPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -25,14 +25,14 @@ namespace snmalloc template inline static SNMALLOC_FAST_PATH void* - finish_alloc(FreeObject::HeadPtr p, sizeclass_t sizeclass) + finish_alloc(freelist::HeadPtr p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); if constexpr (zero_mem == YesZero) SharedStateHandle::Pal::zero(r, sizeclass_to_size(sizeclass)); - // TODO: Should this be zeroing the FreeObject state, in the non-zeroing + // TODO: Should this be zeroing the free Object state, in the non-zeroing // case? return r; @@ -46,7 +46,7 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - FreeListIter<> small_fast_free_lists[NUM_SIZECLASSES] = {}; + freelist::Iter<> small_fast_free_lists[NUM_SIZECLASSES] = {}; // This is the entropy for a particular thread. LocalEntropy entropy; @@ -80,7 +80,7 @@ namespace snmalloc { auto& key = entropy.get_free_list_key(); auto domesticate = - [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 3b31fa4..614dd00 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -25,9 +25,9 @@ namespace snmalloc * Data-structure for building the free list for this slab. */ #ifdef SNMALLOC_CHECK_CLIENT - FreeListBuilder free_queue; + freelist::Builder free_queue; #else - FreeListBuilder free_queue; + freelist::Builder free_queue; #endif /** @@ -153,11 +153,11 @@ namespace snmalloc * available objects for this metaslab. */ template - static SNMALLOC_FAST_PATH std::pair + static SNMALLOC_FAST_PATH std::pair alloc_free_list( Domesticator domesticate, Metaslab* meta, - FreeListIter<>& fast_free_list, + freelist::Iter<>& fast_free_list, LocalEntropy& entropy, sizeclass_t sizeclass) { diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 6e3b0a3..f23f75b 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -35,10 +35,10 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) FreeObject::AtomicQueuePtr back{nullptr}; + alignas(CACHELINE_SIZE) freelist::AtomicQueuePtr back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) FreeObject::QueuePtr front{nullptr}; + alignas(CACHELINE_SIZE) freelist::QueuePtr front{nullptr}; constexpr RemoteAllocator() = default; @@ -48,17 +48,17 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void init(FreeObject::HeadPtr stub) + void init(freelist::HeadPtr stub) { - FreeObject::atomic_store_null(stub, key_global); + freelist::Object::atomic_store_null(stub, key_global); front = capptr_rewild(stub); back.store(front, std::memory_order_relaxed); invariant(); } - FreeObject::QueuePtr destroy() + freelist::QueuePtr destroy() { - FreeObject::QueuePtr fnt = front; + freelist::QueuePtr fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -66,7 +66,7 @@ namespace snmalloc inline bool is_empty() { - FreeObject::QueuePtr bk = back.load(std::memory_order_relaxed); + freelist::QueuePtr bk = back.load(std::memory_order_relaxed); return bk == front; } @@ -77,22 +77,22 @@ namespace snmalloc */ template void enqueue( - FreeObject::HeadPtr first, - FreeObject::HeadPtr last, + freelist::HeadPtr first, + freelist::HeadPtr last, const FreeListKey& key, Domesticator domesticate) { invariant(); - FreeObject::atomic_store_null(last, key); + freelist::Object::atomic_store_null(last, key); // exchange needs to be a release, so nullptr in next is visible. - FreeObject::QueuePtr prev = + freelist::QueuePtr prev = back.exchange(capptr_rewild(last), std::memory_order_release); - FreeObject::atomic_store_next(domesticate(prev), first, key); + freelist::Object::atomic_store_next(domesticate(prev), first, key); } - FreeObject::QueuePtr peek() + freelist::QueuePtr peek() { return front; } @@ -101,12 +101,12 @@ namespace snmalloc * Returns the front message, or null if not possible to return a message. */ template - std::pair + std::pair dequeue(const FreeListKey& key, Domesticator domesticate) { invariant(); - FreeObject::HeadPtr first = domesticate(front); - FreeObject::HeadPtr next = first->atomic_read_next(key, domesticate); + freelist::HeadPtr first = domesticate(front); + freelist::HeadPtr next = first->atomic_read_next(key, domesticate); if (next != nullptr) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 5a739aa..af48527 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -16,7 +16,7 @@ namespace snmalloc */ struct RemoteDeallocCache { - std::array, REMOTE_SLOTS> list; + std::array, REMOTE_SLOTS> list; /** * The total amount of memory we are waiting for before we will dispatch @@ -70,7 +70,7 @@ namespace snmalloc const FreeListKey& key) { SNMALLOC_ASSERT(initialised); - auto r = p.template as_reinterpret>(); + auto r = p.template as_reinterpret>(); list[get_slot(target_id, 0)].add(r, key); } @@ -85,7 +85,7 @@ namespace snmalloc size_t post_round = 0; bool sent_something = false; auto domesticate = - [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; @@ -114,7 +114,7 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - FreeListIter<> resend; + freelist::Iter<> resend; list[my_slot].close(resend, key); post_round++;