From 8ac2adc4e597ff7a39f6b6acecac8d70a9ecc724 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 29 Sep 2021 10:01:31 +0100 Subject: [PATCH] Added a sequential queue This changes the slab lists to use a sequential queue. They were previously stored in a stack. This commit also tidies up some incomplete refactoring from the initial snmalloc2 work. --- src/ds/cdllist.h | 93 --------------------------------------- src/ds/seqqueue.h | 105 ++++++++++++++++++++++++++++++++++++++++++++ src/mem/corealloc.h | 71 ++++++++++++------------------ src/mem/metaslab.h | 25 +++-------- 4 files changed, 139 insertions(+), 155 deletions(-) delete mode 100644 src/ds/cdllist.h create mode 100644 src/ds/seqqueue.h diff --git a/src/ds/cdllist.h b/src/ds/cdllist.h deleted file mode 100644 index 56c601f..0000000 --- a/src/ds/cdllist.h +++ /dev/null @@ -1,93 +0,0 @@ -#pragma once - -#include "address.h" -#include "defines.h" -#include "ptrwrap.h" - -#include -#include - -namespace snmalloc -{ - /** - * TODO Rewrite for actual use, no longer Cyclic or doubly linked. - * - * Special class for cyclic doubly linked non-empty linked list - * - * This code assumes there is always one element in the list. The client - * must ensure there is a sentinal element. - */ - template typename Ptr = Pointer> - class CDLLNode - { - Ptr next{nullptr}; - - constexpr void set_next(Ptr c) - { - next = c; - } - - public: - /** - * Single element cyclic list. This is the empty case. - */ - constexpr CDLLNode() - { - this->set_next(nullptr); - } - - SNMALLOC_FAST_PATH bool is_empty() - { - return next == nullptr; - } - - SNMALLOC_FAST_PATH Ptr get_next() - { - return next; - } - - /** - * Single element cyclic list. This is the uninitialised case. - * - * This entry should never be accessed and is only used to make - * a fake metaslab. - */ - constexpr CDLLNode(bool) {} - - SNMALLOC_FAST_PATH Ptr pop() - { - SNMALLOC_ASSERT(!this->is_empty()); - auto result = get_next(); - set_next(result->get_next()); - return result; - } - - SNMALLOC_FAST_PATH void insert(Ptr item) - { - debug_check(); - item->set_next(this->get_next()); - set_next(item); - debug_check(); - } - - /** - * Checks the lists invariants - * x->next->prev = x - * for all x in the list. - */ - void debug_check() - { -#ifndef NDEBUG - // Ptr item = this->get_next(); - // auto p = Ptr(this); - - // do - // { - // SNMALLOC_ASSERT(item->prev == p); - // p = item; - // item = item->get_next(); - // } while (item != Ptr(this)); -#endif - } - }; -} // namespace snmalloc diff --git a/src/ds/seqqueue.h b/src/ds/seqqueue.h new file mode 100644 index 0000000..420e4e7 --- /dev/null +++ b/src/ds/seqqueue.h @@ -0,0 +1,105 @@ +#pragma once + +#include "address.h" +#include "defines.h" +#include "ptrwrap.h" + +#include +#include + +namespace snmalloc +{ + /** + * Simple sequential queue of T. + * + * Linked using the T::next field. + */ + template + class SeqQueue + { + static_assert( + std::is_same::value, + "T->next must be a queue pointer to T"); + T* head{nullptr}; + T** end{&head}; + + public: + /** + * Empty queue + */ + constexpr SeqQueue() = default; + + /** + * Check for empty + */ + SNMALLOC_FAST_PATH bool is_empty() + { + SNMALLOC_ASSERT(end != nullptr); + return &head == end; + } + + /** + * Remove an element from the queue + * + * Assumes queue is non-empty + */ + SNMALLOC_FAST_PATH T* pop() + { + SNMALLOC_ASSERT(!this->is_empty()); + auto result = head; + if (&(head->next) == end) + end = &head; + else + head = head->next; + return result; + } + + /** + * Filter + * + * Removes all elements that f returns true for. + * If f returns true, then filter is not allowed to look at the + * object again, and f is responsible for its lifetime. + */ + template + SNMALLOC_FAST_PATH void filter(Fn&& f) + { + T** prev = &head; + // Check for empty case. + if (prev == end) + return; + + while (true) + { + T* curr = *prev; + // Note must read curr->next before calling `f` as `f` is allowed to + // mutate that field. + T* next = curr->next; + if (f(curr)) + { + // Remove element; + *prev = next; + } + else + { + // Keep element + prev = &(curr->next); + } + + if (&(curr->next) == end) + break; + } + + end = prev; + } + + /** + * Add an element to the queue. + */ + SNMALLOC_FAST_PATH void insert(T* item) + { + *end = item; + end = &(item->next); + } + }; +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 60d29eb..9b3909b 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -299,32 +299,23 @@ namespace snmalloc SNMALLOC_SLOW_PATH void dealloc_local_slabs(sizeclass_t sizeclass) { // Return unused slabs of sizeclass_t back to global allocator - SlabLink* prev = &alloc_classes[sizeclass]; - auto curr = prev->get_next(); - while (curr != nullptr) - { - auto nxt = curr->get_next(); - auto meta = Metaslab::from_link(curr); - if (meta->needed() == 0) - { - prev->pop(); - alloc_classes[sizeclass].length--; - alloc_classes[sizeclass].unused--; + alloc_classes[sizeclass].queue.filter([this, sizeclass](Metaslab* meta) { + if (meta->needed() != 0) + return false; - // TODO delay the clear to the next user of the slab, or teardown so - // don't touch the cache lines at this point in check_client. - auto chunk_record = clear_slab(meta, sizeclass); - ChunkAllocator::dealloc( - get_backend_local_state(), - chunk_record, - sizeclass_to_slab_sizeclass(sizeclass)); - } - else - { - prev = curr; - } - curr = nxt; - } + alloc_classes[sizeclass].length--; + alloc_classes[sizeclass].unused--; + + // TODO delay the clear to the next user of the slab, or teardown so + // don't touch the cache lines at this point in check_client. + auto chunk_record = clear_slab(meta, sizeclass); + ChunkAllocator::dealloc( + get_backend_local_state(), + chunk_record, + sizeclass_to_slab_sizeclass(sizeclass)); + + return true; + }); } /** @@ -348,7 +339,7 @@ namespace snmalloc // Wake slab up. meta->set_not_sleeping(sizeclass); - alloc_classes[sizeclass].insert(&meta->link); + alloc_classes[sizeclass].queue.insert(meta); alloc_classes[sizeclass].length++; #ifdef SNMALLOC_TRACING @@ -590,10 +581,10 @@ namespace snmalloc size_t rsize = sizeclass_to_size(sizeclass); // Look to see if we can grab a free list. - auto& sl = alloc_classes[sizeclass]; + auto& sl = alloc_classes[sizeclass].queue; if (likely(!(sl.is_empty()))) { - auto meta = Metaslab::from_link(sl.pop()); + auto meta = sl.pop(); // Drop length of sl, and empty count if it was empty. alloc_classes[sizeclass].length--; if (meta->needed() == 0) @@ -734,29 +725,23 @@ namespace snmalloc bool debug_is_empty_impl(bool* result) { auto test = [&result](auto& queue) { - if (!queue.is_empty()) - { - auto curr = queue.get_next(); - while (curr != nullptr) + queue.filter([&result](auto metaslab) { + if (metaslab->needed() != 0) { - auto currmeta = Metaslab::from_link(curr); - if (currmeta->needed() != 0) - { - if (result != nullptr) - *result = false; - else - error("debug_is_empty: found non-empty allocator"); - } - curr = curr->get_next(); + if (result != nullptr) + *result = false; + else + error("debug_is_empty: found non-empty allocator"); } - } + return false; + }); }; bool sent_something = flush(true); for (auto& alloc_class : alloc_classes) { - test(alloc_class); + test(alloc_class.queue); } // Place the static stub message on the queue. diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 0c9954a..88f3721 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -1,8 +1,8 @@ #pragma once -#include "../ds/cdllist.h" #include "../ds/dllist.h" #include "../ds/helpers.h" +#include "../ds/seqqueue.h" #include "../mem/remoteallocator.h" #include "freelist.h" #include "ptrhelpers.h" @@ -12,23 +12,15 @@ namespace snmalloc { class Slab; - using SlabLink = CDLLNode<>; - // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. class alignas(CACHELINE_SIZE) Metaslab { public: - // TODO: Annotate with CHERI subobject unbound for pointer arithmetic - SlabLink link; + // Used to link metaslabs together in various other data-structures. + Metaslab* next{nullptr}; - constexpr Metaslab() : link(true) {} - - /** - * Metaslab::link points at another link field. To get the actual Metaslab, - * use this encapsulation of the container-of logic. - */ - static Metaslab* from_link(SlabLink* ptr); + constexpr Metaslab() = default; /** * Data-structure for building the free list for this slab. @@ -167,12 +159,6 @@ namespace snmalloc } }; - inline Metaslab* Metaslab::from_link(SlabLink* lptr) - { - return pointer_offset_signed( - lptr, -static_cast(offsetof(Metaslab, link))); - } - struct RemoteAllocator; /** @@ -232,8 +218,9 @@ namespace snmalloc } }; - struct MetaslabCache : public CDLLNode<> + struct MetaslabCache { + SeqQueue queue; uint16_t unused = 0; uint16_t length = 0; };