diff --git a/src/ds/seqset.h b/src/ds/seqset.h index 420e4e7..13f9faf 100644 --- a/src/ds/seqset.h +++ b/src/ds/seqset.h @@ -10,34 +10,66 @@ namespace snmalloc { /** - * Simple sequential queue of T. + * Simple sequential set of T. * * Linked using the T::next field. + * + * Can be used in either Fifo or Lifo mode, which is + * specified by template parameter. */ - template - class SeqQueue + template + class SeqSet { + /** + * Field representation for Fifo behaviour. + */ + struct FieldFifo + { + T* head{nullptr}; + }; + + /** + * Field representation for Lifo behaviour. + */ + struct FieldLifo + { + T* head{nullptr}; + T** end{&head}; + }; + static_assert( std::is_same::value, "T->next must be a queue pointer to T"); - T* head{nullptr}; - T** end{&head}; - public: /** - * Empty queue + * Field indirection to actual representation. + * Different numbers of fields are required for the + * two behaviours. */ - constexpr SeqQueue() = default; + std::conditional_t v; /** * Check for empty */ SNMALLOC_FAST_PATH bool is_empty() { - SNMALLOC_ASSERT(end != nullptr); - return &head == end; + if constexpr (Fifo) + { + return v.head == nullptr; + } + else + { + SNMALLOC_ASSERT(v.end != nullptr); + return &(v.head) == v.end; + } } + public: + /** + * Empty queue + */ + constexpr SeqSet() = default; + /** * Remove an element from the queue * @@ -46,11 +78,18 @@ namespace snmalloc SNMALLOC_FAST_PATH T* pop() { SNMALLOC_ASSERT(!this->is_empty()); - auto result = head; - if (&(head->next) == end) - end = &head; + auto result = v.head; + if constexpr (Fifo) + { + v.head = result->next; + } else - head = head->next; + { + if (&(v.head->next) == v.end) + v.end = &(v.head); + else + v.head = v.head->next; + } return result; } @@ -64,13 +103,20 @@ namespace snmalloc template SNMALLOC_FAST_PATH void filter(Fn&& f) { - T** prev = &head; // Check for empty case. - if (prev == end) + if (is_empty()) return; + T** prev = &(v.head); + while (true) { + if constexpr (Fifo) + { + if (*prev == nullptr) + break; + } + T* curr = *prev; // Note must read curr->next before calling `f` as `f` is allowed to // mutate that field. @@ -85,12 +131,16 @@ namespace snmalloc // Keep element prev = &(curr->next); } - - if (&(curr->next) == end) - break; + if constexpr (!Fifo) + { + if (&(curr->next) == v.end) + break; + } + } + if constexpr (!Fifo) + { + v.end = prev; } - - end = prev; } /** @@ -98,8 +148,16 @@ namespace snmalloc */ SNMALLOC_FAST_PATH void insert(T* item) { - *end = item; - end = &(item->next); + if constexpr (Fifo) + { + item->next = v.head; + v.head = item; + } + else + { + *(v.end) = item; + v.end = &(item->next); + } } }; } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index aace7f3..212d14f 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -339,7 +339,8 @@ namespace snmalloc SNMALLOC_SLOW_PATH void dealloc_local_slabs(sizeclass_t sizeclass) { // Return unused slabs of sizeclass_t back to global allocator - alloc_classes[sizeclass].queue.filter([this, sizeclass](Metaslab* meta) { + alloc_classes[sizeclass].available.filter([this, + sizeclass](Metaslab* meta) { auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { auto res = @@ -397,7 +398,7 @@ namespace snmalloc // Wake slab up. meta->set_not_sleeping(sizeclass); - alloc_classes[sizeclass].queue.insert(meta); + alloc_classes[sizeclass].available.insert(meta); alloc_classes[sizeclass].length++; #ifdef SNMALLOC_TRACING @@ -665,7 +666,7 @@ namespace snmalloc size_t rsize = sizeclass_to_size(sizeclass); // Look to see if we can grab a free list. - auto& sl = alloc_classes[sizeclass].queue; + auto& sl = alloc_classes[sizeclass].available; if (likely(alloc_classes[sizeclass].length > 0)) { #ifdef SNMALLOC_CHECK_CLIENT @@ -762,7 +763,7 @@ namespace snmalloc if (still_active) { alloc_classes[sizeclass].length++; - alloc_classes[sizeclass].queue.insert(meta); + alloc_classes[sizeclass].available.insert(meta); } return finish_alloc(p, sizeclass); @@ -861,7 +862,7 @@ namespace snmalloc for (auto& alloc_class : alloc_classes) { - test(alloc_class.queue); + test(alloc_class.available); } // Place the static stub message on the queue. diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 733759b..3576166 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -292,7 +292,13 @@ namespace snmalloc struct MetaslabCache { - SeqQueue queue; +#ifdef SNMALLOC_CHECK_CLIENT + SeqSet available; +#else + // This is slightly faster in some cases, + // but makes memory reuse more predictable. + SeqSet available; +#endif uint16_t unused = 0; uint16_t length = 0; };