Add FIFO behaviour for unchecked code.

Consuming available slabs in LIFO order makes predicting address reuse harder
but appears to have performance implications.  Condition this on CHECK_CLIENT
and instead use FIFO order on !CHECK_CLIENT builds.

src/ds/seqset.h

@@ -10,34 +10,66 @@
 namespace snmalloc
 {
   /**
-   * Simple sequential queue of T.
+   * Simple sequential set of T.
    *
    * Linked using the T::next field.
+   *
+   * Can be used in either Fifo or Lifo mode, which is
+   * specified by template parameter.
    */
-  template<typename T>
-  class SeqQueue
+  template<typename T, bool Fifo = false>
+  class SeqSet
   {
+    /**
+     * Field representation for Fifo behaviour.
+     */
+    struct FieldFifo
+    {
+      T* head{nullptr};
+    };
+
+    /**
+     * Field representation for Lifo behaviour.
+     */
+    struct FieldLifo
+    {
+      T* head{nullptr};
+      T** end{&head};
+    };
+
     static_assert(
       std::is_same<decltype(T::next), T*>::value,
       "T->next must be a queue pointer to T");
-    T* head{nullptr};
-    T** end{&head};
 
-  public:
     /**
-     * Empty queue
+     * Field indirection to actual representation.
+     * Different numbers of fields are required for the
+     * two behaviours.
      */
-    constexpr SeqQueue() = default;
+    std::conditional_t<Fifo, FieldFifo, FieldLifo> v;
 
     /**
      * Check for empty
      */
     SNMALLOC_FAST_PATH bool is_empty()
     {
-      SNMALLOC_ASSERT(end != nullptr);
-      return &head == end;
+      if constexpr (Fifo)
+      {
+        return v.head == nullptr;
+      }
+      else
+      {
+        SNMALLOC_ASSERT(v.end != nullptr);
+        return &(v.head) == v.end;
+      }
     }
 
+  public:
+    /**
+     * Empty queue
+     */
+    constexpr SeqSet() = default;
+
     /**
      * Remove an element from the queue
      *
@@ -46,11 +78,18 @@ namespace snmalloc
     SNMALLOC_FAST_PATH T* pop()
     {
       SNMALLOC_ASSERT(!this->is_empty());
-      auto result = head;
-      if (&(head->next) == end)
-        end = &head;
+      auto result = v.head;
+      if constexpr (Fifo)
+      {
+        v.head = result->next;
+      }
       else
-        head = head->next;
+      {
+        if (&(v.head->next) == v.end)
+          v.end = &(v.head);
+        else
+          v.head = v.head->next;
+      }
       return result;
     }
 
@@ -64,13 +103,20 @@ namespace snmalloc
     template<typename Fn>
     SNMALLOC_FAST_PATH void filter(Fn&& f)
     {
-      T** prev = &head;
       // Check for empty case.
-      if (prev == end)
+      if (is_empty())
         return;
 
+      T** prev = &(v.head);
+
       while (true)
       {
+        if constexpr (Fifo)
+        {
+          if (*prev == nullptr)
+            break;
+        }
+
         T* curr = *prev;
         // Note must read curr->next before calling `f` as `f` is allowed to
         // mutate that field.
@@ -85,12 +131,16 @@ namespace snmalloc
           // Keep element
           prev = &(curr->next);
         }
-
-        if (&(curr->next) == end)
-          break;
+        if constexpr (!Fifo)
+        {
+          if (&(curr->next) == v.end)
+            break;
+        }
+      }
+      if constexpr (!Fifo)
+      {
+        v.end = prev;
       }
-
-      end = prev;
     }
 
     /**
@@ -98,8 +148,16 @@ namespace snmalloc
      */
     SNMALLOC_FAST_PATH void insert(T* item)
     {
-      *end = item;
-      end = &(item->next);
+      if constexpr (Fifo)
+      {
+        item->next = v.head;
+        v.head = item;
+      }
+      else
+      {
+        *(v.end) = item;
+        v.end = &(item->next);
+      }
     }
   };
 } // namespace snmalloc

src/mem/corealloc.h

@@ -339,7 +339,8 @@ namespace snmalloc
     SNMALLOC_SLOW_PATH void dealloc_local_slabs(sizeclass_t sizeclass)
     {
       // Return unused slabs of sizeclass_t back to global allocator
-      alloc_classes[sizeclass].queue.filter([this, sizeclass](Metaslab* meta) {
+      alloc_classes[sizeclass].available.filter([this,
+                                                 sizeclass](Metaslab* meta) {
         auto domesticate =
           [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
             auto res =
@@ -397,7 +398,7 @@ namespace snmalloc
         //  Wake slab up.
         meta->set_not_sleeping(sizeclass);
 
-        alloc_classes[sizeclass].queue.insert(meta);
+        alloc_classes[sizeclass].available.insert(meta);
         alloc_classes[sizeclass].length++;
 
 #ifdef SNMALLOC_TRACING
@@ -665,7 +666,7 @@ namespace snmalloc
       size_t rsize = sizeclass_to_size(sizeclass);
 
       // Look to see if we can grab a free list.
-      auto& sl = alloc_classes[sizeclass].queue;
+      auto& sl = alloc_classes[sizeclass].available;
       if (likely(alloc_classes[sizeclass].length > 0))
       {
 #ifdef SNMALLOC_CHECK_CLIENT
@@ -762,7 +763,7 @@ namespace snmalloc
       if (still_active)
       {
         alloc_classes[sizeclass].length++;
-        alloc_classes[sizeclass].queue.insert(meta);
+        alloc_classes[sizeclass].available.insert(meta);
       }
 
       return finish_alloc<zero_mem, SharedStateHandle>(p, sizeclass);
@@ -861,7 +862,7 @@ namespace snmalloc
 
       for (auto& alloc_class : alloc_classes)
       {
-        test(alloc_class.queue);
+        test(alloc_class.available);
       }
 
       // Place the static stub message on the queue.

src/mem/metaslab.h

@@ -292,7 +292,13 @@ namespace snmalloc
 
   struct MetaslabCache
   {
-    SeqQueue<Metaslab> queue;
+#ifdef SNMALLOC_CHECK_CLIENT
+    SeqSet<Metaslab> available;
+#else
+    // This is slightly faster in some cases,
+    // but makes memory reuse more predictable.
+    SeqSet<Metaslab, true> available;
+#endif
     uint16_t unused = 0;
     uint16_t length = 0;
   };