Move link out of object space.

The link object was previously stored in a disused object.  This is
good for reducing meta-data, but if we want to reduce the meta-data
corruption potential, then this is not a good design choice.

This commit moves it into the Metaslab.

src/ds/cdllist.h

@@ -110,6 +110,17 @@ namespace snmalloc
 #endif
     }
 
+    /**
+     * Nulls the previous pointer
+     *
+     * The Meta-slab uses nullptr in prev to mean that it is not part of a
+     * size class list.
+     **/
+    void null_prev()
+    {
+      prev = nullptr;
+    }
+
     SNMALLOC_FAST_PATH CDLLNode* get_prev()
     {
       return prev;

src/mem/alloc.h

@@ -1074,18 +1074,14 @@ namespace snmalloc
       size_t rsize = sizeclass_to_size(sizeclass);
       auto& sl = small_classes[sizeclass];
 
-      Slab* slab;
-
       if (likely(!sl.is_empty()))
       {
         stats().alloc_request(size);
         stats().sizeclass_alloc(sizeclass);
 
-        SlabLink* link = sl.get_next();
+        auto meta = reinterpret_cast<Metaslab*>(sl.get_next());
-        slab = get_slab(link);
         auto& ffl = small_fast_free_lists[sizeclass];
-        return slab->alloc<zero_mem, typename MemoryProvider::Pal>(
+        return meta->alloc<zero_mem, typename MemoryProvider::Pal>(ffl, rsize);
-          sl, ffl, rsize);
       }
       return small_alloc_rare<zero_mem, allow_reserve>(sizeclass, size);
     }

src/mem/metaslab.h

@@ -9,6 +9,12 @@ namespace snmalloc
 {
   class Slab;
 
+  struct FreeListHead
+  {
+    // Use a value with bottom bit set for empty list.
+    void* value = nullptr;
+  };
+
   using SlabList = CDLLNode;
   using SlabLink = CDLLNode;
 
@@ -23,14 +29,13 @@ namespace snmalloc
 
   // The Metaslab represent the status of a single slab.
   // This can be either a short or a standard slab.
-  class Metaslab
+  class Metaslab : public SlabLink
   {
   public:
     /**
      *  Pointer to first free entry in this slab
      *
-     *  The list will be (allocated - needed - 1) long. The -1 is
+     *  The list will be (allocated - needed) long.
-     *  for the `link` element which is not in the free list.
      */
     void* head = nullptr;
 
@@ -38,7 +43,7 @@ namespace snmalloc
      *  How many entries are not in the free list of slab, i.e.
      *  how many entries are needed to fully free this slab.
      *
-     *  In the case of a fully allocated slab, where link==1 needed
+     *  In the case of a fully allocated slab, where prev==0 needed
      *  will be 1. This enables 'return_object' to detect the slow path
      *  case with a single operation subtract and test.
      */
@@ -49,11 +54,6 @@ namespace snmalloc
      */
     uint16_t allocated = 0;
 
-    // When a slab has free space it will be on the has space list for
-    // that size class.  We use an empty block in this slab to be the
-    // doubly linked node into that size class's free list.
-    Mod<SLAB_SIZE, uint16_t> link;
-
     uint8_t sizeclass;
     // Initially zero to encode the superslabs relative list of slabs.
     uint8_t next = 0;
@@ -77,7 +77,7 @@ namespace snmalloc
 
     bool is_full()
     {
-      auto result = link == 1;
+      auto result = get_prev() == nullptr;
       SNMALLOC_ASSERT(!result || head == nullptr);
       return result;
     }
@@ -85,16 +85,10 @@ namespace snmalloc
     SNMALLOC_FAST_PATH void set_full()
     {
       SNMALLOC_ASSERT(head == nullptr);
-      SNMALLOC_ASSERT(link != 1);
-      link = 1;
       // Set needed to 1, so that "return_object" will return true after calling
       // set_full
       needed = 1;
-    }
+      null_prev();
-
-    SlabLink* get_link(Slab* slab)
-    {
-      return reinterpret_cast<SlabLink*>(pointer_offset(slab, link));
     }
 
     /// Value used to check for corruptions in a block
@@ -157,6 +151,51 @@ namespace snmalloc
         pointer_diff(p, pointer_align_up<SLAB_SIZE>(pointer_offset(p, 1))));
     }
 
+    /**
+     * Takes a free list out of a slabs meta data.
+     * Returns the link as the allocation, and places the free list into the
+     * `fast_free_list` for further allocations.
+     */
+    template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
+    SNMALLOC_FAST_PATH void* alloc(FreeListHead& fast_free_list, size_t rsize)
+    {
+      SNMALLOC_ASSERT(rsize == sizeclass_to_size(sizeclass));
+      SNMALLOC_ASSERT(!is_full());
+
+      auto slab = get_slab(head);
+      debug_slab_invariant(slab);
+
+      // Use first element as the allocation
+      void* p = head;
+      // Put the rest in allocators small_class fast free list.
+      fast_free_list.value = Metaslab::follow_next(p);
+      head = nullptr;
+
+      // Treat stealing the free list as allocating it all.
+      needed = allocated;
+      remove();
+      set_full();
+
+      p = remove_cache_friendly_offset(p, sizeclass);
+      SNMALLOC_ASSERT(is_start_of_object(p));
+
+      debug_slab_invariant(slab);
+
+      if constexpr (zero_mem == YesZero)
+      {
+        if (rsize < PAGE_ALIGNED_SIZE)
+          PAL::zero(p, rsize);
+        else
+          PAL::template zero<true>(p, rsize);
+      }
+      else
+      {
+        UNUSED(rsize);
+      }
+
+      return p;
+    }
+
     /**
      * Check bump-free-list-segment for cycles
      *
@@ -207,7 +246,6 @@ namespace snmalloc
       if (is_full())
       {
         // There is no free list to validate
-        // 'link' value is not important if full.
         return;
       }
 
@@ -236,8 +274,6 @@ namespace snmalloc
         // Account for free elements in free list
         accounted_for += size;
         SNMALLOC_ASSERT(SLAB_SIZE >= accounted_for);
-        // We should never reach the link node in the free list.
-        SNMALLOC_ASSERT(curr != pointer_offset(slab, link));
 
         // Iterate bump/free list segment
         curr = follow_next(curr);
@@ -250,16 +286,7 @@ namespace snmalloc
       // Account for to be bump allocated space
       accounted_for += SLAB_SIZE - bumpptr;
 
-      if (bumpptr != SLAB_SIZE)
-      {
-        // The link should be the first allocation as we
-        // haven't completely filled this block at any point.
-        SNMALLOC_ASSERT(link == get_initial_offset(sizeclass, is_short));
-      }
-
       SNMALLOC_ASSERT(!is_full());
-      // Add the link node.
-      accounted_for += size;
 
       // All space accounted for
       SNMALLOC_ASSERT(SLAB_SIZE == accounted_for);

src/mem/slab.h

@@ -4,12 +4,6 @@
 
 namespace snmalloc
 {
-  struct FreeListHead
-  {
-    // Use a value with bottom bit set for empty list.
-    void* value = nullptr;
-  };
-
   class Slab
   {
   private:
@@ -26,62 +20,6 @@ namespace snmalloc
       return super->get_meta(this);
     }
 
-    SlabLink* get_link()
-    {
-      return get_meta().get_link(this);
-    }
-
-    /**
-     * Takes a free list out of a slabs meta data.
-     * Returns the link as the allocation, and places the free list into the
-     * `fast_free_list` for further allocations.
-     */
-    template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
-    SNMALLOC_FAST_PATH void*
-    alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize)
-    {
-      // Read the head from the metadata stored in the superslab.
-      Metaslab& meta = get_meta();
-      SNMALLOC_ASSERT(meta.link != 1);
-
-      SNMALLOC_ASSERT(rsize == sizeclass_to_size(meta.sizeclass));
-      SNMALLOC_ASSERT(
-        sl.get_next() == (SlabLink*)pointer_offset(this, meta.link));
-      SNMALLOC_ASSERT(!meta.is_full());
-      meta.debug_slab_invariant(this);
-
-      // Put everything in allocators small_class free list.
-      fast_free_list.value = meta.head;
-      meta.head = nullptr;
-
-      // Return the link as the node for this allocation.
-      void* link = pointer_offset(this, meta.link);
-      void* p = remove_cache_friendly_offset(link, meta.sizeclass);
-
-      // Treat stealing the free list as allocating it all.
-      meta.needed = meta.allocated;
-      meta.set_full();
-      sl.get_next()->remove();
-
-      SNMALLOC_ASSERT(meta.is_start_of_object(p));
-
-      meta.debug_slab_invariant(this);
-
-      if constexpr (zero_mem == YesZero)
-      {
-        if (rsize < PAGE_ALIGNED_SIZE)
-          PAL::zero(p, rsize);
-        else
-          PAL::template zero<true>(p, rsize);
-      }
-      else
-      {
-        UNUSED(rsize);
-      }
-
-      return p;
-    }
-
     /**
      * Given a bumpptr and a fast_free_list head reference, builds a new free
      * list, and stores it in the fast_free_list. It will only create a page
@@ -158,21 +96,19 @@ namespace snmalloc
 
           return super->dealloc_slab(this);
         }
-        // Update the head and the sizeclass link.
-        uint16_t index = pointer_to_index(p);
         SNMALLOC_ASSERT(meta.head == nullptr);
-        //        SNMALLOC_ASSERT(meta.fully_allocated(is_short()));
+        meta.head = p;
-        meta.link = index;
+        Metaslab::store_next(p, nullptr);
         meta.needed = meta.allocated - 1;
 
         // Push on the list of slabs for this sizeclass.
-        sl->insert_prev(meta.get_link(this));
+        sl->insert_prev(&meta);
         meta.debug_slab_invariant(this);
         return Superslab::NoSlabReturn;
       }
 
       // Remove from the sizeclass list and dealloc on the superslab.
-      meta.get_link(this)->remove();
+      meta.remove();
 
       if (is_short())
         return super->dealloc_short_slab();

src/mem/superslab.h

@@ -173,8 +173,7 @@ namespace snmalloc
       meta[0].allocated = static_cast<uint16_t>(
         (SLAB_SIZE - get_initial_offset(sizeclass, true)) /
         sizeclass_to_size(sizeclass));
-      meta[0].link = 1;
+      meta[0].set_full();
-      meta[0].needed = 1;
       meta[0].sizeclass = static_cast<uint8_t>(sizeclass);
 
       used++;
@@ -198,8 +197,7 @@ namespace snmalloc
       meta[h].allocated = static_cast<uint16_t>(
         (SLAB_SIZE - get_initial_offset(sizeclass, false)) /
         sizeclass_to_size(sizeclass));
-      meta[h].needed = 1;
+      meta[h].set_full();
-      meta[h].link = 1;
       meta[h].sizeclass = static_cast<uint8_t>(sizeclass);
 
       head = h + n + 1;