Used start of a slab for link.

src/mem/metaslab.h

@@ -19,15 +19,12 @@ namespace snmalloc
     }
   };
 
-  using SlabList = DLList<SlabLink, InvalidPointer<UINTPTR_MAX>>;
+  using SlabList = DLList<SlabLink>;
 
   static_assert(
     sizeof(SlabLink) <= MIN_ALLOC_SIZE,
     "Need to be able to pack a SlabLink into any free small alloc");
 
-  static constexpr uint16_t SLABLINK_INDEX =
-    static_cast<uint16_t>(SLAB_SIZE - sizeof(SlabLink));
-
   // The Metaslab represent the status of a single slab.
   // This can be either a short or a standard slab.
   class Metaslab
@@ -42,16 +39,11 @@ namespace snmalloc
     // of where we have allocated up to in this slab. Otherwise,
     // it represents the location of the first block in the free
     // list.  The free list is chained through deallocated blocks.
-    // It either terminates with a bump ptr, or if all the space is in
-    // the free list, then the last block will be also referenced by
-    // link.
-    // Note that, in the case that this is the first block in the size
-    // class list, where all the unused memory is in the free list,
-    // then the last block can both be interpreted as a final bump
-    // pointer entry, and the first entry in the doubly linked list.
-    // The terminal value in the free list, and the terminal value in
-    // the SlabLink previous field will alias. The SlabLink uses ~0 for
-    // its terminal value to be a valid terminal bump ptr.
+    // It is terminated with a bump ptr.
+    //
+    // Note that, the first entry in a slab is never bump allocated
+    // but is used for the link. This means that 1 represents the fully
+    // bump allocated slab.
     Mod<SLAB_SIZE, uint16_t> head;
     // When a slab has free space it will be on the has space list for
     // that size class.  We use an empty block in this slab to be the
@@ -84,13 +76,14 @@ namespace snmalloc
 
     bool is_full()
     {
-      return (head & 2) != 0;
+      return link == 1;
     }
 
     void set_full()
     {
       assert(head == 1);
-      head = static_cast<uint16_t>(~0);
+      assert(link != 1);
+      link = 1;
     }
 
     SlabLink* get_link(Slab* slab)
@@ -141,28 +134,31 @@ namespace snmalloc
         // Account for free elements in free list
         accounted_for += size;
         assert(SLAB_SIZE >= accounted_for);
-        // We are not guaranteed to hit a bump ptr unless
-        // we are the top element on the size class, so treat as
-        // a list segment.
-        if (curr == link)
-          break;
+        // We should never reach the link node in the free list.
+        assert(curr != link);
+
         // Iterate bump/free list segment
         curr = *reinterpret_cast<uint16_t*>(pointer_offset(slab, curr));
       }
 
-      // Check we terminated traversal on a correctly aligned block
-      uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass);
-      assert((start - offset) % size == 0);
-
-      if (curr != link)
+      if (curr != 1)
       {
-        // The link should be at the special end location as we
-        // haven't completely filled this block at any point.
-        assert(link == SLABLINK_INDEX);
+        // Check we terminated traversal on a correctly aligned block
+        uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass);
+        assert((start - offset) % size == 0);
+
         // Account for to be bump allocated space
         accounted_for += SLAB_SIZE - (curr - 1);
+
+        // The link should be the first allocation as we
+        // haven't completely filled this block at any point.
+        assert(link == (get_slab_offset(sizeclass, is_short) - 1));
       }
 
+      assert(!is_full());
+      // Add the link node.
+      accounted_for += size;
+
       // All space accounted for
       assert(SLAB_SIZE == accounted_for);
 #else

src/mem/slab.h

@@ -58,19 +58,20 @@ namespace snmalloc
       }
       else
       {
-        // This slab is being bump allocated.
-        p = pointer_offset(this, head - 1);
-        meta.head = (head + static_cast<uint16_t>(rsize)) & (SLAB_SIZE - 1);
         if (meta.head == 1)
         {
+          p = pointer_offset(this, meta.link);
+          sc->pop();
           meta.set_full();
         }
+        else
+        {
+          // This slab is being bump allocated.
+          p = pointer_offset(this, head - 1);
+          meta.head = (head + static_cast<uint16_t>(rsize)) & (SLAB_SIZE - 1);
+        }
       }
 
-      // If we're full, we're no longer the current slab for this sizeclass
-      if (meta.is_full())
-        sc->pop();
-
       meta.debug_slab_invariant(is_short(), this);
 
       if constexpr (zero_mem == YesZero)
@@ -110,8 +111,7 @@ namespace snmalloc
         {
           // Update the head and the sizeclass link.
           uint16_t index = pointer_to_index(p);
-          meta.head = index;
-          assert(meta.valid_head(is_short()));
+          assert(meta.head == 1);
           meta.link = index;
 
           // Push on the list of slabs for this sizeclass.

src/mem/superslab.h

@@ -159,9 +159,12 @@ namespace snmalloc
       if ((used & 1) == 1)
         return alloc_slab(sizeclass, memory_provider);
 
-      meta[0].head = get_slab_offset(sizeclass, true);
+      auto rsize = sizeclass_to_size(sizeclass);
+
+      meta[0].head =
+        (get_slab_offset(sizeclass, true) + rsize) & (SLAB_SIZE - 1);
       meta[0].sizeclass = sizeclass;
-      meta[0].link = SLABLINK_INDEX;
+      meta[0].link = get_slab_offset(sizeclass, true) - 1;
 
       if constexpr (decommit_strategy == DecommitAll)
       {
@@ -182,9 +185,11 @@ namespace snmalloc
 
       uint8_t n = meta[h].next;
 
-      meta[h].head = get_slab_offset(sizeclass, false);
+      auto rsize = sizeclass_to_size(sizeclass);
+      meta[h].head =
+        (get_slab_offset(sizeclass, false) + rsize) & (SLAB_SIZE - 1);
       meta[h].sizeclass = sizeclass;
-      meta[h].link = SLABLINK_INDEX;
+      meta[h].link = get_slab_offset(sizeclass, false) - 1;
 
       head = h + n + 1;
       used += 2;