NFC: introduce SlabNext type for intra-small-slab free pointers

Mostly cosmetic, but eliminates some void*-s and makes intention clearer.

src/mem/alloc.h

@@ -743,7 +743,7 @@ namespace snmalloc
         while (pointer_align_up(bp, SLAB_SIZE) != bp)
         {
           Slab::alloc_new_list(bp, ffl, rsize);
-          void* prev = ffl.value;
+          SlabNext* prev = ffl.value;
           while (prev != nullptr)
           {
             auto n = Metaslab::follow_next(prev);
@@ -1027,7 +1027,7 @@ namespace snmalloc
     {
       SNMALLOC_ASSUME(sizeclass < NUM_SMALL_CLASSES);
       auto& fl = small_fast_free_lists[sizeclass];
-      void* head = fl.value;
+      SlabNext* head = fl.value;
       if (likely(head != nullptr))
       {
         stats().alloc_request(size);
@@ -1148,7 +1148,8 @@ namespace snmalloc
       SNMALLOC_ASSERT(ffl.value == nullptr);
       Slab::alloc_new_list(bp, ffl, rsize);
 
-      void* p = remove_cache_friendly_offset(ffl.value, sizeclass);
+      SlabNext* p = static_cast<SlabNext*>(
+        remove_cache_friendly_offset(ffl.value, sizeclass));
       ffl.value = Metaslab::follow_next(p);
 
       if constexpr (zero_mem == YesZero)
@@ -1171,8 +1172,8 @@ namespace snmalloc
       Slab* slab = alloc_slab<allow_reserve>(sizeclass);
       if (slab == nullptr)
         return nullptr;
-      bp =
+      bp = reinterpret_cast<SlabNext*>(
-        pointer_offset(slab, get_initial_offset(sizeclass, slab->is_short()));
+        pointer_offset(slab, get_initial_offset(sizeclass, slab->is_short())));
 
       return small_alloc_build_free_list<zero_mem, allow_reserve>(sizeclass);
     }

src/mem/metaslab.h

@@ -9,10 +9,22 @@ namespace snmalloc
 {
   class Slab;
 
+  /**
+   * Free objects within each slab point directly to the next (contrast
+   * SlabLink, which chain different Slabs of the same sizeclass together).
+   */
+  struct SlabNext
+  {
+    struct SlabNext* next;
+#ifdef CHECK_CLIENT
+    uintptr_t guard;
+#endif
+  };
+
   struct FreeListHead
   {
     // Use a value with bottom bit set for empty list.
-    void* value = nullptr;
+    SlabNext* value = nullptr;
   };
 
   using SlabList = CDLLNode;
@@ -37,7 +49,7 @@ namespace snmalloc
      *
      *  The list will be (allocated - needed) long.
      */
-    void* head = nullptr;
+    SlabNext* head = nullptr;
 
     /**
      *  How many entries are not in the free list of slab, i.e.
@@ -97,31 +109,30 @@ namespace snmalloc
 
     /// Store next pointer in a block. In Debug using magic value to detect some
     /// simple corruptions.
-    static SNMALLOC_FAST_PATH void store_next(void* p, void* head)
+    static SNMALLOC_FAST_PATH void store_next(SlabNext* p, SlabNext* head)
     {
-      *static_cast<void**>(p) = head;
+      p->next = head;
 #if defined(CHECK_CLIENT)
       if constexpr (aal_supports<IntegerPointers>)
       {
-        *(static_cast<uintptr_t*>(p) + 1) = address_cast(head) ^ POISON;
+        p->guard = address_cast(head) ^ POISON;
       }
 #endif
     }
 
     /// Accessor function for the next pointer in a block.
     /// In Debug checks for simple corruptions.
-    static SNMALLOC_FAST_PATH void* follow_next(void* node)
+    static SNMALLOC_FAST_PATH SlabNext* follow_next(SlabNext* node)
     {
 #if defined(CHECK_CLIENT)
       if constexpr (aal_supports<IntegerPointers>)
       {
-        uintptr_t next = *static_cast<uintptr_t*>(node);
+        uintptr_t next = address_cast(node->next);
-        uintptr_t chk = *(static_cast<uintptr_t*>(node) + 1);
+        if ((next ^ node->guard) != POISON)
-        if ((next ^ chk) != POISON)
           error("Detected memory corruption.  Use-after-free.");
       }
 #endif
-      return *static_cast<void**>(node);
+      return node->next;
     }
 
     bool valid_head()
@@ -166,9 +177,9 @@ namespace snmalloc
       debug_slab_invariant(slab);
 
       // Use first element as the allocation
-      void* p = head;
+      SlabNext* h = head;
       // Put the rest in allocators small_class fast free list.
-      fast_free_list.value = Metaslab::follow_next(p);
+      fast_free_list.value = Metaslab::follow_next(h);
       head = nullptr;
 
       // Treat stealing the free list as allocating it all.
@@ -176,7 +187,7 @@ namespace snmalloc
       remove();
       set_full();
 
-      p = remove_cache_friendly_offset(p, sizeclass);
+      void* p = remove_cache_friendly_offset(h, sizeclass);
       SNMALLOC_ASSERT(is_start_of_object(p));
 
       debug_slab_invariant(slab);
@@ -208,8 +219,8 @@ namespace snmalloc
     {
 #ifndef NDEBUG
       size_t length = 0;
-      void* curr = head;
+      SlabNext* curr = head;
-      void* curr_slow = head;
+      SlabNext* curr_slow = head;
       bool both = false;
       while (curr != nullptr)
       {
@@ -264,7 +275,7 @@ namespace snmalloc
       UNUSED(length);
 
       // Walk bump-free-list-segment accounting for unused space
-      void* curr = head;
+      SlabNext* curr = head;
       while (curr != nullptr)
       {
         // Check we are looking at a correctly aligned block

src/mem/slab.h

@@ -29,7 +29,9 @@ namespace snmalloc
     static SNMALLOC_FAST_PATH void
     alloc_new_list(void*& bumpptr, FreeListHead& fast_free_list, size_t rsize)
     {
-      fast_free_list.value = bumpptr;
+      auto snbumpptr = static_cast<SlabNext*>(bumpptr);
+      fast_free_list.value = snbumpptr;
+
       void* newbumpptr = pointer_offset(bumpptr, rsize);
       void* slab_end = pointer_align_up<SLAB_SIZE>(newbumpptr);
       void* slab_end2 =
@@ -39,12 +41,14 @@ namespace snmalloc
 
       while (newbumpptr < slab_end)
       {
-        Metaslab::store_next(bumpptr, newbumpptr);
+        auto newsnbumpptr = static_cast<SlabNext*>(newbumpptr);
+        Metaslab::store_next(snbumpptr, newsnbumpptr);
+        snbumpptr = newsnbumpptr;
         bumpptr = newbumpptr;
         newbumpptr = pointer_offset(bumpptr, rsize);
       }
 
-      Metaslab::store_next(bumpptr, nullptr);
+      Metaslab::store_next(snbumpptr, nullptr);
       bumpptr = newbumpptr;
     }
 
@@ -63,14 +67,16 @@ namespace snmalloc
         return false;
 
       // Update the head and the next pointer in the free list.
-      void* head = meta.head;
+      SlabNext* head = meta.head;
+
+      SlabNext* psn = static_cast<SlabNext*>(p);
 
       // Set the head to the memory being deallocated.
-      meta.head = p;
+      meta.head = psn;
       SNMALLOC_ASSERT(meta.valid_head());
 
       // Set the next pointer to the previous head.
-      Metaslab::store_next(p, head);
+      Metaslab::store_next(psn, head);
 
       return true;
     }
@@ -97,8 +103,9 @@ namespace snmalloc
           return super->dealloc_slab(this);
         }
         SNMALLOC_ASSERT(meta.head == nullptr);
-        meta.head = p;
+        SlabNext* psn = static_cast<SlabNext*>(p);
-        Metaslab::store_next(p, nullptr);
+        meta.head = psn;
+        Metaslab::store_next(psn, nullptr);
         meta.needed = meta.allocated - 1;
 
         // Push on the list of slabs for this sizeclass.