From 2efcddfc3dd69ce02530bc59979d1e17ac2ddcd6 Mon Sep 17 00:00:00 2001
From: David Chisnall <David.Chisnall@microsoft.com>
Date: Fri, 5 Jul 2019 14:20:24 +0100
Subject: [PATCH] Rework free list so that 0 is the placeholder.

This is needed because in some configurations the constructor for the
global placeholder is not called before the first allocation (i.e. when
other globals call the allocator in their constructor) and so we ended
up following a null pointer.
---
 src/ds/bits.h      |  2 +-
 src/mem/alloc.h    |  2 +-
 src/mem/metaslab.h | 10 +++++-----
 src/mem/slab.h     | 10 ++++++----
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/ds/bits.h b/src/ds/bits.h
index e6721cd..2503606 100644
--- a/src/ds/bits.h
+++ b/src/ds/bits.h
@@ -21,7 +21,7 @@
 #  include <emmintrin.h>
 #  define ALWAYSINLINE __attribute__((always_inline))
 #  define NOINLINE __attribute__((noinline))
-#  define SNMALLOC_SLOW_PATH NOINLINE
+#  define SNMALLOC_SLOW_PATH NOINLINE __attribute__((section(".text,slow")))
 #  define SNMALLOC_FAST_PATH inline ALWAYSINLINE
 #  define SNMALLOC_PURE __attribute__((const))
 #  ifdef __clang__
diff --git a/src/mem/alloc.h b/src/mem/alloc.h
index 94d209d..43e6ff7 100644
--- a/src/mem/alloc.h
+++ b/src/mem/alloc.h
@@ -1072,7 +1072,7 @@ namespace snmalloc
       assert(sizeclass < NUM_SMALL_CLASSES);
       auto& fl = small_fast_free_lists[sizeclass];
       auto head = fl.value;
-      if (likely((reinterpret_cast<size_t>(head) & 1) == 0))
+      if (likely(head != nullptr))
       {
         void* p = head;
         // Read the next slot from the memory that's about to be allocated.
diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h
index a7a60b9..7295324 100644
--- a/src/mem/metaslab.h
+++ b/src/mem/metaslab.h
@@ -146,10 +146,10 @@ namespace snmalloc
     {
 #ifndef NDEBUG
       size_t length = 0;
-      void* curr = pointer_offset(slab, head);
-      void* curr_slow = pointer_offset(slab, head);
+      void* curr = (head == 1) ? nullptr : pointer_offset(slab, head);
+      void* curr_slow = (head == 1) ? nullptr : pointer_offset(slab, head);
       bool both = false;
-      while ((reinterpret_cast<size_t>(curr) & 1) == 0)
+      while (curr != nullptr)
       {
         curr = follow_next(curr);
         if (both)
@@ -200,8 +200,8 @@ namespace snmalloc
       UNUSED(length);
 
       // Walk bump-free-list-segment accounting for unused space
-      void* curr = pointer_offset(slab, head);
-      while ((address_cast(curr) & 1) == 0)
+      void* curr = (head == 1) ? nullptr : pointer_offset(slab, head);
+      while (curr != nullptr)
       {
         // Check we are looking at a correctly aligned block
         void* start = curr;
diff --git a/src/mem/slab.h b/src/mem/slab.h
index 24c4fe2..0845d7c 100644
--- a/src/mem/slab.h
+++ b/src/mem/slab.h
@@ -7,7 +7,7 @@ namespace snmalloc
   struct FreeListHead
   {
     // Use a value with bottom bit set for empty list.
-    void* value = pointer_offset<void*>(nullptr, 1);
+    void* value = nullptr;
   };
 
   class Slab
@@ -98,7 +98,8 @@ namespace snmalloc
             }
             else
             {
-              Metaslab::store_next(curr, pointer_offset(this, bumpptr));
+              Metaslab::store_next(
+                curr, (bumpptr == 1) ? nullptr : pointer_offset(this, bumpptr));
             }
             curr = pointer_offset(this, bumpptr);
             bumpptr = newbumpptr;
@@ -106,7 +107,7 @@ namespace snmalloc
           }
 
           assert(curr != nullptr);
-          Metaslab::store_next(curr, pointer_offset<void*>(nullptr, 1));
+          Metaslab::store_next(curr, nullptr);
         }
       }
 
@@ -177,7 +178,8 @@ namespace snmalloc
       assert(meta.valid_head(is_short()));
 
       // Set the next pointer to the previous head.
-      Metaslab::store_next(p, pointer_offset(this, head));
+      Metaslab::store_next(
+        p, (head == 1) ? nullptr : pointer_offset(this, head));
       meta.debug_slab_invariant(is_short(), this);
       return true;
     }