Improved ThreadAlloc::get API

Made the API so that get always returns an initialised Alloc*.  Added
new fast path that doesn't perform checking, but can lead to very slow
behaviour if called and reused.

src/mem/threadalloc.h

@@ -127,10 +127,11 @@ namespace snmalloc
      */
     static void inner_release()
     {
-      if (get() != &GlobalPlaceHolder)
+      Alloc*& a = get_noncachable();
+      if (a != &GlobalPlaceHolder)
       {
-        current_alloc_pool()->release(get());
-        get() = &GlobalPlaceHolder;
+        current_alloc_pool()->release(a);
+        a = &GlobalPlaceHolder;
       }
     }
 
@@ -152,13 +153,38 @@ namespace snmalloc
     /**
      * Public interface, returns the allocator for this thread, constructing
      * one if necessary.
+     *
+     * If no operations have been performed on an allocator returned by either
+     * `get()` nor `get_noncachable`, then the value contained in the return
+     * will be an Alloc* that will always use the slow path.  The Alloc*& will
+     * be updated, and is performant to use for subsequent calls.
+     *
+     * Only use this API if you intend to use the returned allocator just once
+     * per call, you store the indirection (Alloc*&) rather than (Alloc*), or if
+     * you know other calls have already been made to the allocator.
      */
-    static inline Alloc*& get()
+    static inline Alloc*& get_noncachable()
     {
       static thread_local Alloc* alloc = &GlobalPlaceHolder;
       return alloc;
     }
 
+    /**
+     * Public interface, returns the allocator for this thread, constructing
+     * one if necessary.
+     *
+     * The returned Alloc* is guaranteed to be initialised.  This incurs a cost,
+     * so use `get_noncachable` if you can meet its criteria.
+     */
+    static SNMALLOC_FAST_PATH Alloc* get()
+    {
+      auto alloc = get_noncachable();
+      auto new_alloc = lazy_replacement(alloc);
+      return (likely(new_alloc == nullptr)) ?
+        alloc :
+        reinterpret_cast<Alloc*>(new_alloc);
+    }
+
     static void register_cleanup()
     {
       static thread_local OnDestruct<ThreadAllocThreadDestructor::inner_release>
@@ -195,7 +221,7 @@ namespace snmalloc
    */
   SNMALLOC_SLOW_PATH inline void* lazy_replacement_slow()
   {
-    auto*& local_alloc = ThreadAlloc::get();
+    auto*& local_alloc = ThreadAlloc::get_noncachable();
     if ((local_alloc != nullptr) && (local_alloc != &GlobalPlaceHolder))
     {
       return local_alloc;

src/test/func/fixed_region/fixed_region.cc

@@ -37,7 +37,7 @@ int main()
   oe_end = (uint8_t*)oe_base + size;
   std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;
 
-  auto& a = ThreadAlloc::get();
+  auto a = ThreadAlloc::get();
 
   for (size_t i = 0; i < 1000; i++)
   {

src/test/func/memory/memory.cc

@@ -7,7 +7,7 @@ using namespace snmalloc;
 
 void test_alloc_dealloc_64k()
 {
-  auto& alloc = ThreadAlloc::get();
+  auto alloc = ThreadAlloc::get();
 
   constexpr size_t count = 1 << 12;
   constexpr size_t outer_count = 12;
@@ -39,7 +39,7 @@ void test_alloc_dealloc_64k()
 
 void test_random_allocation()
 {
-  auto& alloc = ThreadAlloc::get();
+  auto alloc = ThreadAlloc::get();
   std::unordered_set<void*> allocated;
 
   constexpr size_t count = 10000;
@@ -91,7 +91,7 @@ void test_random_allocation()
 
 void test_calloc()
 {
-  auto& alloc = ThreadAlloc::get();
+  auto alloc = ThreadAlloc::get();
 
   for (size_t size = 16; size <= (1 << 24); size <<= 1)
   {
@@ -162,7 +162,7 @@ void test_double_alloc()
 void test_external_pointer()
 {
   // Malloc does not have an external pointer querying mechanism.
-  auto& alloc = ThreadAlloc::get();
+  auto alloc = ThreadAlloc::get();
 
   for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++)
   {
@@ -208,7 +208,7 @@ void test_external_pointer_large()
 {
   xoroshiro::p128r64 r;
 
-  auto& alloc = ThreadAlloc::get();
+  auto alloc = ThreadAlloc::get();
 
   constexpr size_t count_log = snmalloc::bits::is64() ? 5 : 3;
   constexpr size_t count = 1 << count_log;
@@ -244,7 +244,7 @@ void test_external_pointer_large()
 
 void test_external_pointer_dealloc_bug()
 {
-  auto& alloc = ThreadAlloc::get();
+  auto alloc = ThreadAlloc::get();
   constexpr size_t count = (SUPERSLAB_SIZE / SLAB_SIZE) * 2;
   void* allocs[count];
 
@@ -268,7 +268,7 @@ void test_external_pointer_dealloc_bug()
 
 void test_alloc_16M()
 {
-  auto& alloc = ThreadAlloc::get();
+  auto alloc = ThreadAlloc::get();
   // sizes >= 16M use large_alloc
   const size_t size = 16'000'000;
 
@@ -279,7 +279,7 @@ void test_alloc_16M()
 
 void test_calloc_16M()
 {
-  auto& alloc = ThreadAlloc::get();
+  auto alloc = ThreadAlloc::get();
   // sizes >= 16M use large_alloc
   const size_t size = 16'000'000;
 

src/test/perf/contention/contention.cc

@@ -75,7 +75,7 @@ size_t swapcount;
 
 void test_tasks_f(size_t id)
 {
-  Alloc*& a = ThreadAlloc::get();
+  Alloc* a = ThreadAlloc::get();
   xoroshiro::p128r32 r(id + 5000);
 
   for (size_t n = 0; n < swapcount; n++)
@@ -100,7 +100,7 @@ void test_tasks_f(size_t id)
 
 void test_tasks(size_t num_tasks, size_t count, size_t size)
 {
-  Alloc*& a = ThreadAlloc::get();
+  Alloc* a = ThreadAlloc::get();
 
   contention = new std::atomic<size_t*>[size];
   xoroshiro::p128r32 r;

src/test/perf/external_pointer/externalpointer.cc

@@ -49,7 +49,7 @@ namespace test
 
   void test_external_pointer(xoroshiro::p128r64& r)
   {
-    auto& alloc = ThreadAlloc::get();
+    auto alloc = ThreadAlloc::get();
 #ifdef NDEBUG
     static constexpr size_t iterations = 10000000;
 #else
@@ -79,9 +79,6 @@ namespace test
 int main(int, char**)
 {
   xoroshiro::p128r64 r;
-  // Force a per-thread allocator to actually exist.
-  void* p = ThreadAlloc::get()->alloc(16);
-  ThreadAlloc::get()->dealloc(p);
 #if NDEBUG
   size_t nn = 30;
 #else

src/test/perf/singlethread/singlethread.cc

@@ -7,7 +7,7 @@ using namespace snmalloc;
 template<ZeroMem zero_mem>
 void test_alloc_dealloc(size_t count, size_t size, bool write)
 {
-  auto*& alloc = ThreadAlloc::get();
+  auto* alloc = ThreadAlloc::get();
 
   DO_TIME(
     "Count: " << std::setw(6) << count << ", Size: " << std::setw(6) << size