Minor changes to mpscq fast path.

CMakeLists.txt

@@ -121,7 +121,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY)
   endmacro()
 
   macro(add_shim name)
-    add_library(${name} SHARED src/override/malloc.cc)
+    add_library(${name} SHARED src/override/malloc.cc src/override/new.cc)
     target_link_libraries(${name} snmalloc_lib)
     target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))")
     set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden)

src/ds/mpscq.h

@@ -3,6 +3,8 @@
 #include "bits.h"
 #include "helpers.h"
 
+#include <utility>
+
 namespace snmalloc
 {
   template<class T>
@@ -59,7 +61,7 @@ namespace snmalloc
       prev->next.store(first, std::memory_order_relaxed);
     }
 
-    T* dequeue()
+    std::pair<T*, bool> dequeue()
     {
       // Returns the front message, or null if not possible to return a message.
       invariant();
@@ -69,14 +71,13 @@ namespace snmalloc
       if (next != nullptr)
       {
         front = next;
-
         assert(front);
         std::atomic_thread_fence(std::memory_order_acquire);
         invariant();
-        return first;
+        return std::pair(first, true);
       }
 
-      return nullptr;
+      return std::pair(nullptr, false);
     }
   };
 } // namespace snmalloc

src/mem/alloc.h

@@ -649,7 +649,8 @@ namespace snmalloc
         return (id >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK;
       }
 
-      void dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass)
+      FAST_PATH void
+      dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass)
       {
         this->size += sizeclass_to_size(sizeclass);
 
@@ -838,9 +839,9 @@ namespace snmalloc
       message_queue().init(&stub);
     }
 
-    void handle_dealloc_remote(Remote* p)
+    FAST_PATH void handle_dealloc_remote(Remote* p)
     {
-      if (p != &stub)
+      if (likely(p != &stub))
       {
         Superslab* super = Superslab::get(p);
 
@@ -884,16 +885,16 @@ namespace snmalloc
     {
       for (size_t i = 0; i < REMOTE_BATCH; i++)
       {
-        Remote* r = message_queue().dequeue();
+        auto r = message_queue().dequeue();
 
-        if (r == nullptr)
+        if (unlikely(!r.second))
           break;
 
-        handle_dealloc_remote(r);
+        handle_dealloc_remote(r.first);
       }
 
       // Our remote queues may be larger due to forwarding remote frees.
-      if (remote.size < REMOTE_CACHE)
+      if (likely(remote.size < REMOTE_CACHE))
         return;
 
       stats().remote_post();