Implements protection on remote messages queues

This extends the freelist protection to the remote message queues. They
effectively perform doubly linked list entries for the message queue
with the enqueue operation first linking in the previous pointer, and
then then atomically setting the next.  This ensures that the visible
states always satisfy the invariant that the forward and backward
pointers are correct for any visisble object.

There is a key_global that is used for all remote deallocations. The
remote cache uses the same protection to build the temporary lists
before forwarding to the next allocator.

The mpscq is integrated into the remoteallocator as it is no longer
a reusable datastructure, but a special purpose implementation.

src/ds/mpscq.h

@@ -1,91 +0,0 @@
-#pragma once
-
-#include "bits.h"
-#include "helpers.h"
-
-#include <utility>
-namespace snmalloc
-{
-  template<
-    class T,
-    template<typename> typename Ptr = Pointer,
-    template<typename> typename AtomicPtr = AtomicPointer>
-  class MPSCQ
-  {
-  private:
-    static_assert(
-      std::is_same<decltype(T::next), AtomicPtr<T>>::value,
-      "T->next must be an AtomicPtr<T>");
-
-    AtomicPtr<T> back{nullptr};
-    Ptr<T> front{nullptr};
-
-  public:
-    constexpr MPSCQ() = default;
-
-    void invariant()
-    {
-      SNMALLOC_ASSERT(back != nullptr);
-      SNMALLOC_ASSERT(front != nullptr);
-    }
-
-    void init(Ptr<T> stub)
-    {
-      stub->next.store(nullptr, std::memory_order_relaxed);
-      front = stub;
-      back.store(stub, std::memory_order_relaxed);
-      invariant();
-    }
-
-    Ptr<T> destroy()
-    {
-      Ptr<T> fnt = front;
-      back.store(nullptr, std::memory_order_relaxed);
-      front = nullptr;
-      return fnt;
-    }
-
-    inline bool is_empty()
-    {
-      Ptr<T> bk = back.load(std::memory_order_relaxed);
-
-      return bk == front;
-    }
-
-    void enqueue(Ptr<T> first, Ptr<T> last)
-    {
-      // Pushes a list of messages to the queue. Each message from first to
-      // last should be linked together through their next pointers.
-      invariant();
-      last->next.store(nullptr, std::memory_order_relaxed);
-      std::atomic_thread_fence(std::memory_order_release);
-      Ptr<T> prev = back.exchange(last, std::memory_order_relaxed);
-      prev->next.store(first, std::memory_order_relaxed);
-    }
-
-    Ptr<T> peek()
-    {
-      return front;
-    }
-
-    std::pair<Ptr<T>, bool> dequeue()
-    {
-      // Returns the front message, or null if not possible to return a message.
-      invariant();
-      Ptr<T> first = front;
-      Ptr<T> next = first->next.load(std::memory_order_relaxed);
-
-      Aal::prefetch(&(next->next));
-      if (next != nullptr)
-      {
-        front = next;
-        SNMALLOC_ASSERT(front != nullptr);
-        std::atomic_thread_fence(std::memory_order_acquire);
-        invariant();
-        return {first, true};
-      }
-
-      return {nullptr, false};
-    }
-  };
-} // namespace snmalloc

src/mem/commonconfig.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "../ds/defines.h"
-#include "remoteallocator.h"
+#include "remotecache.h"
 
 namespace snmalloc
 {

src/mem/corealloc.h

@@ -95,7 +95,7 @@ namespace snmalloc
      */
     auto& message_queue()
     {
-      return public_state()->message_queue;
+      return *public_state();
     }
 
     /**
@@ -108,7 +108,7 @@ namespace snmalloc
       // Using an actual allocation removes a conditional from a critical path.
       auto dummy =
         CapPtr<void, CBAlloc>(small_alloc_one(sizeof(MIN_ALLOC_SIZE)))
-          .template as_static<Remote>();
+          .template as_static<FreeObject>();
       if (dummy == nullptr)
       {
         error("Critical error: Out-of-memory during initialisation.");
@@ -141,6 +141,8 @@ namespace snmalloc
     {
       auto slab_end = pointer_offset(bumpptr, slab_size + 1 - rsize);
 
+      FreeListKey key(entropy.get_constant_key());
+
       FreeListBuilder<false> b;
       SNMALLOC_ASSERT(b.empty());
 
@@ -187,14 +189,14 @@ namespace snmalloc
       auto curr_ptr = start_ptr;
       do
       {
-        b.add(FreeObject::make(curr_ptr.as_void()), entropy);
+        b.add(FreeObject::make(curr_ptr.as_void()), key);
         curr_ptr = curr_ptr->next;
       } while (curr_ptr != start_ptr);
 #else
       auto p = bumpptr;
       do
       {
-        b.add(Aal::capptr_bound<FreeObject, CBAlloc>(p, rsize), entropy);
+        b.add(Aal::capptr_bound<FreeObject, CBAlloc>(p, rsize), key);
         p = pointer_offset(p, rsize);
       } while (p < slab_end);
 #endif
@@ -202,14 +204,15 @@ namespace snmalloc
       bumpptr = slab_end;
 
       SNMALLOC_ASSERT(!b.empty());
-      b.close(fast_free_list, entropy);
+      b.close(fast_free_list, key);
     }
 
     ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass)
     {
+      FreeListKey key(entropy.get_constant_key());
       FreeListIter fl;
-      meta->free_queue.close(fl, entropy);
+      meta->free_queue.close(fl, key);
-      void* p = finish_alloc_no_zero(fl.take(entropy), sizeclass);
+      void* p = finish_alloc_no_zero(fl.take(key), sizeclass);
 
 #ifdef CHECK_CLIENT
       // Check free list is well-formed on platforms with
@@ -217,7 +220,7 @@ namespace snmalloc
       size_t count = 1; // Already taken one above.
       while (!fl.empty())
       {
-        fl.take(entropy);
+        fl.take(key);
         count++;
       }
       // Check the list contains all the elements
@@ -335,7 +338,7 @@ namespace snmalloc
         auto& entry = SharedStateHandle::Backend::get_meta_data(
           handle.get_backend_state(), snmalloc::address_cast(p));
 
-        auto r = message_queue().dequeue();
+        auto r = message_queue().dequeue(key_global);
 
         if (unlikely(!r.second))
           break;
@@ -360,7 +363,7 @@ namespace snmalloc
      * need_post will be set to true, if capacity is exceeded.
      */
     void handle_dealloc_remote(
-      const MetaEntry& entry, CapPtr<Remote, CBAlloc> p, bool& need_post)
+      const MetaEntry& entry, CapPtr<FreeObject, CBAlloc> p, bool& need_post)
     {
       // TODO this needs to not double count stats
       // TODO this needs to not double revoke if using MTE
@@ -381,7 +384,7 @@ namespace snmalloc
           need_post = true;
         attached_cache->remote_dealloc_cache
           .template dealloc<sizeof(CoreAllocator)>(
-            entry.get_remote()->trunc_id(), p.as_void());
+            entry.get_remote()->trunc_id(), p.as_void(), key_global);
       }
     }
 
@@ -431,7 +434,7 @@ namespace snmalloc
       // stats().remote_post();  // TODO queue not in line!
       bool sent_something =
         attached_cache->remote_dealloc_cache.post<sizeof(CoreAllocator)>(
-          handle, public_state()->trunc_id());
+          handle, public_state()->trunc_id(), key_global);
 
       return sent_something;
     }
@@ -472,8 +475,10 @@ namespace snmalloc
 
       auto cp = CapPtr<FreeObject, CBAlloc>(reinterpret_cast<FreeObject*>(p));
 
+      FreeListKey key(entropy.get_constant_key());
+
       // Update the head and the next pointer in the free list.
-      meta->free_queue.add(cp, entropy);
+      meta->free_queue.add(cp, key, entropy);
 
       return likely(!meta->return_object());
     }
@@ -533,8 +538,10 @@ namespace snmalloc
       // Set meta slab to empty.
       meta->initialise(sizeclass);
 
+      FreeListKey key(entropy.get_constant_key());
+
       // take an allocation from the free list
-      auto p = fast_free_list.take(entropy);
+      auto p = fast_free_list.take(key);
 
       return finish_alloc<zero_mem, SharedStateHandle>(p, sizeclass);
     }
@@ -550,12 +557,12 @@ namespace snmalloc
 
       if (destroy_queue)
       {
-        CapPtr<Remote, CBAlloc> p = message_queue().destroy();
+        auto p = message_queue().destroy();
 
         while (p != nullptr)
         {
           bool need_post = true; // Always going to post, so ignore.
-          auto n = p->non_atomic_next;
+          auto n = p->atomic_read_next(key_global);
           auto& entry = SharedStateHandle::Backend::get_meta_data(
             handle.get_backend_state(), snmalloc::address_cast(p));
           handle_dealloc_remote(entry, p, need_post);

src/mem/freelist.h

@@ -41,7 +41,18 @@
 
 namespace snmalloc
 {
-  struct Remote;
+  struct FreeListKey
+  {
+    address_t key;
+
+    FreeListKey(uint64_t key_)
+    {
+      if constexpr (bits::BITS == 64)
+        key = static_cast<address_t>(key_);
+      else
+        key = key_ & 0xffff'ffff;
+    }
+  };
 
   /**
    * This function is used to sign back pointers in the free list.
@@ -53,11 +64,11 @@ namespace snmalloc
    * list.
    */
   inline static uintptr_t
-  signed_prev(address_t curr, address_t next, LocalEntropy& entropy)
+  signed_prev(address_t curr, address_t next, FreeListKey& key)
   {
     auto c = curr;
     auto n = next;
-    auto k = entropy.get_constant_key();
+    auto k = key.key;
     return (c + k) * (n - k);
   }
 
@@ -66,10 +77,19 @@ namespace snmalloc
    * There is an optional second field that is effectively a
    * back pointer in a doubly linked list, however, it is encoded
    * to prevent corruption.
+   *
+   * TODO: Consider put prev_encoded at the end of the object, would
+   * require size to be threaded through, but would provide more OOB
+   * detection.
    */
   class FreeObject
   {
-    CapPtr<FreeObject, CBAlloc> next_object;
+    union
+    {
+      CapPtr<FreeObject, CBAlloc> next_object;
+      // TODO: Should really use C++20 atomic_ref rather than a union.
+      AtomicCapPtr<FreeObject, CBAlloc> atomic_next_object;
+    };
 #ifdef CHECK_CLIENT
     // Encoded representation of a back pointer.
     // Hard to fake, and provides consistency on
@@ -83,39 +103,66 @@ namespace snmalloc
       return p.template as_static<FreeObject>();
     }
 
-    /**
-     * Construct a FreeObject for local slabs from a Remote message.
-     */
-    static CapPtr<FreeObject, CBAlloc> make(CapPtr<Remote, CBAlloc> p)
-    {
-      // TODO: Zero the difference between a FreeObject and a Remote
-      return p.template as_reinterpret<FreeObject>();
-    }
-
     /**
      * Assign next_object and update its prev_encoded if CHECK_CLIENT.
-     *
      * Static so that it can be used on reference to a FreeObject.
      *
      * Returns a pointer to the next_object field of the next parameter as an
      * optimization for repeated snoc operations (in which
      * next->next_object is nullptr).
      */
-    static CapPtr<FreeObject, CBAlloc>* store(
+    static CapPtr<FreeObject, CBAlloc>* store_next(
       CapPtr<FreeObject, CBAlloc>* curr,
       CapPtr<FreeObject, CBAlloc> next,
-      LocalEntropy& entropy)
+      FreeListKey& key)
     {
-      *curr = next;
 #ifdef CHECK_CLIENT
       next->prev_encoded =
-        signed_prev(address_cast(curr), address_cast(next), entropy);
+        signed_prev(address_cast(curr), address_cast(next), key);
 #else
-      UNUSED(entropy);
+      UNUSED(key);
 #endif
+      *curr = next;
       return &(next->next_object);
     }
 
+    /**
+     * Assign next_object and update its prev_encoded if CHECK_CLIENT
+     *
+     * Uses the atomic view of next, so can be used in the message queues.
+     */
+    void atomic_store_next(CapPtr<FreeObject, CBAlloc> next, FreeListKey& key)
+    {
+#ifdef CHECK_CLIENT
+      next->prev_encoded =
+        signed_prev(address_cast(this), address_cast(next), key);
+#else
+      UNUSED(key);
+#endif
+      // Signature needs to be visible before item is linked in
+      // so requires release semantics.
+      atomic_next_object.store(next, std::memory_order_release);
+    }
+
+    void atomic_store_null()
+    {
+      atomic_next_object.store(nullptr, std::memory_order_relaxed);
+    }
+
+    CapPtr<FreeObject, CBAlloc> atomic_read_next(FreeListKey& key)
+    {
+      auto n = atomic_next_object.load(std::memory_order_acquire);
+#ifdef CHECK_CLIENT
+      if (n != nullptr)
+      {
+        n->check_prev(signed_prev(address_cast(this), address_cast(n), key));
+      }
+#else
+      UNUSED(key);
+#endif
+      return n;
+    }
+
     /**
      * Check the signature of this FreeObject
      */
@@ -135,6 +182,10 @@ namespace snmalloc
     }
   };
 
+  static_assert(
+    sizeof(FreeObject) <= MIN_ALLOC_SIZE,
+    "Needs to be able to fit in smallest allocation.");
+
   /**
    * Used to iterate a free list in object space.
    *
@@ -179,7 +230,7 @@ namespace snmalloc
     /**
      * Moves the iterator on, and returns the current value.
      */
-    CapPtr<FreeObject, CBAlloc> take(LocalEntropy& entropy)
+    CapPtr<FreeObject, CBAlloc> take(FreeListKey& key)
     {
       auto c = curr;
       auto next = curr->read_next();
@@ -188,9 +239,9 @@ namespace snmalloc
       curr = next;
 #ifdef CHECK_CLIENT
       c->check_prev(prev);
-      prev = signed_prev(address_cast(c), address_cast(next), entropy);
+      prev = signed_prev(address_cast(c), address_cast(next), key);
 #else
-      UNUSED(entropy);
+      UNUSED(key);
 #endif
 
       return c;
@@ -220,25 +271,23 @@ namespace snmalloc
    * If RANDOM is set to false, then the code does not perform any
    * randomisation.
    */
-  template<bool RANDOM, typename S = uint32_t>
+  template<bool RANDOM, bool INIT = true>
   class FreeListBuilder
   {
     static constexpr size_t LENGTH = RANDOM ? 2 : 1;
 
     // Pointer to the first element.
-    CapPtr<FreeObject, CBAlloc> head[LENGTH];
+    std::array<CapPtr<FreeObject, CBAlloc>, LENGTH> head;
     // Pointer to the reference to the last element.
     // In the empty case end[i] == &head[i]
     // This enables branch free enqueuing.
-    CapPtr<FreeObject, CBAlloc>* end[LENGTH];
+    std::array<CapPtr<FreeObject, CBAlloc>*, LENGTH> end{nullptr};
-
-  public:
-    S s;
 
   public:
     constexpr FreeListBuilder()
     {
-      init();
+      if (INIT)
+        init();
     }
 
     /**
@@ -257,7 +306,8 @@ namespace snmalloc
     /**
      * Adds an element to the builder
      */
-    void add(CapPtr<FreeObject, CBAlloc> n, LocalEntropy& entropy)
+    void
+    add(CapPtr<FreeObject, CBAlloc> n, FreeListKey& key, LocalEntropy& entropy)
     {
       uint32_t index;
       if constexpr (RANDOM)
@@ -265,33 +315,45 @@ namespace snmalloc
       else
         index = 0;
 
-      end[index] = FreeObject::store(end[index], n, entropy);
+      end[index] = FreeObject::store_next(end[index], n, key);
+    }
+
+    /**
+     * Adds an element to the builder, if we are guaranteed that
+     * RANDOM is false.  This is useful in certain construction
+     * cases that do not need to introduce randomness, such as
+     * during the initialation construction of a free list, which
+     * uses its own algorithm, or during building remote deallocation
+     * lists, which will be randomised at the other end.
+     */
+    template<bool RANDOM_ = RANDOM>
+    std::enable_if_t<!RANDOM_>
+    add(CapPtr<FreeObject, CBAlloc> n, FreeListKey& key)
+    {
+      static_assert(RANDOM_ == RANDOM, "Don't set template parameter");
+      end[0] = FreeObject::store_next(end[0], n, key);
     }
 
     /**
      * Makes a terminator to a free list.
-     *
-     * Termination uses the bottom bit, this allows the next pointer
-     * to always be to the same slab.
      */
-    SNMALLOC_FAST_PATH void
+    SNMALLOC_FAST_PATH void terminate_list(uint32_t index, FreeListKey& key)
-    terminate_list(uint32_t index, LocalEntropy& entropy)
     {
-      UNUSED(entropy);
+      UNUSED(key);
       *end[index] = nullptr;
     }
 
-    address_t get_fake_signed_prev(uint32_t index, LocalEntropy& entropy)
+    address_t get_fake_signed_prev(uint32_t index, FreeListKey& key)
     {
       return signed_prev(
-        address_cast(&head[index]), address_cast(head[index]), entropy);
+        address_cast(&head[index]), address_cast(head[index]), key);
     }
 
     /**
      * Close a free list, and set the iterator parameter
      * to iterate it.
      */
-    SNMALLOC_FAST_PATH void close(FreeListIter& fl, LocalEntropy& entropy)
+    SNMALLOC_FAST_PATH void close(FreeListIter& fl, FreeListKey& key)
     {
       if constexpr (RANDOM)
       {
@@ -303,27 +365,27 @@ namespace snmalloc
         {
           // The start token has been corrupted.
           // TOCTTOU issue, but small window here.
-          head[1]->check_prev(get_fake_signed_prev(1, entropy));
+          head[1]->check_prev(get_fake_signed_prev(1, key));
 
-          terminate_list(1, entropy);
+          terminate_list(1, key);
 
           // Append 1 to 0
-          FreeObject::store(end[0], head[1], entropy);
+          FreeObject::store_next(end[0], head[1], key);
 
           SNMALLOC_ASSERT(end[1] != &head[0]);
           SNMALLOC_ASSERT(end[0] != &head[1]);
         }
         else
         {
-          terminate_list(0, entropy);
+          terminate_list(0, key);
         }
       }
       else
       {
-        terminate_list(0, entropy);
+        terminate_list(0, key);
       }
 
-      fl = {head[0], get_fake_signed_prev(0, entropy)};
+      fl = {head[0], get_fake_signed_prev(0, key)};
       init();
     }
 
@@ -337,5 +399,22 @@ namespace snmalloc
         end[i] = &head[i];
       }
     }
+
+    std::pair<CapPtr<FreeObject, CBAlloc>, CapPtr<FreeObject, CBAlloc>>
+    extract_segment()
+    {
+      SNMALLOC_ASSERT(!empty());
+      SNMALLOC_ASSERT(!RANDOM); // TODO: Turn this into a static failure.
+
+      auto first = head[0];
+      // end[0] is pointing to the first field in the object,
+      // this is doing a CONTAINING_RECORD like cast to get back
+      // to the actual object.  This isn't true if the builder is
+      // empty, but you are not allowed to call this in the empty case.
+      auto last =
+        CapPtr<FreeObject, CBAlloc>(reinterpret_cast<FreeObject*>(end[0]));
+      init();
+      return {first, last};
+    }
   };
 } // namespace snmalloc

src/mem/globalconfig.h

@@ -73,6 +73,9 @@ namespace snmalloc
       if (initialised)
         return;
 
+      // Initialise key for remote deallocation lists
+      key_global = FreeListKey(get_entropy64<Backend::Pal>());
+
       // Need to initialise pagemap.
       backend_state.init();
 

src/mem/localalloc.h

@@ -228,7 +228,7 @@ namespace snmalloc
         MetaEntry entry = SharedStateHandle::Backend::get_meta_data(
           handle.get_backend_state(), address_cast(p));
         local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
-          entry.get_remote()->trunc_id(), CapPtr<void, CBAlloc>(p));
+          entry.get_remote()->trunc_id(), CapPtr<void, CBAlloc>(p), key_global);
         post_remote_cache();
         return;
       }
@@ -249,7 +249,7 @@ namespace snmalloc
      */
     auto& message_queue()
     {
-      return local_cache.remote_allocator->message_queue;
+      return local_cache.remote_allocator;
     }
 
   public:
@@ -382,7 +382,9 @@ namespace snmalloc
         if (local_cache.remote_dealloc_cache.reserve_space(entry))
         {
           local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
-            entry.get_remote()->trunc_id(), CapPtr<void, CBAlloc>(p));
+            entry.get_remote()->trunc_id(),
+            CapPtr<void, CBAlloc>(p),
+            key_global);
 #ifdef SNMALLOC_TRACING
           std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p)
                     << std::endl;

src/mem/localcache.h

@@ -76,6 +76,8 @@ namespace snmalloc
       typename SharedStateHandle>
     bool flush(DeallocFun dealloc, SharedStateHandle handle)
     {
+      FreeListKey key(entropy.get_constant_key());
+
       // Return all the free lists to the allocator.
       // Used during thread teardown
       for (size_t i = 0; i < NUM_SIZECLASSES; i++)
@@ -84,25 +86,27 @@ namespace snmalloc
         // call.
         while (!small_fast_free_lists[i].empty())
         {
-          auto p = small_fast_free_lists[i].take(entropy);
+          auto p = small_fast_free_lists[i].take(key);
           dealloc(finish_alloc_no_zero(p, i));
         }
       }
 
       return remote_dealloc_cache.post<allocator_size>(
-        handle, remote_allocator->trunc_id());
+        handle, remote_allocator->trunc_id(), key_global);
     }
 
     template<ZeroMem zero_mem, typename SharedStateHandle, typename Slowpath>
     SNMALLOC_FAST_PATH void* alloc(size_t size, Slowpath slowpath)
     {
+      FreeListKey key(entropy.get_constant_key());
+
       sizeclass_t sizeclass = size_to_sizeclass(size);
       stats.alloc_request(size);
       stats.sizeclass_alloc(sizeclass);
       auto& fl = small_fast_free_lists[sizeclass];
       if (likely(!fl.empty()))
       {
-        auto p = fl.take(entropy);
+        auto p = fl.take(key);
         return finish_alloc<zero_mem, SharedStateHandle>(p, sizeclass);
       }
       return slowpath(sizeclass, &fl);

src/mem/metaslab.h

@@ -18,34 +18,6 @@ namespace snmalloc
     sizeof(SlabLink) <= MIN_ALLOC_SIZE,
     "Need to be able to pack a SlabLink into any free small alloc");
 
-  /**
-   * This struct is used inside FreeListBuilder to account for the
-   * alignment space that is wasted in sizeof.
-   *
-   * This is part of Metaslab abstraction.
-   */
-  struct MetaslabEnd
-  {
-    /**
-     * The number of deallocation required until we hit a slow path. This
-     * counts down in two different ways that are handled the same on the
-     * fast path.  The first is
-     *   - deallocations until the slab has sufficient entries to be considered
-     *   useful to allocate from.  This could be as low as 1, or when we have
-     *   a requirement for entropy then it could be much higher.
-     *   - deallocations until the slab is completely unused.  This is needed
-     *   to be detected, so that the statistics can be kept up to date, and
-     *   potentially return memory to the a global pool of slabs/chunks.
-     */
-    uint16_t needed = 0;
-
-    /**
-     * Flag that is used to indicate that the slab is currently not active.
-     * I.e. it is not in a CoreAllocator cache for the appropriate sizeclass.
-     */
-    bool sleeping = false;
-  };
-
   // The Metaslab represent the status of a single slab.
   // This can be either a short or a standard slab.
   class alignas(CACHELINE_SIZE) Metaslab : public SlabLink
@@ -59,19 +31,38 @@ namespace snmalloc
      *  Spare 32bits are used for the fields in MetaslabEnd.
      */
 #ifdef CHECK_CLIENT
-    FreeListBuilder<true, MetaslabEnd> free_queue;
+    FreeListBuilder<true> free_queue;
 #else
-    FreeListBuilder<false, MetaslabEnd> free_queue;
+    FreeListBuilder<false> free_queue;
 #endif
 
+    /**
+     * The number of deallocation required until we hit a slow path. This
+     * counts down in two different ways that are handled the same on the
+     * fast path.  The first is
+     *   - deallocations until the slab has sufficient entries to be considered
+     *   useful to allocate from.  This could be as low as 1, or when we have
+     *   a requirement for entropy then it could be much higher.
+     *   - deallocations until the slab is completely unused.  This is needed
+     *   to be detected, so that the statistics can be kept up to date, and
+     *   potentially return memory to the a global pool of slabs/chunks.
+     */
+    uint16_t needed_ = 0;
+
+    /**
+     * Flag that is used to indicate that the slab is currently not active.
+     * I.e. it is not in a CoreAllocator cache for the appropriate sizeclass.
+     */
+    bool sleeping_ = false;
+
     uint16_t& needed()
     {
-      return free_queue.s.needed;
+      return needed_;
     }
 
     bool& sleeping()
     {
-      return free_queue.s.sleeping;
+      return sleeping_;
     }
 
     /**
@@ -151,13 +142,17 @@ namespace snmalloc
       LocalEntropy& entropy,
       sizeclass_t sizeclass)
     {
+      FreeListKey key(entropy.get_constant_key());
+
       FreeListIter tmp_fl;
-      meta->free_queue.close(tmp_fl, entropy);
+      meta->free_queue.close(tmp_fl, key);
-      auto p = tmp_fl.take(entropy);
+      auto p = tmp_fl.take(key);
       fast_free_list = tmp_fl;
 
 #ifdef CHECK_CLIENT
       entropy.refresh_bits();
+#else
+      UNUSED(entropy);
 #endif
 
       // Treat stealing the free list as allocating it all.

src/mem/remoteallocator.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include "../ds/mpscq.h"
 #include "../mem/allocconfig.h"
 #include "../mem/freelist.h"
 #include "../mem/metaslab.h"
@@ -9,59 +8,102 @@
 #include <array>
 #include <atomic>
 
-#ifdef CHECK_CLIENT
-#  define SNMALLOC_DONT_CACHE_ALLOCATOR_PTR
-#endif
-
 namespace snmalloc
 {
-  /*
-   * A region of memory destined for a remote allocator's dealloc() via the
-   * message passing system.  This structure is placed at the beginning of
-   * the allocation itself when it is queued for sending.
-   */
-  struct Remote
-  {
-    using alloc_id_t = size_t;
-    union
-    {
-      CapPtr<Remote, CBAlloc> non_atomic_next;
-      AtomicCapPtr<Remote, CBAlloc> next{nullptr};
-    };
-
-    constexpr Remote() : next(nullptr) {}
-
-    /** Zero out a Remote tracking structure, return pointer to object base */
-    template<capptr_bounds B>
-    SNMALLOC_FAST_PATH static CapPtr<void, B> clear(CapPtr<Remote, B> self)
-    {
-      pal_zero<Pal>(self, sizeof(Remote));
-      return self.as_void();
-    }
-  };
-
-  static_assert(
-    sizeof(Remote) <= MIN_ALLOC_SIZE,
-    "Needs to be able to fit in smallest allocation.");
-
   // Remotes need to be aligned enough that all the
   // small size classes can fit in the bottom bits.
   static constexpr size_t REMOTE_MIN_ALIGN = bits::min<size_t>(
     CACHELINE_SIZE, bits::next_pow2_const(NUM_SIZECLASSES + 1));
 
+  /**
+   * Global key for all remote lists.
+   */
+  inline static FreeListKey key_global(0xdeadbeef);
+
   struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator
   {
-    using alloc_id_t = Remote::alloc_id_t;
+    using alloc_id_t = address_t;
+
     // Store the message queue on a separate cacheline. It is mutable data that
     // is read by other threads.
+    alignas(CACHELINE_SIZE) AtomicCapPtr<FreeObject, CBAlloc> back{nullptr};
+    // Store the two ends on different cache lines as access by different
+    // threads.
+    alignas(CACHELINE_SIZE) CapPtr<FreeObject, CBAlloc> front{nullptr};
 
-    MPSCQ<Remote, CapPtrCBAlloc, AtomicCapPtrCBAlloc> message_queue;
+    constexpr RemoteAllocator() = default;
+
+    void invariant()
+    {
+      SNMALLOC_ASSERT(back != nullptr);
+      SNMALLOC_ASSERT(front != nullptr);
+    }
+
+    void init(CapPtr<FreeObject, CBAlloc> stub)
+    {
+      stub->atomic_store_null();
+      front = stub;
+      back.store(stub, std::memory_order_relaxed);
+      invariant();
+    }
+
+    CapPtr<FreeObject, CBAlloc> destroy()
+    {
+      CapPtr<FreeObject, CBAlloc> fnt = front;
+      back.store(nullptr, std::memory_order_relaxed);
+      front = nullptr;
+      return fnt;
+    }
+
+    inline bool is_empty()
+    {
+      CapPtr<FreeObject, CBAlloc> bk = back.load(std::memory_order_relaxed);
+
+      return bk == front;
+    }
+
+    void enqueue(
+      CapPtr<FreeObject, CBAlloc> first,
+      CapPtr<FreeObject, CBAlloc> last,
+      FreeListKey& key)
+    {
+      // Pushes a list of messages to the queue. Each message from first to
+      // last should be linked together through their next pointers.
+      invariant();
+      last->atomic_store_null();
+
+      // exchange needs to be a release, so nullptr in next is visible.
+      CapPtr<FreeObject, CBAlloc> prev =
+        back.exchange(last, std::memory_order_release);
+
+      prev->atomic_store_next(first, key);
+    }
+
+    CapPtr<FreeObject, CBAlloc> peek()
+    {
+      return front;
+    }
+
+    std::pair<CapPtr<FreeObject, CBAlloc>, bool> dequeue(FreeListKey& key)
+    {
+      // Returns the front message, or null if not possible to return a message.
+      invariant();
+      CapPtr<FreeObject, CBAlloc> first = front;
+      CapPtr<FreeObject, CBAlloc> next = first->atomic_read_next(key);
+
+      if (next != nullptr)
+      {
+        front = next;
+        invariant();
+        return {first, true};
+      }
+
+      return {nullptr, false};
+    }
 
     alloc_id_t trunc_id()
     {
-      return static_cast<alloc_id_t>(
+      return address_cast(this);
-               reinterpret_cast<uintptr_t>(&message_queue)) &
-        ~SIZECLASS_MASK;
     }
   };
 } // namespace snmalloc

src/mem/remotecache.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include "../ds/mpscq.h"
 #include "../mem/allocconfig.h"
 #include "../mem/freelist.h"
 #include "../mem/metaslab.h"
@@ -17,40 +16,7 @@ namespace snmalloc
    */
   struct RemoteDeallocCache
   {
-    /*
+    std::array<FreeListBuilder<false, false>, REMOTE_SLOTS> list;
-     * A singly-linked list of Remote objects, supporting append and
-     * take-all operations.  Intended only for the private use of this
-     * allocator; the Remote objects here will later be taken and pushed
-     * to the inter-thread message queues.
-     */
-    struct RemoteList
-    {
-      /*
-       * A stub Remote object that will always be the head of this list;
-       * never taken for further processing.
-       */
-      Remote head{};
-
-      /**
-       * Initially is null ptr, and needs to be non-null before anything runs on
-       * this.
-       */
-      CapPtr<Remote, CBAlloc> last{nullptr};
-
-      void clear()
-      {
-        last = CapPtr<Remote, CBAlloc>(&head);
-      }
-
-      bool empty()
-      {
-        return address_cast(last) == address_cast(&head);
-      }
-
-      constexpr RemoteList() = default;
-    };
-
-    std::array<RemoteList, REMOTE_SLOTS> list{};
 
     /**
      * The total amount of memory we are waiting for before we will dispatch
@@ -98,19 +64,22 @@ namespace snmalloc
     }
 
     template<size_t allocator_size>
-    SNMALLOC_FAST_PATH void
+    SNMALLOC_FAST_PATH void dealloc(
-    dealloc(Remote::alloc_id_t target_id, CapPtr<void, CBAlloc> p)
+      RemoteAllocator::alloc_id_t target_id,
+      CapPtr<void, CBAlloc> p,
+      FreeListKey& key)
     {
       SNMALLOC_ASSERT(initialised);
-      auto r = p.template as_reinterpret<Remote>();
+      auto r = p.template as_reinterpret<FreeObject>();
 
-      RemoteList* l = &list[get_slot<allocator_size>(target_id, 0)];
+      list[get_slot<allocator_size>(target_id, 0)].add(r, key);
-      l->last->non_atomic_next = r;
-      l->last = r;
     }
 
     template<size_t allocator_size, typename SharedStateHandle>
-    bool post(SharedStateHandle handle, Remote::alloc_id_t id)
+    bool post(
+      SharedStateHandle handle,
+      RemoteAllocator::alloc_id_t id,
+      FreeListKey& key)
     {
       SNMALLOC_ASSERT(initialised);
       size_t post_round = 0;
@@ -125,46 +94,37 @@ namespace snmalloc
           if (i == my_slot)
             continue;
 
-          RemoteList* l = &list[i];
+          if (!list[i].empty())
-          CapPtr<Remote, CBAlloc> first = l->head.non_atomic_next;
-
-          if (!l->empty())
           {
+            auto [first, last] = list[i].extract_segment();
             MetaEntry entry = SharedStateHandle::Backend::get_meta_data(
               handle.get_backend_state(), address_cast(first));
-            entry.get_remote()->message_queue.enqueue(first, l->last);
+            entry.get_remote()->enqueue(first, last, key);
-            l->clear();
             sent_something = true;
           }
         }
 
-        RemoteList* resend = &list[my_slot];
+        if (list[my_slot].empty())
-        if (resend->empty())
           break;
 
         // Entries could map back onto the "resend" list,
         // so take copy of the head, mark the last element,
         // and clear the original list.
-        CapPtr<Remote, CBAlloc> r = resend->head.non_atomic_next;
+        FreeListIter resend;
-        resend->last->non_atomic_next = nullptr;
+        list[my_slot].close(resend, key);
-        resend->clear();
 
         post_round++;
 
-        while (r != nullptr)
+        while (!resend.empty())
         {
           // Use the next N bits to spread out remote deallocs in our own
           // slot.
+          auto r = resend.take(key);
           MetaEntry entry = SharedStateHandle::Backend::get_meta_data(
             handle.get_backend_state(), address_cast(r));
           auto i = entry.get_remote()->trunc_id();
-          // TODO correct size for slot offset
           size_t slot = get_slot<allocator_size>(i, post_round);
-          RemoteList* l = &list[slot];
+          list[slot].add(r, key);
-          l->last->non_atomic_next = r;
-          l->last = r;
-
-          r = r->non_atomic_next;
         }
       }
 
@@ -190,8 +150,7 @@ namespace snmalloc
 #endif
       for (auto& l : list)
       {
-        SNMALLOC_ASSERT(l.last == nullptr || l.empty());
+        l.init();
-        l.clear();
       }
       capacity = REMOTE_CACHE;
     }