Move to FreeObject::T<capptr::bound<>>

FreeObject itself is now just a namespace (but `friend`-ly); the actual free
list nodes are FreeObject::T-s that are templatized on the (perceived)
`capptr::bound<>` of the pointer they contain.  (These may differ across an
instantiated snmalloc; for example, in the sandboxing design, the in-sandbox
allocators may perceive all remotes to be full of `AllocUser` while the
privileged allocator of sandbox memory should perceive its remote queue as
holding `AllocUserWild` pointers in need of domestication.)

The interfaces to `FreeObject::T`-s now let us distinguish between the base and
inductive cases of the queues:

* in the inductive case, the pointer we hold to a `FreeObject::T` and its
  next_object have the same bounds

* in the base case, the pointer we hold has different bounds (typically,
  domesticated by contrast to the wild pointers in the queues).

To keep the clutter down a bit, we occasionally use raw pointers when we can be
reasonably certain that domestication is assured.  Moreover, we define some type
aliases, `FreeObject::{HeadPtr, QueuePtr, AtomicQueuePtr}`, that are slightly
more convenient labels than, e.g., `CapPtr<FreeObject::T<BQueue>, BView>`.
Because we are using template parameters for the `capptr::bound<>`s themselves,
we cannot use the aliases for `CapPtr<>s` provided within `capptr::`.

The two primary interfaces around free objects (`FreeListIter` AND
`FreeListBuilder`) are adjusted appropriately and their `BView` and `BQueue`
template paramters are plumbed explicitly around the tree.  This makes for quite
a bit of noise at the moment, but means that we'll be able to evolve parts of
the tree separately and can consider putting defaults in once that's done.

src/mem/corealloc.h

@@ -161,8 +161,9 @@ namespace snmalloc
     {
       // Manufacture an allocation to prime the queue
       // Using an actual allocation removes a conditional from a critical path.
-      auto dummy = capptr::AllocFull<void>(small_alloc_one(MIN_ALLOC_SIZE))
-                     .template as_static<FreeObject>();
+      auto dummy =
+        capptr::AllocFull<void>(small_alloc_one(MIN_ALLOC_SIZE))
+          .template as_static<FreeObject::T<capptr::bounds::AllocFull>>();
       if (dummy == nullptr)
       {
         error("Critical error: Out-of-memory during initialisation.");
@@ -181,9 +182,11 @@ namespace snmalloc
       SNMALLOC_ASSERT(attached_cache != nullptr);
       // Use attached cache, and fill it if it is empty.
       return attached_cache->template alloc<NoZero, SharedStateHandle>(
-        size, [&](sizeclass_t sizeclass, FreeListIter* fl) {
-          return small_alloc<NoZero>(sizeclass, *fl);
-        });
+        size,
+        [&](
+          sizeclass_t sizeclass,
+          FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>*
+            fl) { return small_alloc<NoZero>(sizeclass, *fl); });
     }
 
     static SNMALLOC_FAST_PATH void alloc_new_list(
@@ -244,7 +247,10 @@ namespace snmalloc
       auto curr_ptr = start_ptr;
       do
       {
-        b.add(FreeObject::make(curr_ptr.as_void()), key, entropy);
+        b.add(
+          FreeObject::make<capptr::bounds::AllocFull>(curr_ptr.as_void()),
+          key,
+          entropy);
         curr_ptr = curr_ptr->next;
       } while (curr_ptr != start_ptr);
 #else
@@ -252,7 +258,9 @@ namespace snmalloc
       do
       {
         b.add(
-          Aal::capptr_bound<FreeObject, capptr::bounds::AllocFull>(p, rsize),
+          Aal::capptr_bound<
+            FreeObject::T<capptr::bounds::AllocFull>,
+            capptr::bounds::AllocFull>(p, rsize),
           key);
         p = pointer_offset(p, rsize);
       } while (p < slab_end);
@@ -264,7 +272,7 @@ namespace snmalloc
     ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass)
     {
       auto& key = entropy.get_free_list_key();
-      FreeListIter fl;
+      FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull> fl;
       auto more = meta->free_queue.close(fl, key);
       UNUSED(more);
       void* p = finish_alloc_no_zero(fl.take(key), sizeclass);
@@ -426,7 +434,9 @@ namespace snmalloc
      * need_post will be set to true, if capacity is exceeded.
      */
     void handle_dealloc_remote(
-      const MetaEntry& entry, capptr::AllocFull<FreeObject> p, bool& need_post)
+      const MetaEntry& entry,
+      FreeObject::QueuePtr<capptr::bounds::AllocFull> p,
+      bool& need_post)
     {
       // TODO this needs to not double count stats
       // TODO this needs to not double revoke if using MTE
@@ -580,7 +590,8 @@ namespace snmalloc
         Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)),
         "Not deallocating start of an object");
 
-      auto cp = capptr::AllocFull<FreeObject>(reinterpret_cast<FreeObject*>(p));
+      auto cp = FreeObject::QueuePtr<capptr::bounds::AllocFull>(
+        reinterpret_cast<FreeObject::T<capptr::bounds::AllocFull>*>(p));
 
       auto& key = entropy.get_free_list_key();
 
@@ -591,8 +602,10 @@ namespace snmalloc
     }
 
     template<ZeroMem zero_mem>
-    SNMALLOC_SLOW_PATH void*
-    small_alloc(sizeclass_t sizeclass, FreeListIter& fast_free_list)
+    SNMALLOC_SLOW_PATH void* small_alloc(
+      sizeclass_t sizeclass,
+      FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>&
+        fast_free_list)
     {
       size_t rsize = sizeclass_to_size(sizeclass);
 
@@ -650,7 +663,10 @@ namespace snmalloc
 
     template<ZeroMem zero_mem>
     SNMALLOC_SLOW_PATH void* small_alloc_slow(
-      sizeclass_t sizeclass, FreeListIter& fast_free_list, size_t rsize)
+      sizeclass_t sizeclass,
+      FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>&
+        fast_free_list,
+      size_t rsize)
     {
       // No existing free list get a new slab.
       size_t slab_size = sizeclass_to_slab_size(sizeclass);
@@ -712,6 +728,7 @@ namespace snmalloc
           auto& entry = SharedStateHandle::Pagemap::get_metaentry(
             backend_state_ptr(), snmalloc::address_cast(p));
           handle_dealloc_remote(entry, p, need_post);
+          // XXX n is not known to be domesticated
           p = n;
         }
       }

src/mem/freelist.h

@@ -52,44 +52,136 @@ namespace snmalloc
     return (c + key.key1) * (n + key.key2);
   }
 
-  /**
-   * Free objects within each slab point directly to the next.
-   * There is an optional second field that is effectively a
-   * back pointer in a doubly linked list, however, it is encoded
-   * to prevent corruption.
-   *
-   * TODO: Consider putting prev_encoded at the end of the object, would
-   * require size to be threaded through, but would provide more OOB
-   * detection.
-   */
   class FreeObject
   {
-    union
-    {
-      capptr::AllocFull<FreeObject> next_object;
-      // TODO: Should really use C++20 atomic_ref rather than a union.
-      capptr::AtomicAllocFull<FreeObject> atomic_next_object;
-    };
-#ifdef SNMALLOC_CHECK_CLIENT
-    // Encoded representation of a back pointer.
-    // Hard to fake, and provides consistency on
-    // the next pointers.
-    address_t prev_encoded;
-#endif
-
   public:
-    static capptr::AllocFull<FreeObject> make(capptr::AllocFull<void> p)
-    {
-      return p.template as_static<FreeObject>();
-    }
+    template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    class T;
 
     /**
-     * Encode next
+     * This "inductive step" type -- a queue-annotated pointer to a FreeObject
+     * containing a queue-annotated pointer -- shows up all over the place.
+     * Give it a shorter name (FreeObject::QueuePtr<BQueue>) for convenience.
      */
-    inline static capptr::AllocFull<FreeObject> encode_next(
-      address_t curr,
-      capptr::AllocFull<FreeObject> next,
-      const FreeListKey& key)
+    template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    using QueuePtr = CapPtr<FreeObject::T<BQueue>, BQueue>;
+
+    /**
+     * As with QueuePtr, but atomic.
+     */
+    template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    using AtomicQueuePtr = AtomicCapPtr<FreeObject::T<BQueue>, BQueue>;
+
+    /**
+     * This is the "base case" of that induction.  While we can't get rid of the
+     * two different type parameters (in general), we can at least get rid of a
+     * bit of the clutter.  "FreeObject::HeadPtr<BView, BQueue>" looks a little
+     * nicer than "CapPtr<FreeObject::T<BQueue>, BView>".
+     */
+    template<
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    using HeadPtr = CapPtr<FreeObject::T<BQueue>, BView>;
+
+    /**
+     * As with HeadPtr, but atomic.
+     */
+    template<
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    using AtomicHeadPtr = AtomicCapPtr<FreeObject::T<BQueue>, BView>;
+
+    /**
+     * Free objects within each slab point directly to the next.
+     * There is an optional second field that is effectively a
+     * back pointer in a doubly linked list, however, it is encoded
+     * to prevent corruption.
+     *
+     * This is an inner class to avoid the need to specify BQueue when calling
+     * static methods.
+     *
+     * Raw C++ pointers to this type are *assumed to be domesticated*.  In some
+     * cases we still explicitly annotate domesticated FreeObject*-s as
+     * CapPtr<>, but more often CapPtr<FreeObject::T<A>,B> will have B = A.
+     *
+     * TODO: Consider putting prev_encoded at the end of the object, would
+     * require size to be threaded through, but would provide more OOB
+     * detection.
+     */
+    template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    class T
+    {
+      friend class FreeObject;
+
+      union
+      {
+        QueuePtr<BQueue> next_object;
+        // TODO: Should really use C++20 atomic_ref rather than a union.
+        AtomicQueuePtr<BQueue> atomic_next_object;
+      };
+#ifdef SNMALLOC_CHECK_CLIENT
+      // Encoded representation of a back pointer.
+      // Hard to fake, and provides consistency on
+      // the next pointers.
+      address_t prev_encoded;
+#endif
+
+    public:
+      QueuePtr<BQueue> atomic_read_next(const FreeListKey& key)
+      {
+        auto n = FreeObject::decode_next(
+          address_cast(&this->next_object),
+          this->atomic_next_object.load(std::memory_order_acquire),
+          key);
+#ifdef SNMALLOC_CHECK_CLIENT
+        // XXX n is not known to be domesticated here
+        if (n != nullptr)
+        {
+          n->check_prev(signed_prev(address_cast(this), address_cast(n), key));
+        }
+#else
+        UNUSED(key);
+#endif
+        return n;
+      }
+
+      /**
+       * Read the next pointer
+       */
+      QueuePtr<BQueue> read_next(const FreeListKey& key)
+      {
+        return FreeObject::decode_next(
+          address_cast(&this->next_object), this->next_object, key);
+      }
+
+      /**
+       * Check the signature of this FreeObject
+       */
+      void check_prev(address_t signed_prev)
+      {
+        UNUSED(signed_prev);
+        check_client(
+          signed_prev == this->prev_encoded,
+          "Heap corruption - free list corrupted!");
+      }
+    };
+
+    // Note the inverted template argument order, since BView is inferable.
+    template<
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue,
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BView>
+    static HeadPtr<BView, BQueue> make(CapPtr<void, BView> p)
+    {
+      return p.template as_static<FreeObject::T<BQueue>>();
+    }
+
+  private:
+    /**
+     * Involutive encryption with raw pointers
+     */
+    template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    inline static FreeObject::T<BQueue>* code_next(
+      address_t curr, FreeObject::T<BQueue>* next, const FreeListKey& key)
     {
       // Note we can consider other encoding schemes here.
       //   * XORing curr and next.  This doesn't require any key material
@@ -100,8 +192,8 @@ namespace snmalloc
 
       if constexpr (CHECK_CLIENT && !aal_supports<StrictProvenance>)
       {
-        return capptr::AllocFull<FreeObject>(reinterpret_cast<FreeObject*>(
-          reinterpret_cast<uintptr_t>(next.unsafe_ptr()) ^ key.key_next));
+        return reinterpret_cast<FreeObject::T<BQueue>*>(
+          reinterpret_cast<uintptr_t>(next) ^ key.key_next);
       }
       else
       {
@@ -110,19 +202,82 @@ namespace snmalloc
       }
     }
 
+  public:
     /**
-     * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT.
-     * Static so that it can be used on reference to a FreeObject.
+     * Encode next.  We perform two convenient little bits of type-level
+     * sleight of hand here:
+     *
+     *  1) We convert the provided HeadPtr to a QueuePtr, forgetting BView in
+     *  the result; all the callers write the result through a pointer to a
+     *  QueuePtr, though, strictly, the result itself is no less domesticated
+     *  than the input (even if it is obfuscated).
+     *
+     *  2) Speaking of obfuscation, we continue to use a CapPtr<> type even
+     *  though the result is likely not safe to dereference, being an
+     *  obfuscated bundle of bits (on non-CHERI architectures, anyway).  That's
+     *  additional motivation to consider the result BQueue-bounded, as that is
+     *  likely (but not necessarily) Wild.
+     */
+    template<
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    inline static QueuePtr<BQueue> encode_next(
+      address_t curr, HeadPtr<BView, BQueue> next, const FreeListKey& key)
+    {
+      return QueuePtr<BQueue>(code_next(curr, next.unsafe_ptr(), key));
+    }
+
+    /**
+     * Decode next.  While traversing a queue, BView and BQueue here will
+     * often be equal (i.e., CBAllocExportWild) rather than dichotomous.
+     * However, we do occasionally decode an actual head pointer, so be
+     * polymorphic here.
+     */
+    template<
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    inline static HeadPtr<BView, BQueue> decode_next(
+      address_t curr, HeadPtr<BView, BQueue> next, const FreeListKey& key)
+    {
+      return HeadPtr<BView, BQueue>(code_next(curr, next.unsafe_ptr(), key));
+    }
+
+    template<
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    static void assert_view_queue_bounds()
+    {
+      static_assert(
+        BView::wildness == capptr::dimension::Wildness::Tame,
+        "FreeObject View must be domesticated, justifying raw pointers");
+
+      static_assert(
+        std::is_same_v<
+          typename BQueue::template with_wildness<
+            capptr::dimension::Wildness::Tame>,
+          BView>,
+        "FreeObject Queue bounds must match View bounds (but may be Wild)");
+    }
+
+    /**
+     * Assign next_object and update its prev_encoded if
+     * SNMALLOC_CHECK_CLIENT. Static so that it can be used on reference to a
+     * FreeObject.
      *
      * Returns a pointer to the next_object field of the next parameter as an
      * optimization for repeated snoc operations (in which
      * next->next_object is nullptr).
      */
-    static capptr::AllocFull<FreeObject>* store_next(
-      capptr::AllocFull<FreeObject>* curr,
-      capptr::AllocFull<FreeObject> next,
+    template<
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    static QueuePtr<BQueue>* store_next(
+      QueuePtr<BQueue>* curr,
+      HeadPtr<BView, BQueue> next,
       const FreeListKey& key)
     {
+      assert_view_queue_bounds<BView, BQueue>();
+
 #ifdef SNMALLOC_CHECK_CLIENT
       next->prev_encoded =
         signed_prev(address_cast(curr), address_cast(next), key);
@@ -133,10 +288,10 @@ namespace snmalloc
       return &(next->next_object);
     }
 
-    static void
-    store_null(capptr::AllocFull<FreeObject>* curr, const FreeListKey& key)
+    template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    static void store_null(QueuePtr<BQueue>* curr, const FreeListKey& key)
     {
-      *curr = encode_next(address_cast(curr), nullptr, key);
+      *curr = encode_next(address_cast(curr), QueuePtr<BQueue>(nullptr), key);
     }
 
     /**
@@ -144,63 +299,42 @@ namespace snmalloc
      *
      * Uses the atomic view of next, so can be used in the message queues.
      */
-    void atomic_store_next(
-      capptr::AllocFull<FreeObject> next, const FreeListKey& key)
+    template<
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    static void atomic_store_next(
+      HeadPtr<BView, BQueue> curr,
+      HeadPtr<BView, BQueue> next,
+      const FreeListKey& key)
     {
+      static_assert(BView::wildness == capptr::dimension::Wildness::Tame);
+
 #ifdef SNMALLOC_CHECK_CLIENT
       next->prev_encoded =
-        signed_prev(address_cast(this), address_cast(next), key);
+        signed_prev(address_cast(curr), address_cast(next), key);
 #else
       UNUSED(key);
 #endif
       // Signature needs to be visible before item is linked in
       // so requires release semantics.
-      atomic_next_object.store(
-        encode_next(address_cast(&next_object), next, key),
+      curr->atomic_next_object.store(
+        encode_next(address_cast(&curr->next_object), next, key),
         std::memory_order_release);
     }
 
-    void atomic_store_null(const FreeListKey& key)
+    template<
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
+      SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
+    static void
+    atomic_store_null(HeadPtr<BView, BQueue> curr, const FreeListKey& key)
     {
-      atomic_next_object.store(
-        encode_next(address_cast(&next_object), nullptr, key),
+      static_assert(BView::wildness == capptr::dimension::Wildness::Tame);
+
+      curr->atomic_next_object.store(
+        encode_next(
+          address_cast(&curr->next_object), QueuePtr<BQueue>(nullptr), key),
         std::memory_order_relaxed);
     }
-
-    capptr::AllocFull<FreeObject> atomic_read_next(const FreeListKey& key)
-    {
-      auto n = encode_next(
-        address_cast(&next_object),
-        atomic_next_object.load(std::memory_order_acquire),
-        key);
-#ifdef SNMALLOC_CHECK_CLIENT
-      if (n != nullptr)
-      {
-        n->check_prev(signed_prev(address_cast(this), address_cast(n), key));
-      }
-#else
-      UNUSED(key);
-#endif
-      return n;
-    }
-
-    /**
-     * Check the signature of this FreeObject
-     */
-    void check_prev(address_t signed_prev)
-    {
-      UNUSED(signed_prev);
-      check_client(
-        signed_prev == prev_encoded, "Heap corruption - free list corrupted!");
-    }
-
-    /**
-     * Read the next pointer
-     */
-    capptr::AllocFull<FreeObject> read_next(const FreeListKey& key)
-    {
-      return encode_next(address_cast(&next_object), next_object, key);
-    }
   };
 
   static_assert(
@@ -212,16 +346,19 @@ namespace snmalloc
    *
    * Checks signing of pointers
    */
+  template<
+    SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
+    SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
   class FreeListIter
   {
-    capptr::AllocFull<FreeObject> curr{nullptr};
+    FreeObject::HeadPtr<BView, BQueue> curr{nullptr};
 #ifdef SNMALLOC_CHECK_CLIENT
     address_t prev{0};
 #endif
 
   public:
     constexpr FreeListIter(
-      capptr::AllocFull<FreeObject> head, address_t prev_value)
+      FreeObject::HeadPtr<BView, BQueue> head, address_t prev_value)
     : curr(head)
     {
 #ifdef SNMALLOC_CHECK_CLIENT
@@ -243,7 +380,7 @@ namespace snmalloc
     /**
      * Returns current head without affecting the iterator.
      */
-    capptr::AllocFull<FreeObject> peek()
+    FreeObject::HeadPtr<BView, BQueue> peek()
     {
       return curr;
     }
@@ -251,7 +388,7 @@ namespace snmalloc
     /**
      * Moves the iterator on, and returns the current value.
      */
-    capptr::AllocFull<FreeObject> take(const FreeListKey& key)
+    FreeObject::HeadPtr<BView, BQueue> take(const FreeListKey& key)
     {
       auto c = curr;
       auto next = curr->read_next(key);
@@ -286,17 +423,49 @@ namespace snmalloc
    * If RANDOM is set to false, then the code does not perform any
    * randomisation.
    */
-  template<bool RANDOM, bool INIT = true>
+  template<
+    bool RANDOM,
+    SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
+    SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue,
+    bool INIT = true>
   class FreeListBuilder
   {
     static constexpr size_t LENGTH = RANDOM ? 2 : 1;
 
+    /*
+     * We use native pointers below so that we don't run afoul of strict
+     * aliasing rules.  head is a FreeObject::HeadPtr<BView, BQueue> -- that
+     * is, a known-domesticated pointer to a queue of wild pointers -- and
+     * it's usually the case that end is a FreeObject::QueuePtr<BQueue>* --
+     * that is, a known-domesticated pointer to a wild pointer to a queue of
+     * wild pointers.  However, in order to do branchless inserts, we set end
+     * = &head, which breaks strict aliasing rules with the types as given.
+     * Fortunately, these are private members and so we can use native
+     * pointers and just expose a more strongly typed interface.
+     */
+
     // Pointer to the first element.
-    std::array<capptr::AllocFull<FreeObject>, LENGTH> head;
+    std::array<void*, LENGTH> head{nullptr};
     // Pointer to the reference to the last element.
     // In the empty case end[i] == &head[i]
     // This enables branch free enqueuing.
-    std::array<capptr::AllocFull<FreeObject>*, LENGTH> end{nullptr};
+    std::array<void**, LENGTH> end{nullptr};
+
+    FreeObject::QueuePtr<BQueue>* cast_end(uint32_t ix)
+    {
+      return reinterpret_cast<FreeObject::QueuePtr<BQueue>*>(end[ix]);
+    }
+
+    void set_end(uint32_t ix, FreeObject::QueuePtr<BQueue>* p)
+    {
+      end[ix] = reinterpret_cast<void**>(p);
+    }
+
+    FreeObject::HeadPtr<BView, BQueue> cast_head(uint32_t ix)
+    {
+      return FreeObject::HeadPtr<BView, BQueue>(
+        static_cast<FreeObject::T<BQueue>*>(head[ix]));
+    }
 
     std::array<uint16_t, RANDOM ? 2 : 0> length{};
 
@@ -304,7 +473,9 @@ namespace snmalloc
     constexpr FreeListBuilder()
     {
       if (INIT)
+      {
         init();
+      }
     }
 
     /**
@@ -314,8 +485,10 @@ namespace snmalloc
     {
       for (size_t i = 0; i < LENGTH; i++)
       {
-        if (address_cast(end[i]) != address_cast(&head[i]))
+        if (end[i] != &head[i])
+        {
           return false;
+        }
       }
       return true;
     }
@@ -324,7 +497,7 @@ namespace snmalloc
      * Adds an element to the builder
      */
     void add(
-      capptr::AllocFull<FreeObject> n,
+      FreeObject::HeadPtr<BView, BQueue> n,
       const FreeListKey& key,
       LocalEntropy& entropy)
     {
@@ -334,7 +507,7 @@ namespace snmalloc
       else
         index = 0;
 
-      end[index] = FreeObject::store_next(end[index], n, key);
+      set_end(index, FreeObject::store_next(cast_end(index), n, key));
       if constexpr (RANDOM)
       {
         length[index]++;
@@ -351,10 +524,10 @@ namespace snmalloc
      */
     template<bool RANDOM_ = RANDOM>
     std::enable_if_t<!RANDOM_>
-    add(capptr::AllocFull<FreeObject> n, const FreeListKey& key)
+    add(FreeObject::HeadPtr<BView, BQueue> n, const FreeListKey& key)
     {
       static_assert(RANDOM_ == RANDOM, "Don't set template parameter");
-      end[0] = FreeObject::store_next(end[0], n, key);
+      set_end(0, FreeObject::store_next(cast_end(0), n, key));
     }
 
     /**
@@ -363,7 +536,7 @@ namespace snmalloc
     SNMALLOC_FAST_PATH void
     terminate_list(uint32_t index, const FreeListKey& key)
     {
-      FreeObject::store_null(end[index], key);
+      FreeObject::store_null(cast_end(index), key);
     }
 
     /**
@@ -375,11 +548,11 @@ namespace snmalloc
      * and is thus subject to encoding if the next_object pointers
      * encoded.
      */
-    capptr::AllocFull<FreeObject>
+    FreeObject::HeadPtr<BView, BQueue>
     read_head(uint32_t index, const FreeListKey& key)
     {
-      return FreeObject::encode_next(
-        address_cast(&head[index]), head[index], key);
+      return FreeObject::decode_next(
+        address_cast(&head[index]), cast_head(index), key);
     }
 
     address_t get_fake_signed_prev(uint32_t index, const FreeListKey& key)
@@ -396,7 +569,8 @@ namespace snmalloc
      *
      * The return value is how many entries are still contained in the builder.
      */
-    SNMALLOC_FAST_PATH uint16_t close(FreeListIter& fl, const FreeListKey& key)
+    SNMALLOC_FAST_PATH uint16_t
+    close(FreeListIter<BView, BQueue>& fl, const FreeListKey& key)
     {
       uint32_t i;
       if constexpr (RANDOM)
@@ -447,7 +621,9 @@ namespace snmalloc
     template<bool RANDOM_ = RANDOM>
     std::enable_if_t<
       !RANDOM_,
-      std::pair<capptr::AllocFull<FreeObject>, capptr::AllocFull<FreeObject>>>
+      std::pair<
+        FreeObject::HeadPtr<BView, BQueue>,
+        FreeObject::HeadPtr<BView, BQueue>>>
     extract_segment(const FreeListKey& key)
     {
       static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!");
@@ -458,8 +634,8 @@ namespace snmalloc
       // this is doing a CONTAINING_RECORD like cast to get back
       // to the actual object.  This isn't true if the builder is
       // empty, but you are not allowed to call this in the empty case.
-      auto last =
-        capptr::AllocFull<FreeObject>(reinterpret_cast<FreeObject*>(end[0]));
+      auto last = FreeObject::HeadPtr<BView, BQueue>(
+        reinterpret_cast<FreeObject::T<BQueue>*>(end[0]));
       init();
       return {first, last};
     }

src/mem/localalloc.h

@@ -211,25 +211,33 @@ namespace snmalloc
     SNMALLOC_FAST_PATH void* small_alloc(size_t size)
     {
       //      SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES));
-      auto slowpath = [&](
-                        sizeclass_t sizeclass,
-                        FreeListIter* fl) SNMALLOC_FAST_PATH_LAMBDA {
-        if (likely(core_alloc != nullptr))
-        {
-          return core_alloc->handle_message_queue(
-            [](CoreAlloc* core_alloc, sizeclass_t sizeclass, FreeListIter* fl) {
-              return core_alloc->template small_alloc<zero_mem>(sizeclass, *fl);
+      auto slowpath =
+        [&](
+          sizeclass_t sizeclass,
+          FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>*
+            fl) SNMALLOC_FAST_PATH_LAMBDA {
+          if (likely(core_alloc != nullptr))
+          {
+            return core_alloc->handle_message_queue(
+              [](
+                CoreAlloc* core_alloc,
+                sizeclass_t sizeclass,
+                FreeListIter<
+                  capptr::bounds::AllocFull,
+                  capptr::bounds::AllocFull>* fl) {
+                return core_alloc->template small_alloc<zero_mem>(
+                  sizeclass, *fl);
+              },
+              core_alloc,
+              sizeclass,
+              fl);
+          }
+          return lazy_init(
+            [&](CoreAlloc*, sizeclass_t sizeclass) {
+              return small_alloc<zero_mem>(sizeclass_to_size(sizeclass));
             },
-            core_alloc,
-            sizeclass,
-            fl);
-        }
-        return lazy_init(
-          [&](CoreAlloc*, sizeclass_t sizeclass) {
-            return small_alloc<zero_mem>(sizeclass_to_size(sizeclass));
-          },
-          sizeclass);
-      };
+            sizeclass);
+        };
 
       return local_cache.template alloc<zero_mem, SharedStateHandle>(
         size, slowpath);

src/mem/localcache.h

@@ -12,8 +12,9 @@ namespace snmalloc
 {
   using Stats = AllocStats<NUM_SIZECLASSES, NUM_LARGE_CLASSES>;
 
-  inline static SNMALLOC_FAST_PATH void*
-  finish_alloc_no_zero(capptr::AllocFull<FreeObject> p, sizeclass_t sizeclass)
+  inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero(
+    FreeObject::HeadPtr<capptr::bounds::AllocFull, capptr::bounds::AllocFull> p,
+    sizeclass_t sizeclass)
   {
     SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p)));
     UNUSED(sizeclass);
@@ -24,8 +25,9 @@ namespace snmalloc
   }
 
   template<ZeroMem zero_mem, typename SharedStateHandle>
-  inline static SNMALLOC_FAST_PATH void*
-  finish_alloc(capptr::AllocFull<FreeObject> p, sizeclass_t sizeclass)
+  inline static SNMALLOC_FAST_PATH void* finish_alloc(
+    FreeObject::HeadPtr<capptr::bounds::AllocFull, capptr::bounds::AllocFull> p,
+    sizeclass_t sizeclass)
   {
     auto r = finish_alloc_no_zero(p, sizeclass);
 
@@ -46,7 +48,8 @@ namespace snmalloc
     // Free list per small size class.  These are used for
     // allocation on the fast path. This part of the code is inspired by
     // mimalloc.
-    FreeListIter small_fast_free_lists[NUM_SIZECLASSES];
+    FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>
+      small_fast_free_lists[NUM_SIZECLASSES];
 
     // This is the entropy for a particular thread.
     LocalEntropy entropy;

src/mem/metaslab.h

@@ -25,9 +25,11 @@ namespace snmalloc
      *  Data-structure for building the free list for this slab.
      */
 #ifdef SNMALLOC_CHECK_CLIENT
-    FreeListBuilder<true> free_queue;
+    FreeListBuilder<true, capptr::bounds::AllocFull, capptr::bounds::AllocFull>
+      free_queue;
 #else
-    FreeListBuilder<false> free_queue;
+    FreeListBuilder<false, capptr::bounds::AllocFull, capptr::bounds::AllocFull>
+      free_queue;
 #endif
 
     /**
@@ -152,16 +154,18 @@ namespace snmalloc
      * component, but with randomisation, it may only return part of the
      * available objects for this metaslab.
      */
-    static SNMALLOC_FAST_PATH std::pair<capptr::AllocFull<FreeObject>, bool>
-    alloc_free_list(
-      Metaslab* meta,
-      FreeListIter& fast_free_list,
-      LocalEntropy& entropy,
-      sizeclass_t sizeclass)
+    static SNMALLOC_FAST_PATH
+      std::pair<FreeObject::QueuePtr<capptr::bounds::AllocFull>, bool>
+      alloc_free_list(
+        Metaslab* meta,
+        FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>&
+          fast_free_list,
+        LocalEntropy& entropy,
+        sizeclass_t sizeclass)
     {
       auto& key = entropy.get_free_list_key();
 
-      FreeListIter tmp_fl;
+      std::remove_reference_t<decltype(fast_free_list)> tmp_fl;
       auto remaining = meta->free_queue.close(tmp_fl, key);
       auto p = tmp_fl.take(key);
       fast_free_list = tmp_fl;

src/mem/remoteallocator.h

@@ -35,10 +35,12 @@ namespace snmalloc
 
     // Store the message queue on a separate cacheline. It is mutable data that
     // is read by other threads.
-    alignas(CACHELINE_SIZE) capptr::AtomicAllocFull<FreeObject> back{nullptr};
+    alignas(CACHELINE_SIZE)
+      FreeObject::AtomicQueuePtr<capptr::bounds::AllocFull> back{nullptr};
     // Store the two ends on different cache lines as access by different
     // threads.
-    alignas(CACHELINE_SIZE) capptr::AllocFull<FreeObject> front{nullptr};
+    alignas(CACHELINE_SIZE)
+      FreeObject::QueuePtr<capptr::bounds::AllocFull> front{nullptr};
 
     constexpr RemoteAllocator() = default;
 
@@ -48,17 +50,17 @@ namespace snmalloc
       SNMALLOC_ASSERT(front != nullptr);
     }
 
-    void init(capptr::AllocFull<FreeObject> stub)
+    void init(FreeObject::QueuePtr<capptr::bounds::AllocFull> stub)
     {
-      stub->atomic_store_null(key_global);
+      FreeObject::atomic_store_null(stub, key_global);
       front = stub;
       back.store(stub, std::memory_order_relaxed);
       invariant();
     }
 
-    capptr::AllocFull<FreeObject> destroy()
+    FreeObject::QueuePtr<capptr::bounds::AllocFull> destroy()
     {
-      capptr::AllocFull<FreeObject> fnt = front;
+      FreeObject::QueuePtr<capptr::bounds::AllocFull> fnt = front;
       back.store(nullptr, std::memory_order_relaxed);
       front = nullptr;
       return fnt;
@@ -66,7 +68,8 @@ namespace snmalloc
 
     inline bool is_empty()
     {
-      capptr::AllocFull<FreeObject> bk = back.load(std::memory_order_relaxed);
+      FreeObject::QueuePtr<capptr::bounds::AllocFull> bk =
+        back.load(std::memory_order_relaxed);
 
       return bk == front;
     }
@@ -76,21 +79,22 @@ namespace snmalloc
      * last should be linked together through their next pointers.
      */
     void enqueue(
-      capptr::AllocFull<FreeObject> first,
-      capptr::AllocFull<FreeObject> last,
+      FreeObject::QueuePtr<capptr::bounds::AllocFull> first,
+      FreeObject::QueuePtr<capptr::bounds::AllocFull> last,
       const FreeListKey& key)
     {
       invariant();
-      last->atomic_store_null(key);
+      FreeObject::atomic_store_null(last, key);
 
       // exchange needs to be a release, so nullptr in next is visible.
-      capptr::AllocFull<FreeObject> prev =
+      FreeObject::QueuePtr<capptr::bounds::AllocFull> prev =
         back.exchange(last, std::memory_order_release);
 
-      prev->atomic_store_next(first, key);
+      // XXX prev is not known to be domesticated
+      FreeObject::atomic_store_next(prev, first, key);
     }
 
-    capptr::AllocFull<FreeObject> peek()
+    FreeObject::QueuePtr<capptr::bounds::AllocFull> peek()
     {
       return front;
     }
@@ -98,12 +102,13 @@ namespace snmalloc
     /**
      * Returns the front message, or null if not possible to return a message.
      */
-    std::pair<capptr::AllocFull<FreeObject>, bool>
+    std::pair<FreeObject::QueuePtr<capptr::bounds::AllocFull>, bool>
     dequeue(const FreeListKey& key)
     {
       invariant();
-      capptr::AllocFull<FreeObject> first = front;
-      capptr::AllocFull<FreeObject> next = first->atomic_read_next(key);
+      FreeObject::QueuePtr<capptr::bounds::AllocFull> first = front;
+      FreeObject::QueuePtr<capptr::bounds::AllocFull> next =
+        first->atomic_read_next(key);
 
       if (next != nullptr)
       {

src/mem/remotecache.h

@@ -16,7 +16,14 @@ namespace snmalloc
    */
   struct RemoteDeallocCache
   {
-    std::array<FreeListBuilder<false, false>, REMOTE_SLOTS> list;
+    std::array<
+      FreeListBuilder<
+        false,
+        capptr::bounds::AllocFull,
+        capptr::bounds::AllocFull,
+        false>,
+      REMOTE_SLOTS>
+      list;
 
     /**
      * The total amount of memory we are waiting for before we will dispatch
@@ -70,7 +77,8 @@ namespace snmalloc
       const FreeListKey& key)
     {
       SNMALLOC_ASSERT(initialised);
-      auto r = p.template as_reinterpret<FreeObject>();
+      auto r =
+        p.template as_reinterpret<FreeObject::T<capptr::bounds::AllocFull>>();
 
       list[get_slot<allocator_size>(target_id, 0)].add(r, key);
     }
@@ -110,7 +118,8 @@ namespace snmalloc
         // Entries could map back onto the "resend" list,
         // so take copy of the head, mark the last element,
         // and clear the original list.
-        FreeListIter resend;
+        FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>
+          resend;
         list[my_slot].close(resend, key);
 
         post_round++;