CapPtr: shift free lists to Alloc bounds

This is incomplete, yet still more reflective of what's going on: we take the
exported pointers back from userspace and thread them directly into the free
lists.

So: move capptr_to_user_address_control to list construction time rather than
list consumption time.

src/mem/corealloc.h

@@ -162,8 +162,8 @@ namespace snmalloc
       // Manufacture an allocation to prime the queue
       // Using an actual allocation removes a conditional from a critical path.
       auto dummy =
-        capptr::AllocFull<void>(small_alloc_one(MIN_ALLOC_SIZE))
+        capptr::Alloc<void>(small_alloc_one(MIN_ALLOC_SIZE))
-          .template as_static<FreeObject::T<capptr::bounds::AllocFull>>();
+          .template as_static<FreeObject::T<capptr::bounds::Alloc>>();
       if (dummy == nullptr)
       {
         error("Critical error: Out-of-memory during initialisation.");
@@ -185,8 +185,9 @@ namespace snmalloc
         size,
         [&](
           sizeclass_t sizeclass,
-          FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>*
+          FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>* fl) {
-            fl) { return small_alloc<NoZero>(sizeclass, *fl); });
+          return small_alloc<NoZero>(sizeclass, *fl);
+        });
     }
 
     static SNMALLOC_FAST_PATH void alloc_new_list(
@@ -248,7 +249,8 @@ namespace snmalloc
       do
       {
         b.add(
-          FreeObject::make<capptr::bounds::AllocFull>(curr_ptr.as_void()),
+          FreeObject::make<capptr::bounds::Alloc>(
+            capptr_to_user_address_control(curr_ptr.as_void())),
           key,
           entropy);
         curr_ptr = curr_ptr->next;
@@ -258,9 +260,10 @@ namespace snmalloc
       do
       {
         b.add(
-          Aal::capptr_bound<
+          FreeObject::make<capptr::bounds::Alloc>(
-            FreeObject::T<capptr::bounds::AllocFull>,
+            capptr_to_user_address_control(
-            capptr::bounds::AllocFull>(p, rsize),
+              Aal::capptr_bound<void, capptr::bounds::AllocFull>(
+                p.as_void(), rsize))),
           key);
         p = pointer_offset(p, rsize);
       } while (p < slab_end);
@@ -272,7 +275,7 @@ namespace snmalloc
     ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass)
     {
       auto& key = entropy.get_free_list_key();
-      FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull> fl;
+      FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc> fl;
       auto more = meta->free_queue.close(fl, key);
       UNUSED(more);
       void* p = finish_alloc_no_zero(fl.take(key), sizeclass);
@@ -435,7 +438,7 @@ namespace snmalloc
      */
     void handle_dealloc_remote(
       const MetaEntry& entry,
-      CapPtr<void, capptr::bounds::AllocFull> p,
+      CapPtr<void, capptr::bounds::Alloc> p,
       bool& need_post)
     {
       // TODO this needs to not double count stats
@@ -444,7 +447,7 @@ namespace snmalloc
 
       if (likely(entry.get_remote() == public_state()))
       {
-        if (likely(dealloc_local_object_fast(entry, p.unsafe_ptr(), entropy)))
+        if (likely(dealloc_local_object_fast(entry, p.as_void(), entropy)))
           return;
 
         dealloc_local_object_slow(entry);
@@ -569,7 +572,8 @@ namespace snmalloc
       return handle_message_queue_inner(action, args...);
     }
 
-    SNMALLOC_FAST_PATH void dealloc_local_object(void* p)
+    SNMALLOC_FAST_PATH void
+    dealloc_local_object(CapPtr<void, capptr::bounds::Alloc> p)
     {
       auto entry = SharedStateHandle::Pagemap::get_metaentry(
         backend_state_ptr(), snmalloc::address_cast(p));
@@ -580,7 +584,9 @@ namespace snmalloc
     }
 
     SNMALLOC_FAST_PATH static bool dealloc_local_object_fast(
-      const MetaEntry& entry, void* p, LocalEntropy& entropy)
+      const MetaEntry& entry,
+      CapPtr<void, capptr::bounds::Alloc> p,
+      LocalEntropy& entropy)
     {
       auto meta = entry.get_metaslab();
 
@@ -590,8 +596,7 @@ namespace snmalloc
         Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)),
         "Not deallocating start of an object");
 
-      auto cp = FreeObject::QueuePtr<capptr::bounds::AllocFull>(
+      auto cp = p.as_static<FreeObject::T<capptr::bounds::Alloc>>();
-        reinterpret_cast<FreeObject::T<capptr::bounds::AllocFull>*>(p));
 
       auto& key = entropy.get_free_list_key();
 
@@ -604,7 +609,7 @@ namespace snmalloc
     template<ZeroMem zero_mem>
     SNMALLOC_SLOW_PATH void* small_alloc(
       sizeclass_t sizeclass,
-      FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>&
+      FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>&
         fast_free_list)
     {
       size_t rsize = sizeclass_to_size(sizeclass);
@@ -664,7 +669,7 @@ namespace snmalloc
     template<ZeroMem zero_mem>
     SNMALLOC_SLOW_PATH void* small_alloc_slow(
       sizeclass_t sizeclass,
-      FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>&
+      FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>&
         fast_free_list,
       size_t rsize)
     {
@@ -717,7 +722,7 @@ namespace snmalloc
     {
       SNMALLOC_ASSERT(attached_cache != nullptr);
       // TODO: Placeholder
-      auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::AllocFull> p)
+      auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
                            SNMALLOC_FAST_PATH_LAMBDA { return p; };
 
       if (destroy_queue)

src/mem/freelist.h

@@ -249,8 +249,8 @@ namespace snmalloc
 
     /**
      * Decode next.  While traversing a queue, BView and BQueue here will often
-     * be equal (i.e., CBAllocExportWild) rather than dichotomous.  However,
+     * be equal (i.e., AllocUserWild) rather than dichotomous.  However, we do
-     * we do occasionally decode an actual head pointer, so be polymorphic here.
+     * occasionally decode an actual head pointer, so be polymorphic here.
      *
      * TODO: We'd like, in some sense, to more tightly couple or integrate this
      * into to the domestication process.  We could introduce an additional

src/mem/localalloc.h

@@ -214,30 +214,29 @@ namespace snmalloc
       auto slowpath =
         [&](
           sizeclass_t sizeclass,
-          FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>*
+          FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>* fl)
-            fl) SNMALLOC_FAST_PATH_LAMBDA {
+          SNMALLOC_FAST_PATH_LAMBDA {
-          if (likely(core_alloc != nullptr))
+            if (likely(core_alloc != nullptr))
-          {
+            {
-            return core_alloc->handle_message_queue(
+              return core_alloc->handle_message_queue(
-              [](
+                [](
-                CoreAlloc* core_alloc,
+                  CoreAlloc* core_alloc,
-                sizeclass_t sizeclass,
+                  sizeclass_t sizeclass,
-                FreeListIter<
+                  FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>*
-                  capptr::bounds::AllocFull,
+                    fl) {
-                  capptr::bounds::AllocFull>* fl) {
+                  return core_alloc->template small_alloc<zero_mem>(
-                return core_alloc->template small_alloc<zero_mem>(
+                    sizeclass, *fl);
-                  sizeclass, *fl);
+                },
+                core_alloc,
+                sizeclass,
+                fl);
+            }
+            return lazy_init(
+              [&](CoreAlloc*, sizeclass_t sizeclass) {
+                return small_alloc<zero_mem>(sizeclass_to_size(sizeclass));
               },
-              core_alloc,
+              sizeclass);
-              sizeclass,
+          };
-              fl);
-          }
-          return lazy_init(
-            [&](CoreAlloc*, sizeclass_t sizeclass) {
-              return small_alloc<zero_mem>(sizeclass_to_size(sizeclass));
-            },
-            sizeclass);
-        };
 
       return local_cache.template alloc<zero_mem, SharedStateHandle>(
         size, slowpath);
@@ -260,20 +259,18 @@ namespace snmalloc
      * In the second case we need to recheck if this is a remote deallocation,
      * as we might acquire the originating allocator.
      */
-    SNMALLOC_SLOW_PATH void dealloc_remote_slow(void* p)
+    SNMALLOC_SLOW_PATH void dealloc_remote_slow(capptr::Alloc<void> p)
     {
       if (core_alloc != nullptr)
       {
 #ifdef SNMALLOC_TRACING
-        std::cout << "Remote dealloc post" << p << " size " << alloc_size(p)
+        std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size "
-                  << std::endl;
+                  << alloc_size(p.unsafe_ptr()) << std::endl;
 #endif
         MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry(
           core_alloc->backend_state_ptr(), address_cast(p));
         local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
-          entry.get_remote()->trunc_id(),
+          entry.get_remote()->trunc_id(), p, key_global);
-          capptr::AllocFull<void>(p),
-          key_global);
         post_remote_cache();
         return;
       }
@@ -281,8 +278,8 @@ namespace snmalloc
       // Recheck what kind of dealloc we should do incase, the allocator we get
       // from lazy_init is the originating allocator.
       lazy_init(
-        [&](CoreAlloc*, void* p) {
+        [&](CoreAlloc*, CapPtr<void, capptr::bounds::Alloc> p) {
-          dealloc(p); // TODO don't double count statistics
+          dealloc(p.unsafe_ptr()); // TODO don't double count statistics
           return nullptr;
         },
         p);
@@ -450,10 +447,10 @@ namespace snmalloc
       return alloc<zero_mem>(size);
     }
 
-    SNMALLOC_FAST_PATH void dealloc(void* p)
+    SNMALLOC_FAST_PATH void dealloc(void* p_raw)
     {
 #ifdef SNMALLOC_PASS_THROUGH
-      external_alloc::free(p);
+      external_alloc::free(p_raw);
 #else
       // TODO:
       // Care is needed so that dealloc(nullptr) works before init
@@ -461,6 +458,7 @@ namespace snmalloc
       //  before init, that maps null to a remote_deallocator that will never be
       //  in thread local state.
 
+      auto p = capptr_from_client(p_raw);
       const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(
         core_alloc->backend_state_ptr(), address_cast(p));
       if (likely(local_cache.remote_allocator == entry.get_remote()))
@@ -478,12 +476,10 @@ namespace snmalloc
         if (local_cache.remote_dealloc_cache.reserve_space(entry))
         {
           local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
-            entry.get_remote()->trunc_id(),
+            entry.get_remote()->trunc_id(), p, key_global);
-            capptr::AllocFull<void>(p),
-            key_global);
 #  ifdef SNMALLOC_TRACING
-          std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p)
+          std::cout << "Remote dealloc fast" << p_raw << " size "
-                    << std::endl;
+                    << alloc_size(p_raw) << std::endl;
 #  endif
           return;
         }
@@ -493,7 +489,7 @@ namespace snmalloc
       }
 
       // Large deallocation or null.
-      if (likely(p != nullptr))
+      if (likely(p.unsafe_ptr() != nullptr))
       {
         // Check this is managed by this pagemap.
         check_client(entry.get_sizeclass() != 0, "Not allocated by snmalloc.");
@@ -511,7 +507,14 @@ namespace snmalloc
 #  endif
         ChunkRecord* slab_record =
           reinterpret_cast<ChunkRecord*>(entry.get_metaslab());
-        slab_record->chunk = capptr::Chunk<void>(p);
+        /*
+         * StrictProvenance TODO: this is a subversive amplification.  p is
+         * AllocWild-bounded, but we're coercing it to Chunk-bounded.  We
+         * should, instead, not be storing ->chunk here, but should be keeping
+         * a CapPtr<void, Chunk> to this region internally even while it's
+         * allocated.
+         */
+        slab_record->chunk = capptr::Chunk<void>(p.unsafe_ptr());
         check_init(
           [](
             CoreAlloc* core_alloc,

src/mem/localcache.h

@@ -13,20 +13,20 @@ namespace snmalloc
   using Stats = AllocStats<NUM_SIZECLASSES, NUM_LARGE_CLASSES>;
 
   inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero(
-    FreeObject::HeadPtr<capptr::bounds::AllocFull, capptr::bounds::AllocFull> p,
+    FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc> p,
     sizeclass_t sizeclass)
   {
     SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p)));
     UNUSED(sizeclass);
 
-    auto r = capptr_reveal(capptr_to_user_address_control(p.as_void()));
+    auto r = capptr_reveal(p.as_void());
 
     return r;
   }
 
   template<ZeroMem zero_mem, typename SharedStateHandle>
   inline static SNMALLOC_FAST_PATH void* finish_alloc(
-    FreeObject::HeadPtr<capptr::bounds::AllocFull, capptr::bounds::AllocFull> p,
+    FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc> p,
     sizeclass_t sizeclass)
   {
     auto r = finish_alloc_no_zero(p, sizeclass);
@@ -48,7 +48,7 @@ namespace snmalloc
     // Free list per small size class.  These are used for
     // allocation on the fast path. This part of the code is inspired by
     // mimalloc.
-    FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>
+    FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>
       small_fast_free_lists[NUM_SIZECLASSES];
 
     // This is the entropy for a particular thread.
@@ -90,7 +90,8 @@ namespace snmalloc
         while (!small_fast_free_lists[i].empty())
         {
           auto p = small_fast_free_lists[i].take(key);
-          dealloc(finish_alloc_no_zero(p, i));
+          SNMALLOC_ASSERT(Metaslab::is_start_of_object(i, address_cast(p)));
+          dealloc(p.as_void());
         }
       }
 

src/mem/metaslab.h

@@ -25,10 +25,10 @@ namespace snmalloc
      *  Data-structure for building the free list for this slab.
      */
 #ifdef SNMALLOC_CHECK_CLIENT
-    FreeListBuilder<true, capptr::bounds::AllocFull, capptr::bounds::AllocFull>
+    FreeListBuilder<true, capptr::bounds::Alloc, capptr::bounds::Alloc>
       free_queue;
 #else
-    FreeListBuilder<false, capptr::bounds::AllocFull, capptr::bounds::AllocFull>
+    FreeListBuilder<false, capptr::bounds::Alloc, capptr::bounds::Alloc>
       free_queue;
 #endif
 
@@ -155,10 +155,10 @@ namespace snmalloc
      * available objects for this metaslab.
      */
     static SNMALLOC_FAST_PATH
-      std::pair<FreeObject::QueuePtr<capptr::bounds::AllocFull>, bool>
+      std::pair<FreeObject::QueuePtr<capptr::bounds::Alloc>, bool>
       alloc_free_list(
         Metaslab* meta,
-        FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>&
+        FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>&
           fast_free_list,
         LocalEntropy& entropy,
         sizeclass_t sizeclass)

src/mem/remoteallocator.h

@@ -36,11 +36,11 @@ namespace snmalloc
     // Store the message queue on a separate cacheline. It is mutable data that
     // is read by other threads.
     alignas(CACHELINE_SIZE)
-      FreeObject::AtomicQueuePtr<capptr::bounds::AllocFull> back{nullptr};
+      FreeObject::AtomicQueuePtr<capptr::bounds::Alloc> back{nullptr};
     // Store the two ends on different cache lines as access by different
     // threads.
-    alignas(CACHELINE_SIZE)
+    alignas(CACHELINE_SIZE) FreeObject::QueuePtr<capptr::bounds::Alloc> front{
-      FreeObject::QueuePtr<capptr::bounds::AllocFull> front{nullptr};
+      nullptr};
 
     constexpr RemoteAllocator() = default;
 
@@ -50,7 +50,7 @@ namespace snmalloc
       SNMALLOC_ASSERT(front != nullptr);
     }
 
-    void init(FreeObject::QueuePtr<capptr::bounds::AllocFull> stub)
+    void init(FreeObject::QueuePtr<capptr::bounds::Alloc> stub)
     {
       FreeObject::atomic_store_null(stub, key_global);
       front = stub;
@@ -58,9 +58,9 @@ namespace snmalloc
       invariant();
     }
 
-    FreeObject::QueuePtr<capptr::bounds::AllocFull> destroy()
+    FreeObject::QueuePtr<capptr::bounds::Alloc> destroy()
     {
-      FreeObject::QueuePtr<capptr::bounds::AllocFull> fnt = front;
+      FreeObject::QueuePtr<capptr::bounds::Alloc> fnt = front;
       back.store(nullptr, std::memory_order_relaxed);
       front = nullptr;
       return fnt;
@@ -68,7 +68,7 @@ namespace snmalloc
 
     inline bool is_empty()
     {
-      FreeObject::QueuePtr<capptr::bounds::AllocFull> bk =
+      FreeObject::QueuePtr<capptr::bounds::Alloc> bk =
         back.load(std::memory_order_relaxed);
 
       return bk == front;
@@ -79,22 +79,22 @@ namespace snmalloc
      * last should be linked together through their next pointers.
      */
     void enqueue(
-      FreeObject::QueuePtr<capptr::bounds::AllocFull> first,
+      FreeObject::QueuePtr<capptr::bounds::Alloc> first,
-      FreeObject::QueuePtr<capptr::bounds::AllocFull> last,
+      FreeObject::QueuePtr<capptr::bounds::Alloc> last,
       const FreeListKey& key)
     {
       invariant();
       FreeObject::atomic_store_null(last, key);
 
       // exchange needs to be a release, so nullptr in next is visible.
-      FreeObject::QueuePtr<capptr::bounds::AllocFull> prev =
+      FreeObject::QueuePtr<capptr::bounds::Alloc> prev =
         back.exchange(last, std::memory_order_release);
 
       // XXX prev is not known to be domesticated
       FreeObject::atomic_store_next(prev, first, key);
     }
 
-    FreeObject::QueuePtr<capptr::bounds::AllocFull> peek()
+    FreeObject::QueuePtr<capptr::bounds::Alloc> peek()
     {
       return front;
     }
@@ -103,16 +103,16 @@ namespace snmalloc
      * Returns the front message, or null if not possible to return a message.
      */
     std::pair<
-      FreeObject::HeadPtr<capptr::bounds::AllocFull, capptr::bounds::AllocFull>,
+      FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc>,
       bool>
     dequeue(const FreeListKey& key)
     {
-      auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::AllocFull> p)
+      auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
                            SNMALLOC_FAST_PATH_LAMBDA { return p; };
       invariant();
-      FreeObject::HeadPtr<capptr::bounds::AllocFull, capptr::bounds::AllocFull>
+      FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc> first =
-        first = domesticate(front);
+        domesticate(front);
-      FreeObject::QueuePtr<capptr::bounds::AllocFull> next =
+      FreeObject::QueuePtr<capptr::bounds::Alloc> next =
         first->atomic_read_next(key, domesticate);
 
       if (next != nullptr)

src/mem/remotecache.h

@@ -19,8 +19,8 @@ namespace snmalloc
     std::array<
       FreeListBuilder<
         false,
-        capptr::bounds::AllocFull,
+        capptr::bounds::Alloc,
-        capptr::bounds::AllocFull,
+        capptr::bounds::Alloc,
         false>,
       REMOTE_SLOTS>
       list;
@@ -73,12 +73,12 @@ namespace snmalloc
     template<size_t allocator_size>
     SNMALLOC_FAST_PATH void dealloc(
       RemoteAllocator::alloc_id_t target_id,
-      capptr::AllocFull<void> p,
+      capptr::Alloc<void> p,
       const FreeListKey& key)
     {
       SNMALLOC_ASSERT(initialised);
       auto r =
-        p.template as_reinterpret<FreeObject::T<capptr::bounds::AllocFull>>();
+        p.template as_reinterpret<FreeObject::T<capptr::bounds::Alloc>>();
 
       list[get_slot<allocator_size>(target_id, 0)].add(r, key);
     }
@@ -118,8 +118,7 @@ namespace snmalloc
         // Entries could map back onto the "resend" list,
         // so take copy of the head, mark the last element,
         // and clear the original list.
-        FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>
+        FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc> resend;
-          resend;
         list[my_slot].close(resend, key);
 
         post_round++;