From cf50fc5e5565ce5666628908987c91d2b59b0375 Mon Sep 17 00:00:00 2001
From: Nathaniel Filardo <nfilardo@microsoft.com>
Date: Sat, 13 Mar 2021 15:20:12 +0000
Subject: [PATCH] SP: mediumslab: reduce capptr_export calls

Mediumslabs are strung on a dllist and used to feed the allocator there.  If we
ensure that these (and the root pointer to the list itself) are already
exported, then our alloc paths can bound these to arrive at exposible pointers.

The dealloc paths, where we might want a non-exported pointer, already have one,
as they have gone through amplification to get an arena-bounded pointer.

The sole wrinkle in this plan is that we might need a pointer without platform
constraints to manipulate the memory map for page-based zeroing.  Since we have
ample room in the Mediumslab header (a few kilobytes end up being used for
padding; the curious should see b8b5f305 and 3d3b0487), just cache therein a
copy of the CBChunk-bound pointer used in Mediumslab::init() for ::alloc().
---
 src/mem/alloc.h      | 20 ++++++++++++++------
 src/mem/mediumslab.h | 18 ++++++++++++------
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/src/mem/alloc.h b/src/mem/alloc.h
index 5eaaab7..fcdf501 100644
--- a/src/mem/alloc.h
+++ b/src/mem/alloc.h
@@ -638,7 +638,7 @@ namespace snmalloc
     };
 
     SlabList small_classes[NUM_SMALL_CLASSES];
-    DLList<Mediumslab, CapPtrCBChunk> medium_classes[NUM_MEDIUM_CLASSES];
+    DLList<Mediumslab, CapPtrCBChunkE> medium_classes[NUM_MEDIUM_CLASSES];
 
     DLList<Superslab, CapPtrCBChunk> super_available;
     DLList<Superslab, CapPtrCBChunk> super_only_short_available;
@@ -1383,7 +1383,7 @@ namespace snmalloc
       sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
 
       auto sc = &medium_classes[medium_class];
-      CapPtr<Mediumslab, CBChunk> slab = sc->get_head();
+      CapPtr<Mediumslab, CBChunkE> slab = sc->get_head();
       CapPtr<void, CBAllocE> p;
 
       if (slab != nullptr)
@@ -1424,11 +1424,14 @@ namespace snmalloc
 
         Mediumslab::init(newslab, public_state(), sizeclass, rsize);
         chunkmap().set_slab(newslab);
+
+        auto newslab_export = capptr_export(newslab);
+
         p = Mediumslab::alloc<zero_mem, typename MemoryProvider::Pal>(
-          newslab, rsize);
+          newslab_export, rsize);
 
         if (!Mediumslab::full(newslab))
-          sc->insert(newslab);
+          sc->insert(newslab_export);
       }
 
       stats().alloc_request(size);
@@ -1515,7 +1518,12 @@ namespace snmalloc
         {
           sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
           auto sc = &medium_classes[medium_class];
-          sc->remove(slab_bounded);
+          /*
+           * This unsafety lets us avoid applying platform constraints to a
+           * pointer we are just about to drop on the floor; remove() uses its
+           * argument but does not persist it.
+           */
+          sc->remove(CapPtr<Mediumslab, CBChunkE>(slab_bounded.unsafe_capptr));
         }
 
         chunkmap().clear_slab(slab_bounded);
@@ -1527,7 +1535,7 @@ namespace snmalloc
       {
         sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
         auto sc = &medium_classes[medium_class];
-        sc->insert(slab_bounded);
+        sc->insert(capptr_export(slab_bounded));
       }
     }
 
diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h
index ec3e83c..1877961 100644
--- a/src/mem/mediumslab.h
+++ b/src/mem/mediumslab.h
@@ -12,12 +12,16 @@ namespace snmalloc
     // This is the view of a 16 mb area when it is being used to allocate
     // medium sized classes: 64 kb to 16 mb, non-inclusive.
   private:
-    friend DLList<Mediumslab, CapPtrCBChunk>;
+    friend DLList<Mediumslab, CapPtrCBChunkE>;
 
     // Keep the allocator pointer on a separate cache line. It is read by
     // other threads, and does not change, so we avoid false sharing.
-    alignas(CACHELINE_SIZE) CapPtr<Mediumslab, CBChunk> next;
-    CapPtr<Mediumslab, CBChunk> prev;
+    alignas(CACHELINE_SIZE) CapPtr<Mediumslab, CBChunkE> next;
+    CapPtr<Mediumslab, CBChunkE> prev;
+
+    // Store a pointer to ourselves without platform constraints applied,
+    // as we need this to be able to zero memory by manipulating the VM map
+    CapPtr<void, CBChunk> self_chunk;
 
     uint16_t free;
     uint8_t head;
@@ -78,6 +82,7 @@ namespace snmalloc
       // initialise the allocation stack.
       if ((self->kind != Medium) || (self->sizeclass != sc))
       {
+        self->self_chunk = self.as_void();
         self->sizeclass = static_cast<uint8_t>(sc);
         uint16_t ssize = static_cast<uint16_t>(rsize >> 8);
         self->kind = Medium;
@@ -89,6 +94,7 @@ namespace snmalloc
       else
       {
         SNMALLOC_ASSERT(self->free == medium_slab_free(sc));
+        SNMALLOC_ASSERT(self->self_chunk == self.as_void());
       }
     }
 
@@ -99,7 +105,7 @@ namespace snmalloc
 
     template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
     static CapPtr<void, CBAllocE>
-    alloc(CapPtr<Mediumslab, CBChunk> self, size_t size)
+    alloc(CapPtr<Mediumslab, CBChunkE> self, size_t size)
     {
       SNMALLOC_ASSERT(!full(self));
 
@@ -108,11 +114,11 @@ namespace snmalloc
       self->free--;
 
       if constexpr (zero_mem == YesZero)
-        pal_zero<PAL>(p, size);
+        pal_zero<PAL>(Aal::capptr_rebound(self->self_chunk, p), size);
       else
         UNUSED(size);
 
-      return capptr_export(Aal::capptr_bound<void, CBAlloc>(p, size));
+      return Aal::capptr_bound<void, CBAllocE>(p, size);
     }
 
     static bool