Merge pull request #107 from CTSRD-CHERI/upstream-aal-features

Introduce AAL feature flags

src/aal/aal.h

@@ -11,6 +11,19 @@
 
 namespace snmalloc
 {
+  /**
+   * Flags in a bitfield of attributes of this architecture, much like
+   * PalFeatures.
+   */
+  enum AalFeatures : uint64_t
+  {
+    /**
+     * This architecture does not discriminate between integers and pointers,
+     * and so may use bit operations on pointer values.
+     */
+    IntegerPointers = (1 << 0),
+  };
+
   /**
    * Architecture Abstraction Layer. Includes default implementations of some
    * functions using compiler builtins.  Falls back to the definitions in the
@@ -63,7 +76,10 @@ namespace snmalloc
 
 namespace snmalloc
 {
-  using AAL = AAL_Generic<AAL_Arch>;
+  using Aal = AAL_Generic<AAL_Arch>;
+
+  template<AalFeatures F, typename AAL = Aal>
+  constexpr static bool aal_supports = (AAL::aal_features & F) == F;
 } // namespace snmalloc
 
 #if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_32)

src/aal/aal_x86.h

@@ -55,6 +55,11 @@ namespace snmalloc
     }
 
   public:
+    /**
+     * Bitmap of AalFeature flags
+     */
+    static constexpr uint64_t aal_features = IntegerPointers;
+
     /**
      * On pipelined processors, notify the core that we are in a spin loop and
      * that speculative execution past this point may not be a performance gain.

src/ds/flaglock.h

@@ -13,7 +13,7 @@ namespace snmalloc
     FlagLock(std::atomic_flag& lock) : lock(lock)
     {
       while (lock.test_and_set(std::memory_order_acquire))
-        AAL::pause();
+        Aal::pause();
     }
 
     ~FlagLock()

src/ds/mpscq.h

@@ -70,7 +70,7 @@ namespace snmalloc
       if (next != nullptr)
       {
         front = next;
-        AAL::prefetch(&(next->next));
+        Aal::prefetch(&(next->next));
         assert(front);
         std::atomic_thread_fence(std::memory_order_acquire);
         invariant();

src/mem/alloc.h

@@ -1120,7 +1120,7 @@ namespace snmalloc
           else if constexpr (decommit_strategy == DecommitSuperLazy)
           {
             static_assert(
-              pal_supports<LowMemoryNotification, MemoryProvider>(),
+              pal_supports<LowMemoryNotification, MemoryProvider>,
               "A lazy decommit strategy cannot be implemented on platforms "
               "without low memory notifications");
           }

src/mem/allocstats.h

@@ -66,7 +66,7 @@ namespace snmalloc
     {
       CurrentMaxPair count;
       CurrentMaxPair slab_count;
-      uint64_t time = AAL::tick();
+      uint64_t time = Aal::tick();
       uint64_t ticks = 0;
       double online_average = 0;
 
@@ -83,7 +83,7 @@ namespace snmalloc
 
       void addToRunningAverage()
       {
-        uint64_t now = AAL::tick();
+        uint64_t now = Aal::tick();
 
         if (slab_count.current != 0)
         {

src/mem/chunkmap.h

@@ -47,7 +47,7 @@ namespace snmalloc
 // Use flat map is under a single node.
 #  define SNMALLOC_MAX_FLATPAGEMAP_SIZE PAGEMAP_NODE_SIZE
 #endif
-  static constexpr bool USE_FLATPAGEMAP = pal_supports<LazyCommit>() ||
+  static constexpr bool USE_FLATPAGEMAP = pal_supports<LazyCommit> ||
     (SNMALLOC_MAX_FLATPAGEMAP_SIZE >=
      sizeof(FlatPagemap<SUPERSLAB_BITS, uint8_t>));
 

src/mem/largealloc.h

@@ -190,7 +190,7 @@ namespace snmalloc
     SNMALLOC_FAST_PATH
     uint64_t low_memory_epoch()
     {
-      if constexpr (pal_supports<LowMemoryNotification, PAL>())
+      if constexpr (pal_supports<LowMemoryNotification, PAL>)
       {
         return PAL::low_memory_epoch();
       }
@@ -203,7 +203,7 @@ namespace snmalloc
     template<bool committed>
     void* reserve(size_t* size, size_t align) noexcept
     {
-      if constexpr (pal_supports<AlignedAllocation, PAL>())
+      if constexpr (pal_supports<AlignedAllocation, PAL>)
       {
         return PAL::template reserve<committed>(size, align);
       }

src/mem/metaslab.h

@@ -109,11 +109,12 @@ namespace snmalloc
     /// simple corruptions.
     static SNMALLOC_FAST_PATH void store_next(void* p, void* head)
     {
-#ifndef CHECK_CLIENT
       *static_cast<void**>(p) = head;
-#else
-      *static_cast<void**>(p) = head;
-      *(static_cast<uintptr_t*>(p) + 1) = address_cast(head) ^ POISON;
+#if defined(CHECK_CLIENT)
+      if constexpr (aal_supports<IntegerPointers>)
+      {
+        *(static_cast<uintptr_t*>(p) + 1) = address_cast(head) ^ POISON;
+      }
 #endif
     }
 
@@ -121,11 +122,14 @@ namespace snmalloc
     /// In Debug checks for simple corruptions.
     static SNMALLOC_FAST_PATH void* follow_next(void* node)
     {
-#ifdef CHECK_CLIENT
-      uintptr_t next = *static_cast<uintptr_t*>(node);
-      uintptr_t chk = *(static_cast<uintptr_t*>(node) + 1);
-      if ((next ^ chk) != POISON)
-        error("Detected memory corruption.  Use-after-free.");
+#if defined(CHECK_CLIENT)
+      if constexpr (aal_supports<IntegerPointers>)
+      {
+        uintptr_t next = *static_cast<uintptr_t*>(node);
+        uintptr_t chk = *(static_cast<uintptr_t*>(node) + 1);
+        if ((next ^ chk) != POISON)
+          error("Detected memory corruption.  Use-after-free.");
+      }
 #endif
       return *static_cast<void**>(node);
     }

src/mem/pagemap.h

@@ -162,7 +162,7 @@ namespace snmalloc
           while (address_cast(e->load(std::memory_order_relaxed)) ==
                  LOCKED_ENTRY)
           {
-            AAL::pause();
+            Aal::pause();
           }
           value = e->load(std::memory_order_acquire);
         }

src/pal/pal.h

@@ -61,8 +61,5 @@ namespace snmalloc
    * Query whether the PAL supports a specific feature.
    */
   template<PalFeatures F, typename PAL = Pal>
-  constexpr static bool pal_supports()
-  {
-    return (PAL::pal_features & F) == F;
-  }
+  constexpr static bool pal_supports = (PAL::pal_features & F) == F;
 } // namespace snmalloc

src/test/perf/contention/contention.cc

@@ -31,18 +31,18 @@ private:
     auto prev = ready.fetch_add(1);
     if (prev + 1 == cores)
     {
-      start = AAL::tick();
+      start = Aal::tick();
       flag = true;
     }
     while (!flag)
-      AAL::pause();
+      Aal::pause();
 
     f(id);
 
     prev = complete.fetch_add(1);
     if (prev + 1 == cores)
     {
-      end = AAL::tick();
+      end = Aal::tick();
     }
   }