Enable a seconary allocator support (e.g. GWP-Asan) (#737)

src/snmalloc/backend/globalconfig.h

@@ -3,6 +3,7 @@
 #include "../backend_helpers/backend_helpers.h"
 #include "backend.h"
 #include "meta_protected_range.h"
+#include "snmalloc/mem/secondary.h"
 #include "standard_range.h"
 
 namespace snmalloc
@@ -107,6 +108,8 @@ namespace snmalloc
         if (initialised)
           return;
 
+        SecondaryAllocator::initialize();
+
         LocalEntropy entropy;
         entropy.init<Pal>();
         // Initialise key for remote deallocation lists

src/snmalloc/ds_core/ptrwrap.h

@@ -4,6 +4,8 @@
 #include "defines.h"
 #include "snmalloc/stl/atomic.h"
 
+#include <stdint.h>
+
 namespace snmalloc
 {
   /*

src/snmalloc/mem/corealloc.h

@@ -5,6 +5,7 @@
 #include "metadata.h"
 #include "pool.h"
 #include "remotecache.h"
+#include "secondary.h"
 #include "sizeclasstable.h"
 #include "snmalloc/stl/new.h"
 #include "ticker.h"
@@ -817,6 +818,14 @@ namespace snmalloc
     SNMALLOC_SLOW_PATH capptr::Alloc<void>
     small_alloc(smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list)
     {
+      void* result = SecondaryAllocator::allocate(
+        [sizeclass]() -> stl::Pair<size_t, size_t> {
+          auto size = sizeclass_to_size(sizeclass);
+          return {size, natural_alignment(size)};
+        });
+
+      if (result != nullptr)
+        return capptr::Alloc<void>::unsafe_from(result);
       // Look to see if we can grab a free list.
       auto& sl = alloc_classes[sizeclass].available;
       if (SNMALLOC_LIKELY(alloc_classes[sizeclass].length > 0))

src/snmalloc/mem/localalloc.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include "snmalloc/aal/address.h"
+#include "snmalloc/mem/remoteallocator.h"
+#include "snmalloc/mem/secondary.h"
 #if defined(_MSC_VER)
 #  define ALLOCATOR __declspec(allocator) __declspec(restrict)
 #elif __has_attribute(malloc)
@@ -194,6 +196,15 @@ namespace snmalloc
           errno = ENOMEM;
           return capptr::Alloc<void>{nullptr};
         }
+
+        // Check if secondary allocator wants to offer the memory
+        void* result =
+          SecondaryAllocator::allocate([size]() -> stl::Pair<size_t, size_t> {
+            return {size, natural_alignment(size)};
+          });
+        if (result != nullptr)
+          return capptr::Alloc<void>::unsafe_from(result);
+
         // Grab slab of correct size
         // Set remote as large allocator remote.
         auto [chunk, meta] = Config::Backend::alloc_chunk(
@@ -606,17 +617,13 @@ namespace snmalloc
 #endif
     }
 
-    SNMALLOC_FAST_PATH void dealloc(void* p_raw)
-    {
-#ifdef SNMALLOC_PASS_THROUGH
-      external_alloc::free(p_raw);
-#else
-      // Care is needed so that dealloc(nullptr) works before init
-      //  The backend allocator must ensure that a minimal page map exists
-      //  before init, that maps null to a remote_deallocator that will never
-      //  be in thread local state.
+    // The domestic pointer with its origin allocator
+    using DomesticInfo = stl::Pair<capptr::Alloc<void>, const PagemapEntry&>;
 
-#  ifdef __CHERI_PURE_CAPABILITY__
+    // Check whether the raw pointer is owned by snmalloc
+    SNMALLOC_FAST_PATH_INLINE DomesticInfo get_domestic_info(const void* p_raw)
+    {
+#ifdef __CHERI_PURE_CAPABILITY__
       /*
        * On CHERI platforms, snap the provided pointer to its base, ignoring
        * any client-provided offset, which may have taken the pointer out of
@@ -632,10 +639,29 @@ namespace snmalloc
        * start of the allocation and so the offset is zero.
        */
       p_raw = __builtin_cheri_offset_set(p_raw, 0);
-#  endif
+#endif
+      capptr::AllocWild<void> p_wild =
+        capptr_from_client(const_cast<void*>(p_raw));
+      auto p_tame =
+        capptr_domesticate<Config>(core_alloc->backend_state_ptr(), p_wild);
+      const PagemapEntry& entry =
+        Config::Backend::get_metaentry(address_cast(p_tame));
+      return {p_tame, entry};
+    }
 
-      capptr::AllocWild<void> p_wild = capptr_from_client(p_raw);
+    // Check if a pointer is domestic to SnMalloc
+    SNMALLOC_FAST_PATH bool is_snmalloc_owned(const void* p_raw)
+    {
+      auto [_, entry] = get_domestic_info(p_raw);
+      RemoteAllocator* remote = entry.get_remote();
+      return remote != nullptr;
+    }
 
+    SNMALLOC_FAST_PATH void dealloc(void* p_raw)
+    {
+#ifdef SNMALLOC_PASS_THROUGH
+      external_alloc::free(p_raw);
+#else
       /*
        * p_tame may be nullptr, even if p_raw/p_wild are not, in the case
        * where domestication fails.  We exclusively use p_tame below so that
@@ -648,11 +674,7 @@ namespace snmalloc
        * well-formedness) of this pointer.  The remainder of the logic will
        * deal with the object's extent.
        */
-      capptr::Alloc<void> p_tame =
-        capptr_domesticate<Config>(core_alloc->backend_state_ptr(), p_wild);
-
-      const PagemapEntry& entry =
-        Config::Backend::get_metaentry(address_cast(p_tame));
+      auto [p_tame, entry] = get_domestic_info(p_raw);
 
       if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote()))
       {
@@ -697,18 +719,15 @@ namespace snmalloc
         return;
       }
 
-      // If p_tame is not null, then dealloc has been call on something
-      // it shouldn't be called on.
-      // TODO: Should this be tested even in the !CHECK_CLIENT case?
-      snmalloc_check_client(
-        mitigations(sanity_checks),
-        p_tame == nullptr,
-        "Not allocated by snmalloc.");
-
+      if (SNMALLOC_LIKELY(p_tame == nullptr))
+      {
 #  ifdef SNMALLOC_TRACING
-      message<1024>("nullptr deallocation");
+        message<1024>("nullptr deallocation");
 #  endif
-      return;
+        return;
+      }
+
+      SecondaryAllocator::deallocate(p_tame.unsafe_ptr());
 #endif
     }
 
@@ -719,6 +738,8 @@ namespace snmalloc
 #else
       if constexpr (mitigations(sanity_checks))
       {
+        if (!is_snmalloc_owned(p))
+          return;
         size = size == 0 ? 1 : size;
         auto sc = size_to_sizeclass_full(size);
         auto pm_sc =
@@ -767,6 +788,11 @@ namespace snmalloc
 #ifdef SNMALLOC_PASS_THROUGH
       return external_alloc::malloc_usable_size(const_cast<void*>(p_raw));
 #else
+
+      if (
+        !SecondaryAllocator::pass_through && !is_snmalloc_owned(p_raw) &&
+        p_raw != nullptr)
+        return SecondaryAllocator::alloc_size(p_raw);
       // TODO What's the domestication policy here?  At the moment we just
       // probe the pagemap with the raw address, without checks.  There could
       // be implicit domestication through the `Config::Pagemap` or

src/snmalloc/mem/secondary.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "snmalloc/ds_core/defines.h"
+#include "snmalloc/ds_core/ptrwrap.h"
+
+#ifdef SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION
+#  include "snmalloc/mem/secondary/gwp_asan.h"
+
+namespace snmalloc
+{
+  using SecondaryAllocator = GwpAsanSecondaryAllocator;
+} // namespace snmalloc
+#else
+#  include "snmalloc/mem/secondary/default.h"
+
+namespace snmalloc
+{
+  using SecondaryAllocator = DefaultSecondaryAllocator;
+} // namespace snmalloc
+#endif

src/snmalloc/mem/secondary/default.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "snmalloc/ds_core/defines.h"
+#include "snmalloc/ds_core/mitigations.h"
+
+#include <stddef.h>
+
+namespace snmalloc
+{
+  class DefaultSecondaryAllocator
+  {
+  public:
+    // This flag is used to turn off checks on fast paths if the secondary
+    // allocator does not own the memory at all.
+    static constexpr inline bool pass_through = true;
+
+    SNMALLOC_FAST_PATH
+    static void initialize() {}
+
+    template<class SizeAlign>
+    SNMALLOC_FAST_PATH static void* allocate(SizeAlign&&)
+    {
+      return nullptr;
+    }
+
+    SNMALLOC_FAST_PATH
+    static void deallocate(void* pointer)
+    {
+      // If pointer is not null, then dealloc has been call on something
+      // it shouldn't be called on.
+      // TODO: Should this be tested even in the !CHECK_CLIENT case?
+      snmalloc_check_client(
+        mitigations(sanity_checks),
+        pointer == nullptr,
+        "Not allocated by snmalloc.");
+    }
+
+    SNMALLOC_FAST_PATH
+    static size_t alloc_size(const void*)
+    {
+      SNMALLOC_ASSERT(
+        false &&
+        "secondary alloc_size should never be invoked with default setup");
+      return 0;
+    }
+  };
+} // namespace snmalloc

src/snmalloc/mem/secondary/gwp_asan.h

@@ -0,0 +1,71 @@
+#pragma once
+
+#include "gwp_asan/guarded_pool_allocator.h"
+#include "snmalloc/ds_core/defines.h"
+#include "snmalloc/mem/sizeclasstable.h"
+#if defined(SNMALLOC_BACKTRACE_HEADER)
+#  include SNMALLOC_BACKTRACE_HEADER
+#endif
+
+namespace snmalloc
+{
+  class GwpAsanSecondaryAllocator
+  {
+    static inline gwp_asan::GuardedPoolAllocator singleton;
+    static inline size_t max_allocation_size;
+
+  public:
+    static constexpr inline bool pass_through = false;
+
+    static void initialize() noexcept
+    {
+      // for now, we use default options
+      gwp_asan::options::Options opt;
+      opt.setDefaults();
+#ifdef SNMALLOC_BACKTRACE_HEADER
+      opt.Backtrace = [](uintptr_t* buf, size_t length) {
+        return static_cast<size_t>(
+          ::backtrace(reinterpret_cast<void**>(buf), static_cast<int>(length)));
+      };
+#endif
+      singleton.init(opt);
+      max_allocation_size =
+        singleton.getAllocatorState()->maximumAllocationSize();
+    }
+
+    // Use thunk to avoid extra computation when allocation decision can be made
+    // before size and alignment are computed.
+    template<class SizeAlign>
+    SNMALLOC_FAST_PATH static void* allocate(SizeAlign&& getter)
+    {
+      // TODO: this `shouldSample` is only triggered on snmalloc's slowpath,
+      // which may reduce the chance of error detection. We may reconsider
+      // the logic to improve the precision in future commits.
+      if (SNMALLOC_UNLIKELY(singleton.shouldSample()))
+      {
+        auto [size, align] = getter();
+        if (size > max_allocation_size)
+          return nullptr;
+        return singleton.allocate(size, align);
+      }
+      return nullptr;
+    }
+
+    SNMALLOC_FAST_PATH
+    static void deallocate(void* pointer)
+    {
+      snmalloc_check_client(
+        mitigations(sanity_checks),
+        singleton.pointerIsMine(pointer),
+        "Not allocated by snmalloc or secondary allocator");
+
+      singleton.deallocate(pointer);
+    }
+
+    SNMALLOC_FAST_PATH
+    static size_t alloc_size(const void* pointer)
+    {
+      return singleton.getSize(pointer);
+    }
+  };
+} // namespace snmalloc

src/test/func/client_meta/client_meta.cc

@@ -23,7 +23,8 @@ namespace snmalloc
 
 int main()
 {
-#ifdef SNMALLOC_PASS_THROUGH
+#if defined(SNMALLOC_PASS_THROUGH) || \
+  defined(SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION)
   // This test does not make sense in pass-through
   return 0;
 #else

src/test/func/fixed_region/fixed_region.cc

@@ -1,3 +1,4 @@
+#include "snmalloc/mem/secondary.h"
 #include "test/setup.h"
 
 #include <iostream>
@@ -38,18 +39,26 @@ int main()
   while (true)
   {
     auto r1 = a.alloc(object_size);
+
     count += object_size;
     i++;
 
+    // Run until we exhaust the fixed region.
+    // This should return null.
+    if (r1 == nullptr)
+      break;
+
+    if (!a.is_snmalloc_owned(r1))
+    {
+      a.dealloc(r1);
+      continue;
+    }
+
     if (i == 1024)
     {
       i = 0;
       std::cout << ".";
     }
-    // Run until we exhaust the fixed region.
-    // This should return null.
-    if (r1 == nullptr)
-      break;
 
     if (oe_base > r1)
     {

src/test/perf/external_pointer/externalpointer.cc

@@ -76,12 +76,16 @@ namespace test
         size_t rand = (size_t)r.next();
         size_t oid = rand & (((size_t)1 << count_log) - 1);
         size_t* external_ptr = objects[oid];
+        if (!alloc.is_snmalloc_owned(external_ptr))
+          continue;
         size_t size = *external_ptr;
         size_t offset = (size >> 4) * (rand & 15);
         void* interior_ptr = pointer_offset(external_ptr, offset);
         void* calced_external = alloc.external_pointer(interior_ptr);
         if (calced_external != external_ptr)
+        {
           abort();
+        }
       }
     }