Configurable client meta-data (#662)

This provide a way to configure snmalloc to provide per object meta-data that is out of band. This can be used to provide different mitigations on top of snmalloc, such as storing memory tags in a compressed form, or provide a miracle pointer like feature.

This also includes a couple of TSAN fixes as it wasn't fully on in CI.

src/test/func/client_meta/client_meta.cc

@@ -0,0 +1,68 @@
+/**
+ * This test performs a very simple use of the client_meta data feature in
+ * snmalloc.
+ */
+
+#include "test/setup.h"
+
+#include <iostream>
+#include <snmalloc/backend/globalconfig.h>
+#include <snmalloc/snmalloc_core.h>
+#include <vector>
+
+namespace snmalloc
+{
+  // Create an allocator that stores an std::atomic<size_t>> per allocation.
+  using Alloc = snmalloc::LocalAllocator<snmalloc::StandardConfigClientMeta<
+    ArrayClientMetaDataProvider<std::atomic<size_t>>>>;
+}
+#define SNMALLOC_PROVIDE_OWN_CONFIG
+#include <snmalloc/snmalloc.h>
+
+int main()
+{
+#ifdef SNMALLOC_PASS_THROUGH
+  // This test does not make sense in pass-through
+  return 0;
+#else
+  // Allocate a bunch of objects, and store the index into the meta-data.
+  std::vector<void*> ptrs;
+  for (size_t i = 0; i < 10000; i++)
+  {
+    auto p = snmalloc::libc::malloc(1024);
+    auto& meta = snmalloc::libc::get_client_meta_data(p);
+    meta = i;
+    ptrs.push_back(p);
+    memset(p, (uint8_t)i, 1024);
+  }
+
+  // Check meta-data contains expected value, and that the memory contains
+  // the expected pattern.
+  for (size_t i = 0; i < 10000; i++)
+  {
+    auto p = ptrs[i];
+    auto& meta = snmalloc::libc::get_client_meta_data(p);
+    if (meta != i)
+    {
+      std::cout << "Failed at index " << i << std::endl;
+      abort();
+    }
+    for (size_t j = 0; j < 1024; j++)
+    {
+      if (reinterpret_cast<uint8_t*>(p)[j] != (uint8_t)i)
+      {
+        std::cout << "Failed at index " << i << " byte " << j << std::endl;
+        abort();
+      }
+    }
+    snmalloc::libc::free(p);
+  }
+
+  // Access in a read-only way meta-data associated with the stack.
+  // This would fail if it was accessed for write.
+  auto& meta = snmalloc::libc::get_client_meta_data_const(&ptrs);
+  std::cout << "meta for stack" << meta << std::endl;
+
+  return 0;
+#endif
+}

src/test/func/domestication/domestication.cc

@@ -23,7 +23,8 @@ namespace snmalloc
   {
   public:
     using Pal = DefaultPal;
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<NoClientMetaDataProvider>;
+    using ClientMeta = NoClientMetaDataProvider;
 
   private:
     using ConcretePagemap =

src/test/func/malloc/malloc.cc

@@ -375,6 +375,6 @@ int main(int argc, char** argv)
     our_malloc_usable_size(nullptr) == 0,
     "malloc_usable_size(nullptr) should be zero");
 
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
   return 0;
 }

src/test/func/memory/memory.cc

@@ -184,7 +184,7 @@ void test_calloc()
     alloc.dealloc(p, size);
   }
 
-  snmalloc::debug_check_empty<StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
 }
 
 void test_double_alloc()
@@ -229,7 +229,7 @@ void test_double_alloc()
       }
     }
   }
-  snmalloc::debug_check_empty<StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
 }
 
 void test_external_pointer()
@@ -275,7 +275,7 @@ void test_external_pointer()
     alloc.dealloc(p1, size);
   }
 
-  snmalloc::debug_check_empty<StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
 };
 
 void check_offset(void* base, void* interior)

src/test/func/miracle_ptr/miracle_ptr.cc

@@ -0,0 +1,203 @@
+/**
+ * This file demonstrates how the snmalloc library could be implemented to
+ * provide a miracle pointer like feature.  This is not a hardened
+ * implementation and is purely for illustrative purposes.
+ *
+ * Do not use as is.
+ */
+
+#ifdef SNMALLOC_THREAD_SANITIZER_ENABLED
+int main()
+{
+  return 0;
+}
+#else
+
+#  include "test/setup.h"
+
+#  include <iostream>
+#  include <memory>
+#  include <snmalloc/backend/globalconfig.h>
+#  include <snmalloc/snmalloc_core.h>
+
+namespace snmalloc
+{
+  // Instantiate the allocator with a client meta data provider that uses an
+  // atomic size_t to store the reference count.
+  using Alloc = snmalloc::LocalAllocator<snmalloc::StandardConfigClientMeta<
+    ArrayClientMetaDataProvider<std::atomic<size_t>>>>;
+}
+#  define SNMALLOC_PROVIDE_OWN_CONFIG
+#  include <snmalloc/snmalloc.h>
+
+SNMALLOC_SLOW_PATH void error(std::string msg)
+{
+  std::cout << msg << std::endl;
+  abort();
+}
+
+SNMALLOC_FAST_PATH_INLINE void check(bool b, std::string msg)
+{
+  if (SNMALLOC_UNLIKELY(!b))
+    error(msg);
+}
+
+namespace snmalloc::miracle
+{
+  // snmalloc meta-data representation
+  //   * 2n + 1:  Represents an object that has not been deallocated with n
+  //              additional references to it
+  //   * 2n    :  Represents a deallocated object that
+  //              has n additional references to it
+
+  inline void* malloc(size_t size)
+  {
+    auto p = snmalloc::libc::malloc(size);
+    if (SNMALLOC_UNLIKELY(p == nullptr))
+      return nullptr;
+
+    snmalloc::libc::get_client_meta_data(p) = 1;
+    return p;
+  }
+
+  inline void free(void* ptr)
+  {
+    if (ptr == nullptr)
+      return;
+
+    // TODO could build a check into this that it is the start of the object?
+    auto previous =
+      snmalloc::libc::get_client_meta_data(ptr).fetch_add((size_t)-1);
+
+    if (SNMALLOC_LIKELY(previous == 1))
+    {
+      std::cout << "Freeing " << ptr << std::endl;
+      snmalloc::libc::free(ptr);
+      return;
+    }
+
+    check((previous & 1) == 1, "Double free detected");
+
+    // We have additional references to this object.
+    // We should not free it.
+    // TOOD this assumes this is not an internal pointer.
+    memset(ptr, 0, snmalloc::libc::malloc_usable_size(ptr));
+  }
+
+  inline void acquire(void* p)
+  {
+    auto previous =
+      snmalloc::libc::get_client_meta_data(p).fetch_add((size_t)2);
+
+    // Can we take new pointers to a deallocated object?
+    check((previous & 1) == 1, "Acquiring a deallocated object");
+  }
+
+  inline void release(void* p)
+  {
+    auto previous =
+      snmalloc::libc::get_client_meta_data(p).fetch_add((size_t)-2);
+
+    if (previous > 2)
+      return;
+
+    check(previous == 2, "Releasing an object with insufficient references");
+
+    std::cout << "Freeing from release " << p << std::endl;
+    snmalloc::libc::free(p);
+  }
+
+  /**
+   * This class can be used to replace a raw pointer. It will automatically use
+   * the underlying backup reference counting design from the miracle pointer
+   * docs.
+   */
+  template<typename T>
+  class raw_ptr
+  {
+    T* p;
+
+  public:
+    raw_ptr() : p(nullptr) {}
+
+    raw_ptr(T* p) : p(p)
+    {
+      snmalloc::miracle::acquire(p);
+    }
+
+    T& operator*()
+    {
+      return *p;
+    }
+
+    ~raw_ptr()
+    {
+      if (p == nullptr)
+        return;
+      snmalloc::miracle::release(p);
+    }
+
+    raw_ptr(const raw_ptr& rp) : p(rp.p)
+    {
+      snmalloc::miracle::acquire(p);
+    }
+
+    raw_ptr& operator=(const raw_ptr& other)
+    {
+      p = other.p;
+      snmalloc::miracle::acquire(other.p);
+      return *this;
+    }
+
+    raw_ptr(raw_ptr&& other) : p(other.p)
+    {
+      other.p = nullptr;
+    }
+
+    raw_ptr& operator=(raw_ptr&& other)
+    {
+      p = other.p;
+      other.p = nullptr;
+      return *this;
+    }
+  };
+} // namespace snmalloc::miracle
+
+/**
+ * Overload new and delete to use the "miracle pointer" implementation.
+ */
+void* operator new(size_t size)
+{
+  return snmalloc::miracle::malloc(size);
+}
+
+void operator delete(void* p)
+{
+  snmalloc::miracle::free(p);
+}
+
+void operator delete(void* p, size_t)
+{
+  snmalloc::miracle::free(p);
+}
+
+int main()
+{
+#  ifndef SNMALLOC_PASS_THROUGH
+  snmalloc::miracle::raw_ptr<int> p;
+  {
+    auto up1 = std::make_unique<int>(41);
+    auto up = std::make_unique<int>(42);
+    auto up2 = std::make_unique<int>(40);
+    auto up3 = std::make_unique<int>(39);
+    p = up.get();
+    check(*p == 42, "Failed to set p");
+  }
+  // Still safe to access here.  The unique_ptr has been destroyed, but the
+  // raw_ptr has kept the memory live.
+  // Current implementation zeros the memory when the unique_ptr is destroyed.
+  check(*p == 0, "Failed to keep memory live");
+#  endif
+  return 0;
+}
+#endif

src/test/func/statistics/stats.cc

@@ -17,7 +17,7 @@ void debug_check_empty_1()
 
   auto r = a.alloc(size);
 
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
   if (result != false)
   {
     std::cout << "debug_check_empty failed to detect leaked memory:" << size
@@ -27,7 +27,7 @@ void debug_check_empty_1()
 
   a.dealloc(r);
 
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
   if (result != true)
   {
     std::cout << "debug_check_empty failed to say empty:" << size << std::endl;
@@ -36,7 +36,7 @@ void debug_check_empty_1()
 
   r = a.alloc(size);
 
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
   if (result != false)
   {
     std::cout << "debug_check_empty failed to detect leaked memory:" << size
@@ -46,7 +46,7 @@ void debug_check_empty_1()
 
   a.dealloc(r);
 
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
   if (result != true)
   {
     std::cout << "debug_check_empty failed to say empty:" << size << std::endl;
@@ -72,7 +72,7 @@ void debug_check_empty_2()
     }
     auto r = a.alloc(size);
     allocs.push_back(r);
-    snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+    snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
     if (result != false)
     {
       std::cout << "False empty after " << i << " allocations of " << size
@@ -88,7 +88,7 @@ void debug_check_empty_2()
     {
       std::cout << "." << std::flush;
     }
-    snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+    snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
     if (result != false)
     {
       std::cout << "False empty after " << i << " deallocations of " << size
@@ -98,7 +98,7 @@ void debug_check_empty_2()
     a.dealloc(allocs[i]);
   }
   std::cout << std::endl;
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
 }
 
 int main()

src/test/func/thread_alloc_external/thread_alloc_external.cc

@@ -12,7 +12,8 @@
 
 namespace snmalloc
 {
-  using Alloc = snmalloc::LocalAllocator<snmalloc::StandardConfig>;
+  using Alloc = snmalloc::LocalAllocator<
+    snmalloc::StandardConfigClientMeta<NoClientMetaDataProvider>>;
 }
 
 using namespace snmalloc;