C++20: Add TrivialInitAtomic

C++20 does away with trivial initializers for std::atomic<T>, which means our
global pagemaps always get zeroed, sometimes after other static ctors have run
(fun fun!).  Use the new std::atomic_ref<T> when available.  Abstract all this
behind an #ifdef-ful wrapper.

src/ds/helpers.h

@@ -3,6 +3,8 @@
 #include "bits.h"
 #include "flaglock.h"
 
+#include <type_traits>
+
 namespace snmalloc
 {
   /*
@@ -156,4 +158,69 @@ namespace snmalloc
   {
     UNUSED(t);
   }
+
+  /**
+   * Sometimes we need atomics with trivial initializer.  Unfortunately, this
+   * became harder to accomplish in C++20.  Fortunately, our rules for accessing
+   * these are at least as strong as those required by C++20's atomic_ref:
+   *
+   *   * The objects outlive any references to them
+   *
+   *   * We always access the objects through references (though we'd be allowed
+   *     to access them without if we knew there weren't other references)
+   *
+   *   * We don't access sub-objects at all, much less concurrently through
+   *     other references.
+   */
+  template<typename T>
+  class TrivialInitAtomic
+  {
+    static_assert(
+      std::is_trivially_default_constructible_v<T>,
+      "TrivialInitAtomic should not attempt to call nontrivial constructors");
+
+#ifdef __cpp_lib_atomic_ref
+    using Val = T;
+    using Ref = std::atomic_ref<T>;
+#else
+    using Val = std::atomic<T>;
+    using Ref = std::atomic<T>&;
+#endif
+    Val v;
+
+  public:
+    /**
+     * Construct a reference to this value; use .load and .store to manipulate
+     * the value.
+     */
+    SNMALLOC_FAST_PATH Ref ref()
+    {
+#ifdef __cpp_lib_atomic_ref
+      return std::atomic_ref<T>(this->v);
+#else
+      return this->v;
+#endif
+    }
+
+    SNMALLOC_FAST_PATH T
+    load(std::memory_order mo = std::memory_order_seq_cst) noexcept
+    {
+      return this->ref().load(mo);
+    }
+
+    SNMALLOC_FAST_PATH void
+    store(T n, std::memory_order mo = std::memory_order_seq_cst) noexcept
+    {
+      return this->ref().store(n, mo);
+    }
+
+    SNMALLOC_FAST_PATH bool compare_exchange_strong(
+      T& exp, T des, std::memory_order mo = std::memory_order_seq_cst) noexcept
+    {
+      return this->ref().compare_exchange_strong(exp, des, mo);
+    }
+  };
+
+  static_assert(sizeof(TrivialInitAtomic<char>) == sizeof(char));
+  static_assert(alignof(TrivialInitAtomic<char>) == alignof(char));
 } // namespace snmalloc

src/mem/pagemap.h

@@ -107,12 +107,20 @@ namespace snmalloc
 
     struct Leaf
     {
-      std::atomic<T> values[ENTRIES_PER_LEAF];
+      TrivialInitAtomic<T> values[ENTRIES_PER_LEAF];
+
+      static_assert(sizeof(TrivialInitAtomic<T>) == sizeof(T));
+      static_assert(alignof(TrivialInitAtomic<T>) == alignof(T));
     };
 
     struct PagemapEntry
     {
-      std::atomic<PagemapEntry*> entries[ENTRIES_PER_INDEX_LEVEL];
+      TrivialInitAtomic<PagemapEntry*> entries[ENTRIES_PER_INDEX_LEVEL];
+
+      static_assert(
+        sizeof(TrivialInitAtomic<PagemapEntry*>) == sizeof(PagemapEntry*));
+      static_assert(
+        alignof(TrivialInitAtomic<PagemapEntry*>) == alignof(PagemapEntry*));
     };
 
     static_assert(
@@ -127,11 +135,11 @@ namespace snmalloc
     // allocation.
     //  TODO: This is fragile that it is not being memset, and we should review
     //  to ensure we don't get bitten by this in the future.
-    std::atomic<PagemapEntry*> top[TOPLEVEL_ENTRIES]; // = {nullptr};
+    TrivialInitAtomic<PagemapEntry*> top[TOPLEVEL_ENTRIES];
 
     template<bool create_addr>
     SNMALLOC_FAST_PATH PagemapEntry*
-    get_node(std::atomic<PagemapEntry*>* e, bool& result)
+    get_node(TrivialInitAtomic<PagemapEntry*>* e, bool& result)
     {
       // The page map nodes are all allocated directly from the OS zero
       // initialised with a system call.  We don't need any ordered to guarantee
@@ -156,7 +164,7 @@ namespace snmalloc
     }
 
     SNMALLOC_SLOW_PATH PagemapEntry*
-    get_node_slow(std::atomic<PagemapEntry*>* e, bool& result)
+    get_node_slow(TrivialInitAtomic<PagemapEntry*>* e, bool& result)
     {
       // The page map nodes are all allocated directly from the OS zero
       // initialised with a system call.  We don't need any ordered to guarantee
@@ -198,7 +206,7 @@ namespace snmalloc
 #endif
       size_t ix = addr >> TOPLEVEL_SHIFT;
       size_t shift = TOPLEVEL_SHIFT;
-      std::atomic<PagemapEntry*>* e = &top[ix];
+      TrivialInitAtomic<PagemapEntry*>* e = &top[ix];
 
       // This is effectively a
       //   for (size_t i = 0; i < INDEX_LEVELS; i++)
@@ -242,13 +250,14 @@ namespace snmalloc
     }
 
     template<bool create_addr>
-    SNMALLOC_FAST_PATH std::atomic<T>* get_addr(uintptr_t p, bool& success)
+    SNMALLOC_FAST_PATH TrivialInitAtomic<T>*
+    get_addr(uintptr_t p, bool& success)
     {
       auto leaf_ix = get_leaf_index<create_addr>(p, success);
       return &(leaf_ix.first->values[leaf_ix.second]);
     }
 
-    std::atomic<T>* get_ptr(uintptr_t p)
+    TrivialInitAtomic<T>* get_ptr(uintptr_t p)
     {
       bool success;
       return get_addr<true>(p, success);
@@ -333,7 +342,7 @@ namespace snmalloc
         for (; ix < last; ix++)
         {
           SNMALLOC_ASSUME(leaf_ix.first != nullptr);
-          leaf_ix.first->values[ix] = x;
+          leaf_ix.first->values[ix].store(x);
         }
 
         length = length - diff;
@@ -355,7 +364,10 @@ namespace snmalloc
     static constexpr size_t ENTRIES = 1ULL << COVERED_BITS;
     static constexpr size_t SHIFT = GRANULARITY_BITS;
 
-    std::atomic<T> top[ENTRIES];
+    TrivialInitAtomic<T> top[ENTRIES];
+
+    static_assert(sizeof(TrivialInitAtomic<T>) == sizeof(T));
+    static_assert(alignof(TrivialInitAtomic<T>) == alignof(T));
 
   public:
     /**