diff --git a/src/ds/helpers.h b/src/ds/helpers.h index 6994e82..4ba019b 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -3,6 +3,8 @@ #include "bits.h" #include "flaglock.h" +#include + namespace snmalloc { /* @@ -156,4 +158,69 @@ namespace snmalloc { UNUSED(t); } + + /** + * Sometimes we need atomics with trivial initializer. Unfortunately, this + * became harder to accomplish in C++20. Fortunately, our rules for accessing + * these are at least as strong as those required by C++20's atomic_ref: + * + * * The objects outlive any references to them + * + * * We always access the objects through references (though we'd be allowed + * to access them without if we knew there weren't other references) + * + * * We don't access sub-objects at all, much less concurrently through + * other references. + */ + template + class TrivialInitAtomic + { + static_assert( + std::is_trivially_default_constructible_v, + "TrivialInitAtomic should not attempt to call nontrivial constructors"); + +#ifdef __cpp_lib_atomic_ref + using Val = T; + using Ref = std::atomic_ref; +#else + using Val = std::atomic; + using Ref = std::atomic&; +#endif + Val v; + + public: + /** + * Construct a reference to this value; use .load and .store to manipulate + * the value. + */ + SNMALLOC_FAST_PATH Ref ref() + { +#ifdef __cpp_lib_atomic_ref + return std::atomic_ref(this->v); +#else + return this->v; +#endif + } + + SNMALLOC_FAST_PATH T + load(std::memory_order mo = std::memory_order_seq_cst) noexcept + { + return this->ref().load(mo); + } + + SNMALLOC_FAST_PATH void + store(T n, std::memory_order mo = std::memory_order_seq_cst) noexcept + { + return this->ref().store(n, mo); + } + + SNMALLOC_FAST_PATH bool compare_exchange_strong( + T& exp, T des, std::memory_order mo = std::memory_order_seq_cst) noexcept + { + return this->ref().compare_exchange_strong(exp, des, mo); + } + }; + + static_assert(sizeof(TrivialInitAtomic) == sizeof(char)); + static_assert(alignof(TrivialInitAtomic) == alignof(char)); } // namespace snmalloc diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index f15502e..5f0972c 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -107,12 +107,20 @@ namespace snmalloc struct Leaf { - std::atomic values[ENTRIES_PER_LEAF]; + TrivialInitAtomic values[ENTRIES_PER_LEAF]; + + static_assert(sizeof(TrivialInitAtomic) == sizeof(T)); + static_assert(alignof(TrivialInitAtomic) == alignof(T)); }; struct PagemapEntry { - std::atomic entries[ENTRIES_PER_INDEX_LEVEL]; + TrivialInitAtomic entries[ENTRIES_PER_INDEX_LEVEL]; + + static_assert( + sizeof(TrivialInitAtomic) == sizeof(PagemapEntry*)); + static_assert( + alignof(TrivialInitAtomic) == alignof(PagemapEntry*)); }; static_assert( @@ -127,11 +135,11 @@ namespace snmalloc // allocation. // TODO: This is fragile that it is not being memset, and we should review // to ensure we don't get bitten by this in the future. - std::atomic top[TOPLEVEL_ENTRIES]; // = {nullptr}; + TrivialInitAtomic top[TOPLEVEL_ENTRIES]; template SNMALLOC_FAST_PATH PagemapEntry* - get_node(std::atomic* e, bool& result) + get_node(TrivialInitAtomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -156,7 +164,7 @@ namespace snmalloc } SNMALLOC_SLOW_PATH PagemapEntry* - get_node_slow(std::atomic* e, bool& result) + get_node_slow(TrivialInitAtomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -198,7 +206,7 @@ namespace snmalloc #endif size_t ix = addr >> TOPLEVEL_SHIFT; size_t shift = TOPLEVEL_SHIFT; - std::atomic* e = &top[ix]; + TrivialInitAtomic* e = &top[ix]; // This is effectively a // for (size_t i = 0; i < INDEX_LEVELS; i++) @@ -242,13 +250,14 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH std::atomic* get_addr(uintptr_t p, bool& success) + SNMALLOC_FAST_PATH TrivialInitAtomic* + get_addr(uintptr_t p, bool& success) { auto leaf_ix = get_leaf_index(p, success); return &(leaf_ix.first->values[leaf_ix.second]); } - std::atomic* get_ptr(uintptr_t p) + TrivialInitAtomic* get_ptr(uintptr_t p) { bool success; return get_addr(p, success); @@ -333,7 +342,7 @@ namespace snmalloc for (; ix < last; ix++) { SNMALLOC_ASSUME(leaf_ix.first != nullptr); - leaf_ix.first->values[ix] = x; + leaf_ix.first->values[ix].store(x); } length = length - diff; @@ -355,7 +364,10 @@ namespace snmalloc static constexpr size_t ENTRIES = 1ULL << COVERED_BITS; static constexpr size_t SHIFT = GRANULARITY_BITS; - std::atomic top[ENTRIES]; + TrivialInitAtomic top[ENTRIES]; + + static_assert(sizeof(TrivialInitAtomic) == sizeof(T)); + static_assert(alignof(TrivialInitAtomic) == alignof(T)); public: /**