diff --git a/src/ds/aba.h b/src/ds/aba.h index 5d4e092..4b8f071 100644 --- a/src/ds/aba.h +++ b/src/ds/aba.h @@ -1,6 +1,7 @@ #pragma once #include "bits.h" +#include "ptrwrap.h" /** * This file contains an abstraction of ABA protection. This API should be @@ -23,19 +24,23 @@ namespace snmalloc // fall back to locked implementation. #if defined(PLATFORM_IS_X86) && \ !(defined(GCC_NOT_CLANG) && defined(OPEN_ENCLAVE)) - template + template< + typename T, + Construction c = RequiresInit, + template typename Ptr = Pointer, + template typename AtomicPtr = AtomicPointer> class ABA { public: struct alignas(2 * sizeof(std::size_t)) Linked { - T* ptr; + Ptr ptr; uintptr_t aba; }; struct Independent { - std::atomic ptr; + AtomicPtr ptr; std::atomic aba; }; @@ -60,7 +65,7 @@ namespace snmalloc init(nullptr); } - void init(T* x) + void init(Ptr x) { independent.ptr.store(x, std::memory_order_relaxed); independent.aba.store(0, std::memory_order_relaxed); @@ -92,18 +97,18 @@ namespace snmalloc */ Cmp(Linked old, ABA* parent) : old(old), parent(parent) {} - T* ptr() + Ptr ptr() { return old.ptr; } - bool store_conditional(T* value) + bool store_conditional(Ptr value) { # if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_64) auto result = _InterlockedCompareExchange128( (volatile __int64*)parent, (__int64)(old.aba + (uintptr_t)1), - (__int64)value, + (__int64)address_cast(value), (__int64*)&old); # else # if defined(__GNUC__) && defined(SNMALLOC_VA_BITS_64) && \ @@ -132,7 +137,7 @@ namespace snmalloc }; // This method is used in Verona - T* peek() + Ptr peek() { return independent.ptr.load(std::memory_order_relaxed); } @@ -141,10 +146,14 @@ namespace snmalloc /** * Naive implementation of ABA protection using a spin lock. */ - template + template< + typename T, + Construction c = RequiresInit, + template typename Ptr = Pointer, + template typename AtomicPtr = AtomicPointer> class ABA { - std::atomic ptr = nullptr; + AtomicPtr ptr = nullptr; std::atomic_flag lock = ATOMIC_FLAG_INIT; public: @@ -169,12 +178,12 @@ namespace snmalloc ABA* parent; public: - T* ptr() + Ptr ptr() { return parent->ptr; } - bool store_conditional(T* t) + bool store_conditional(Ptr t) { parent->ptr = t; return true; @@ -190,7 +199,7 @@ namespace snmalloc }; // This method is used in Verona - T* peek() + Ptr peek() { return ptr.load(std::memory_order_relaxed); } diff --git a/src/ds/mpmcstack.h b/src/ds/mpmcstack.h index 153bb1e..bd16c08 100644 --- a/src/ds/mpmcstack.h +++ b/src/ds/mpmcstack.h @@ -4,43 +4,47 @@ namespace snmalloc { - template + template< + class T, + Construction c = RequiresInit, + template typename Ptr = Pointer, + template typename AtomicPtr = AtomicPointer> class MPMCStack { - using ABAT = ABA; + using ABAT = ABA; private: static_assert( - std::is_same>::value, - "T->next must be a std::atomic"); + std::is_same>::value, + "T->next must be an AtomicPtr"); ABAT stack; public: - void push(T* item) + void push(Ptr item) { return push(item, item); } - void push(T* first, T* last) + void push(Ptr first, Ptr last) { // Pushes an item on the stack. auto cmp = stack.read(); do { - T* top = cmp.ptr(); + Ptr top = cmp.ptr(); last->next.store(top, std::memory_order_release); } while (!cmp.store_conditional(first)); } - T* pop() + Ptr pop() { // Returns the next item. If the returned value is decommitted, it is // possible for the read of top->next to segfault. auto cmp = stack.read(); - T* top; - T* next; + Ptr top; + Ptr next; do { @@ -55,11 +59,11 @@ namespace snmalloc return top; } - T* pop_all() + Ptr pop_all() { // Returns all items as a linked list, leaving an empty stack. auto cmp = stack.read(); - T* top; + Ptr top; do { diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index b3433b5..f0540cd 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -22,7 +22,13 @@ namespace snmalloc // This is the view of a contiguous memory area when it is being kept // in the global size-classed caches of available contiguous memory areas. private: - template + template< + class a, + Construction c, + template + typename P, + template + typename AP> friend class MPMCStack; template friend class MemoryProviderStateMixin; diff --git a/src/mem/pooled.h b/src/mem/pooled.h index 4778a8f..a4ffa1e 100644 --- a/src/mem/pooled.h +++ b/src/mem/pooled.h @@ -10,7 +10,13 @@ namespace snmalloc private: template friend class Pool; - template + template< + class a, + Construction c, + template + typename P, + template + typename AP> friend class MPMCStack; /// Used by the pool for chaining together entries when not in use.