diff --git a/src/ds/spmcstack.h b/src/ds/spmcstack.h new file mode 100644 index 0000000..5651a09 --- /dev/null +++ b/src/ds/spmcstack.h @@ -0,0 +1,72 @@ +#pragma once + +#include "aba.h" +#include "ptrwrap.h" + +namespace snmalloc +{ + /** + * Concurrent Stack + * + * This stack supports the following clients + * (push|pop)* || pop_all* || ... || pop_all* + * + * That is a single thread that can do push and pop, and other threads + * that do pop_all. pop_all if it returns a value, returns all of the + * stack, however, it may return nullptr if it races with either a push + * or a pop. + * + * The primary use case is single-threaded access, where other threads + * can attempt to steal all the values. + */ + template + class SPMCStack + { + private: + alignas(CACHELINE_SIZE) std::atomic stack{}; + + public: + constexpr SPMCStack() = default; + + void push(T* item) + { + static_assert( + std::is_same>::value, + "T->next must be an std::atomic"); + + return push(item, item); + } + + void push(T* first, T* last) + { + T* old_head = stack.exchange(nullptr, std::memory_order_relaxed); + last->next.store(old_head, std::memory_order_relaxed); + // Assume stays null as not allowed to race with pop or other pushes. + SNMALLOC_ASSERT(stack.load() == nullptr); + stack.store(first, std::memory_order_release); + } + + T* pop() + { + if (stack.load(std::memory_order_relaxed) == nullptr) + return nullptr; + T* old_head = stack.exchange(nullptr); + if (unlikely(old_head == nullptr)) + return nullptr; + + auto next = old_head->next.load(std::memory_order_relaxed); + + // Assume stays null as not allowed to race with pop or other pushes. + SNMALLOC_ASSERT(stack.load() == nullptr); + + stack.store(next, std::memory_order_release); + + return old_head; + } + + T* pop_all() + { + return stack.exchange(nullptr); + } + }; +} // namespace snmalloc diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index cdfe8c7..0a8e41e 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -1,6 +1,7 @@ #pragma once #include "../ds/mpmcstack.h" +#include "../ds/spmcstack.h" #include "../mem/metaslab.h" #include "../mem/sizeclasstable.h" #include "../pal/pal_ds.h" @@ -55,9 +56,7 @@ namespace snmalloc /** * Stack of slabs that have been returned for reuse. */ - ModArray< - NUM_SLAB_SIZES, - ModArray>> + ModArray>> chunk_stack; /**