diff --git a/src/ds/helpers.h b/src/ds/helpers.h index e34387d..f8ad92c 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -13,7 +13,7 @@ namespace snmalloc * initialised. This singleton class is designed to not depend on the * runtime. */ - template + template class Singleton { inline static std::atomic_flag flag; @@ -36,7 +36,7 @@ namespace snmalloc FlagLock lock(flag); if (!initialised) { - obj = init(); + init(&obj); initialised.store(true, std::memory_order_release); if (first != nullptr) *first = true; diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 8c34844..b67df18 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -4,7 +4,7 @@ #include "allocconfig.h" #include "localcache.h" #include "metaslab.h" -#include "pooled.h" +#include "pool.h" #include "remotecache.h" #include "sizeclasstable.h" #include "slaballocator.h" @@ -785,4 +785,13 @@ namespace snmalloc return debug_is_empty_impl(result); } }; + + /** + * Use this alias to access the pool of allocators throughout snmalloc. + */ + template + using AllocPool = Pool< + CoreAllocator, + SharedStateHandle, + SharedStateHandle::pool>; } // namespace snmalloc diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index cc4ecd5..674b7ae 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -11,8 +11,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto* alloc = Pool>::template iterate< - SharedStateHandle>(); + auto* alloc = AllocPool::iterate(); while (alloc != nullptr) { @@ -20,8 +19,7 @@ namespace snmalloc if (a != nullptr) stats.add(*a); stats.add(alloc->stats()); - alloc = Pool>::template iterate< - SharedStateHandle>(alloc); + alloc = AllocPool::iterate(alloc); } } @@ -32,16 +30,14 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto alloc = Pool>::template iterate< - SharedStateHandle>(); + auto alloc = AllocPool::iterate(); while (alloc != nullptr) { auto stats = alloc->stats(); if (stats != nullptr) stats->template print(o, dumpid, alloc->id()); - alloc = Pool>::template iterate< - SharedStateHandle>(alloc); + alloc = AllocPool::iterate(alloc); } } #else @@ -64,7 +60,7 @@ namespace snmalloc // allocators that are not currently in use by any thread. // One atomic operation to extract the stack, another to restore it. // Handling the message queue for each stack is non-atomic. - auto* first = Pool>::extract(); + auto* first = AllocPool::extract(); auto* alloc = first; decltype(alloc) last; @@ -74,10 +70,10 @@ namespace snmalloc { alloc->flush(); last = alloc; - alloc = Pool>::extract(alloc); + alloc = AllocPool::extract(alloc); } - Pool>::restore(first, last); + AllocPool::restore(first, last); } #endif } @@ -96,8 +92,7 @@ namespace snmalloc "Global status is available only for pool-allocated configurations"); // This is a debugging function. It checks that all memory from all // allocators has been freed. - auto* alloc = Pool>::template iterate< - SharedStateHandle>(); + auto* alloc = AllocPool::iterate(); # ifdef SNMALLOC_TRACING std::cout << "debug check empty: first " << alloc << std::endl; @@ -111,8 +106,7 @@ namespace snmalloc std::cout << "debug_check_empty: Check all allocators!" << std::endl; # endif done = true; - alloc = Pool>::template iterate< - SharedStateHandle>(); + alloc = AllocPool::iterate(); okay = true; while (alloc != nullptr) @@ -134,8 +128,7 @@ namespace snmalloc # ifdef SNMALLOC_TRACING std::cout << "debug check empty: okay = " << okay << std::endl; # endif - alloc = Pool>::template iterate< - SharedStateHandle>(alloc); + alloc = AllocPool::iterate(alloc); } } @@ -148,13 +141,11 @@ namespace snmalloc // Redo check so abort is on allocator with allocation left. if (!okay) { - alloc = Pool>::template iterate< - SharedStateHandle>(); + alloc = AllocPool::iterate(); while (alloc != nullptr) { alloc->debug_is_empty(nullptr); - alloc = Pool>::template iterate< - SharedStateHandle>(alloc); + alloc = AllocPool::iterate(alloc); } } #else @@ -168,8 +159,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global status is available only for pool-allocated configurations"); - auto alloc = Pool>::template iterate< - SharedStateHandle>(); + auto alloc = AllocPool::iterate(); while (alloc != nullptr) { if (alloc->debug_is_in_use()) @@ -180,8 +170,7 @@ namespace snmalloc } count--; } - alloc = Pool>::template iterate< - SharedStateHandle>(alloc); + alloc = AllocPool::iterate(alloc); if (count != 0) { diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 39255cc..f4a576e 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -58,9 +58,12 @@ namespace snmalloc template class LocalAllocator { - using CoreAlloc = CoreAllocator; + public: + using StateHandle = SharedStateHandle; private: + using CoreAlloc = CoreAllocator; + // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. @@ -341,6 +344,8 @@ namespace snmalloc public: constexpr LocalAllocator() = default; + LocalAllocator(const LocalAllocator&) = delete; + LocalAllocator& operator=(const LocalAllocator&) = delete; /** * Initialise the allocator. For allocators that support local @@ -375,8 +380,7 @@ namespace snmalloc // Initialise the global allocator structures ensure_init(); // Grab an allocator for this thread. - init(Pool::template acquire( - &(this->local_cache))); + init(AllocPool::acquire(&(this->local_cache))); } // Return all state in the fast allocator and release the underlying @@ -398,7 +402,7 @@ namespace snmalloc // Return underlying allocator to the system. if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) { - Pool::template release(core_alloc); + AllocPool::release(core_alloc); } // Set up thread local allocator to look like diff --git a/src/mem/pool.h b/src/mem/pool.h index d3c25b4..f8245c5 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -4,6 +4,7 @@ #include "../ds/mpmcstack.h" #include "../pal/pal_concept.h" #include "pooled.h" +#include "slaballocator.h" namespace snmalloc { @@ -20,7 +21,10 @@ namespace snmalloc template class PoolState { - template + template< + typename TT, + typename SharedStateHandle, + PoolState& get_state()> friend class Pool; private: @@ -32,14 +36,92 @@ namespace snmalloc constexpr PoolState() = default; }; - template + /** + * Helper class used to instantiate a global PoolState. + * + * SingletonPoolState::pool is the default provider for the PoolState within + * the Pool class. + */ + template + class SingletonPoolState + { + /** + * SFINAE helper. Matched only if `T` implements `ensure_init`. Calls it + * if it exists. + */ + template + SNMALLOC_FAST_PATH static auto call_ensure_init(SharedStateHandle_*, int) + -> decltype(SharedStateHandle_::ensure_init()) + { + static_assert( + std::is_same::value, + "SFINAE parameter, should only be used with SharedStateHandle"); + SharedStateHandle_::ensure_init(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `ensure_init`. + * Does nothing if called. + */ + template + SNMALLOC_FAST_PATH static auto call_ensure_init(SharedStateHandle_*, long) + { + static_assert( + std::is_same::value, + "SFINAE parameter, should only be used with SharedStateHandle"); + } + + /** + * Call `SharedStateHandle::ensure_init()` if it is implemented, do nothing + * otherwise. + */ + SNMALLOC_FAST_PATH static void ensure_init() + { + call_ensure_init(nullptr, 0); + } + + static void make_pool(PoolState*) noexcept + { + ensure_init(); + // Default initializer already called on PoolState, no need to use + // placement new. + } + + public: + /** + * Returns a reference for the global PoolState for the given type. + * Also forces the initialization of the backend state, if needed. + */ + SNMALLOC_FAST_PATH static PoolState& pool() + { + return Singleton, &make_pool>::get(); + } + }; + + /** + * Wrapper class to access a pool of a particular type of object. + * + * The third template argument is a method to retrieve the actual PoolState. + * + * For the pool of allocators, refer to the AllocPool alias defined in + * corealloc.h. + * + * For a pool of another type, it is recommended to leave the + * third template argument with its default value. The SingletonPoolState + * class is used as a helper to provide a default PoolState management for + * this use case. + */ + template< + typename T, + typename SharedStateHandle, + PoolState& get_state() = SingletonPoolState::pool> class Pool { public: - template + template static T* acquire(Args&&... args) { - PoolState& pool = SharedStateHandle::pool(); + PoolState& pool = get_state(); T* p = pool.stack.pop(); if (p != nullptr) @@ -70,22 +152,20 @@ namespace snmalloc * * Do not return objects from `extract`. */ - template static void release(T* p) { // The object's destructor is not run. If the object is "reallocated", it // is returned without the constructor being run, so the object is reused // without re-initialisation. p->reset_in_use(); - SharedStateHandle::pool().stack.push(p); + get_state().stack.push(p); } - template static T* extract(T* p = nullptr) { // Returns a linked list of all objects in the stack, emptying the stack. if (p == nullptr) - return SharedStateHandle::pool().stack.pop_all(); + return get_state().stack.pop_all(); return p->next; } @@ -95,19 +175,17 @@ namespace snmalloc * * Do not return objects from `acquire`. */ - template static void restore(T* first, T* last) { // Pushes a linked list of objects onto the stack. Use to put a linked // list returned by extract back onto the stack. - SharedStateHandle::pool().stack.push(first, last); + get_state().stack.push(first, last); } - template static T* iterate(T* p = nullptr) { if (p == nullptr) - return SharedStateHandle::pool().list; + return get_state().list; return p->list_next; } diff --git a/src/mem/pooled.h b/src/mem/pooled.h index 93daad9..d5b988c 100644 --- a/src/mem/pooled.h +++ b/src/mem/pooled.h @@ -4,11 +4,17 @@ namespace snmalloc { + template + class PoolState; + template class Pooled { - private: - template + public: + template< + typename TT, + typename SharedStateHandle, + PoolState& get_state()> friend class Pool; template friend class MPMCStack; diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index dc9cee6..3777d44 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -106,11 +106,9 @@ namespace snmalloc /** * Used to give correct signature to the pthread call for the Singleton class. */ - inline pthread_key_t pthread_create() noexcept + inline void pthread_create(pthread_key_t* key) noexcept { - pthread_key_t key; - pthread_key_create(&key, &pthread_cleanup); - return key; + pthread_key_create(key, &pthread_cleanup); } /** * Performs thread local teardown for the allocator using the pthread library. diff --git a/src/test/func/pool/pool.cc b/src/test/func/pool/pool.cc new file mode 100644 index 0000000..f479f4e --- /dev/null +++ b/src/test/func/pool/pool.cc @@ -0,0 +1,138 @@ +#include +#include +#include +#include + +using namespace snmalloc; + +struct PoolAEntry : Pooled +{ + int field; + + PoolAEntry() : field(1){}; +}; + +using PoolA = Pool; + +struct PoolBEntry : Pooled +{ + int field; + + PoolBEntry() : field(0){}; + PoolBEntry(int f) : field(f){}; +}; + +using PoolB = Pool; + +void test_alloc() +{ + auto ptr = PoolA::acquire(); + SNMALLOC_CHECK(ptr != nullptr); + // Pool allocations should not be visible to debug_check_empty. + snmalloc::debug_check_empty(); + PoolA::release(ptr); +} + +void test_constructor() +{ + auto ptr1 = PoolA::acquire(); + SNMALLOC_CHECK(ptr1 != nullptr); + SNMALLOC_CHECK(ptr1->field == 1); + + auto ptr2 = PoolB::acquire(); + SNMALLOC_CHECK(ptr2 != nullptr); + SNMALLOC_CHECK(ptr2->field == 0); + + auto ptr3 = PoolB::acquire(1); + SNMALLOC_CHECK(ptr3 != nullptr); + SNMALLOC_CHECK(ptr3->field == 1); + + PoolA::release(ptr1); + PoolB::release(ptr2); + PoolB::release(ptr3); +} + +void test_alloc_many() +{ + constexpr size_t count = 16'000'000 / MIN_CHUNK_SIZE; + + std::unordered_set allocated; + + for (size_t i = 0; i < count; ++i) + { + auto ptr = PoolA::acquire(); + SNMALLOC_CHECK(ptr != nullptr); + allocated.insert(ptr); + } + + for (auto ptr : allocated) + { + PoolA::release(ptr); + } +} + +void test_double_alloc() +{ + auto ptr1 = PoolA::acquire(); + SNMALLOC_CHECK(ptr1 != nullptr); + auto ptr2 = PoolA::acquire(); + SNMALLOC_CHECK(ptr2 != nullptr); + SNMALLOC_CHECK(ptr1 != ptr2); + PoolA::release(ptr2); + auto ptr3 = PoolA::acquire(); + SNMALLOC_CHECK(ptr2 == ptr3); + PoolA::release(ptr1); + PoolA::release(ptr3); +} + +void test_different_alloc() +{ + auto ptr1 = PoolA::acquire(); + SNMALLOC_CHECK(ptr1 != nullptr); + PoolA::release(ptr1); + auto ptr2 = PoolB::acquire(); + SNMALLOC_CHECK(ptr2 != nullptr); + SNMALLOC_CHECK(static_cast(ptr1) != static_cast(ptr2)); + PoolB::release(ptr2); +} + +void test_iterator() +{ + PoolAEntry* before_iteration_ptr = PoolA::acquire(); + + PoolAEntry* ptr = nullptr; + while ((ptr = PoolA::iterate(ptr)) != nullptr) + { + ptr->field = 2; + } + + SNMALLOC_CHECK(before_iteration_ptr->field == 2); + + PoolAEntry* after_iteration_ptr = PoolA::acquire(); + + SNMALLOC_CHECK(after_iteration_ptr->field == 2); + + PoolA::release(before_iteration_ptr); + PoolA::release(after_iteration_ptr); +} + +int main(int argc, char** argv) +{ + setup(); +#ifdef USE_SYSTEMATIC_TESTING + opt::Opt opt(argc, argv); + size_t seed = opt.is("--seed", 0); + Virtual::systematic_bump_ptr() += seed << 17; +#else + UNUSED(argc); + UNUSED(argv); +#endif + + test_alloc(); + test_constructor(); + test_alloc_many(); + test_double_alloc(); + test_different_alloc(); + test_iterator(); + return 0; +}