From 99f57646da42a9e9f03b186bbe3d1fb533a82e93 Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Thu, 19 Aug 2021 19:22:16 +0100 Subject: [PATCH] Re-enabled generic Pool which uses ChunkAllocator. Allocator pool renamed to AllocPool. --- src/mem/corealloc.h | 2 +- src/mem/globalalloc.h | 28 +++++++------- src/mem/localalloc.h | 9 +++-- src/mem/pool.h | 86 +++++++++++++++++++++++++++++++++++++++++-- src/mem/pooled.h | 4 +- 5 files changed, 107 insertions(+), 22 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 8c34844..467fa81 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -4,7 +4,7 @@ #include "allocconfig.h" #include "localcache.h" #include "metaslab.h" -#include "pooled.h" +#include "pool.h" #include "remotecache.h" #include "sizeclasstable.h" #include "slaballocator.h" diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index cc4ecd5..c44de93 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -11,7 +11,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto* alloc = Pool>::template iterate< + auto* alloc = AllocPool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) @@ -20,7 +20,7 @@ namespace snmalloc if (a != nullptr) stats.add(*a); stats.add(alloc->stats()); - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(alloc); } } @@ -32,7 +32,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto alloc = Pool>::template iterate< + auto alloc = AllocPool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) @@ -40,7 +40,7 @@ namespace snmalloc auto stats = alloc->stats(); if (stats != nullptr) stats->template print(o, dumpid, alloc->id()); - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(alloc); } } @@ -64,7 +64,7 @@ namespace snmalloc // allocators that are not currently in use by any thread. // One atomic operation to extract the stack, another to restore it. // Handling the message queue for each stack is non-atomic. - auto* first = Pool>::extract(); + auto* first = AllocPool>::extract(); auto* alloc = first; decltype(alloc) last; @@ -74,10 +74,10 @@ namespace snmalloc { alloc->flush(); last = alloc; - alloc = Pool>::extract(alloc); + alloc = AllocPool>::extract(alloc); } - Pool>::restore(first, last); + AllocPool>::restore(first, last); } #endif } @@ -96,7 +96,7 @@ namespace snmalloc "Global status is available only for pool-allocated configurations"); // This is a debugging function. It checks that all memory from all // allocators has been freed. - auto* alloc = Pool>::template iterate< + auto* alloc = AllocPool>::template iterate< SharedStateHandle>(); # ifdef SNMALLOC_TRACING @@ -111,7 +111,7 @@ namespace snmalloc std::cout << "debug_check_empty: Check all allocators!" << std::endl; # endif done = true; - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(); okay = true; @@ -134,7 +134,7 @@ namespace snmalloc # ifdef SNMALLOC_TRACING std::cout << "debug check empty: okay = " << okay << std::endl; # endif - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(alloc); } } @@ -148,12 +148,12 @@ namespace snmalloc // Redo check so abort is on allocator with allocation left. if (!okay) { - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) { alloc->debug_is_empty(nullptr); - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(alloc); } } @@ -168,7 +168,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global status is available only for pool-allocated configurations"); - auto alloc = Pool>::template iterate< + auto alloc = AllocPool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) { @@ -180,7 +180,7 @@ namespace snmalloc } count--; } - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(alloc); if (count != 0) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 0d9e747..357e897 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -58,9 +58,12 @@ namespace snmalloc template class LocalAllocator { - using CoreAlloc = CoreAllocator; + public: + using StateHandle = SharedStateHandle; private: + using CoreAlloc = CoreAllocator; + // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. @@ -375,7 +378,7 @@ namespace snmalloc // Initialise the global allocator structures ensure_init(); // Grab an allocator for this thread. - init(Pool::template acquire( + init(AllocPool::template acquire( &(this->local_cache))); } @@ -398,7 +401,7 @@ namespace snmalloc // Return underlying allocator to the system. if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) { - Pool::template release(core_alloc); + AllocPool::template release(core_alloc); } // Set up thread local allocator to look like diff --git a/src/mem/pool.h b/src/mem/pool.h index d3c25b4..49e3451 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -4,11 +4,12 @@ #include "../ds/mpmcstack.h" #include "../pal/pal_concept.h" #include "pooled.h" +#include "slaballocator.h" namespace snmalloc { /** - * Pool of a particular type of object. + * Pool of Allocators. * * This pool will never return objects to the OS. It maintains a list of all * objects ever allocated that can be iterated (not concurrency safe). Pooled @@ -20,8 +21,10 @@ namespace snmalloc template class PoolState { - template + template friend class Pool; + template + friend class AllocPool; private: std::atomic_flag lock = ATOMIC_FLAG_INIT; @@ -32,8 +35,85 @@ namespace snmalloc constexpr PoolState() = default; }; - template + template class Pool + { + PoolState state; + + public: + static Pool* make() noexcept + { + return ChunkAllocator::alloc_meta_data(nullptr); + } + + template + T* acquire(Args&&... args) + { + T* p = state.stack.pop(); + + if (p != nullptr) + { + p->set_in_use(); + return p; + } + + p = ChunkAllocator::alloc_meta_data( + nullptr, std::forward(args)...); + + FlagLock f(state.lock); + p->list_next = state.list; + state.list = p; + + p->set_in_use(); + return p; + } + + /** + * Return to the pool an object previously retrieved by `acquire` + * + * Do not return objects from `extract`. + */ + void release(T* p) + { + // The object's destructor is not run. If the object is "reallocated", it + // is returned without the constructor being run, so the object is reused + // without re-initialisation. + p->reset_in_use(); + state.stack.push(p); + } + + T* extract(T* p = nullptr) + { + // Returns a linked list of all objects in the stack, emptying the stack. + if (p == nullptr) + return state.stack.pop_all(); + + return p->next; + } + + /** + * Return to the pool a list of object previously retrieved by `extract` + * + * Do not return objects from `acquire`. + */ + void restore(T* first, T* last) + { + // Pushes a linked list of objects onto the stack. Use to put a linked + // list returned by extract back onto the stack. + state.stack.push(first, last); + } + + T* iterate(T* p = nullptr) + { + if (p == nullptr) + return state.list; + + return p->list_next; + } + }; + + template + class AllocPool { public: template diff --git a/src/mem/pooled.h b/src/mem/pooled.h index 93daad9..b685357 100644 --- a/src/mem/pooled.h +++ b/src/mem/pooled.h @@ -8,8 +8,10 @@ namespace snmalloc class Pooled { private: - template + template friend class Pool; + template + friend class AllocPool; template friend class MPMCStack;