diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 45937da..00690ef 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -68,8 +68,6 @@ namespace snmalloc // Used to isolate values on cache lines to prevent false sharing. static constexpr size_t CACHELINE_SIZE = 64; - static constexpr size_t PAGE_ALIGNED_SIZE = OS_PAGE_SIZE << INTERMEDIATE_BITS; - // Minimum allocation size is space for two pointers. static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*)); static constexpr size_t MIN_ALLOC_SIZE = 2 * sizeof(void*); diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index 03b95b2..cdfe8c7 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -3,6 +3,7 @@ #include "../ds/mpmcstack.h" #include "../mem/metaslab.h" #include "../mem/sizeclasstable.h" +#include "../pal/pal_ds.h" #ifdef SNMALLOC_TRACING # include @@ -36,6 +37,35 @@ namespace snmalloc static_assert( sizeof(Metaslab) >= sizeof(ChunkRecord), "We conflate these two types."); + /** + * Number of free stacks per chunk size that each allocator will use. + * For performance ideally a power of 2. We will return to the central + * pool anything that has not be used in the last NUM_EPOCHS - 1, where + * each epoch is separated by DecayMemoryTimerObject::PERIOD. + * I.e. if period is 500ms and num of epochs is 4, then we will return to + * the central pool anything not used for the last 1500-2000ms. + */ + constexpr size_t NUM_EPOCHS = 4; + static_assert(bits::is_pow2(NUM_EPOCHS), "Code assumes power of two."); + + class ChunkAllocatorLocalState + { + friend class ChunkAllocator; + + /** + * Stack of slabs that have been returned for reuse. + */ + ModArray< + NUM_SLAB_SIZES, + ModArray>> + chunk_stack; + + /** + * Used for list of all ChunkAllocatorLocalStates. + */ + std::atomic next{nullptr}; + }; + /** * This is the global state required for the chunk allocator. * It must be provided as a part of the shared state handle @@ -47,7 +77,8 @@ namespace snmalloc /** * Stack of slabs that have been returned for reuse. */ - ModArray> chunk_stack; + ModArray> + decommitted_chunk_stack; /** * All memory issued by this address space manager @@ -56,6 +87,17 @@ namespace snmalloc std::atomic memory_in_stacks{0}; + std::atomic all_local{nullptr}; + + /** + * Which is the current epoch to place dealloced chunks, and the + * first place we look for allocating chunks. + */ + std::atomic epoch{0}; + + // Flag to ensure one-shot registration with the PAL for notifications. + std::atomic_flag register_decay{}; + public: size_t unused_memory() { @@ -78,10 +120,68 @@ namespace snmalloc class ChunkAllocator { + template + class DecayMemoryTimerObject : public PalTimerObject + { + ChunkAllocatorState* state; + + /*** + * Method for callback object to perform lazy decommit. + */ + static void process(PalTimerObject* p) + { + // Unsafe downcast here. Don't want vtable and RTTI. + auto self = reinterpret_cast(p); + ChunkAllocator::handle_decay_tick(self->state); + } + + // Specify that we notify the ChunkAllocator every 500ms. + static constexpr size_t PERIOD = 500; + + public: + DecayMemoryTimerObject(ChunkAllocatorState* state) + : PalTimerObject(&process, PERIOD), state(state) + {} + }; + + static void handle_decay_tick(ChunkAllocatorState* state) + { + auto new_epoch = (state->epoch + 1) % NUM_EPOCHS; + // Flush old index for all threads. + ChunkAllocatorLocalState* curr = state->all_local; + while (curr != nullptr) + { + for (size_t sc = 0; sc < NUM_SLAB_SIZES; sc++) + { + auto& old_stack = curr->chunk_stack[sc][new_epoch]; + ChunkRecord* record = old_stack.pop_all(); + while (record != nullptr) + { + auto next = record->next.load(); + + // Disable pages for this + Pal::notify_not_using( + record->meta_common.chunk.unsafe_ptr(), + slab_sizeclass_to_size(sc)); + + // Add to global state + state->decommitted_chunk_stack[sc].push(record); + + record = next; + } + } + curr = curr->next; + } + + // Advance current index + state->epoch = new_epoch; + } + public: template static std::pair, Metaslab*> alloc_chunk( typename SharedStateHandle::LocalState& local_state, + ChunkAllocatorLocalState& chunk_alloc_local_state, sizeclass_t sizeclass, sizeclass_t slab_sizeclass, // TODO sizeclass_t size_t slab_size, @@ -96,8 +196,24 @@ namespace snmalloc return {nullptr, nullptr}; } - // Pop a slab - auto chunk_record = state.chunk_stack[slab_sizeclass].pop(); + ChunkRecord* chunk_record = nullptr; + // Try local cache of chunks first + for (size_t e = 0; e < NUM_EPOCHS && chunk_record == nullptr; e++) + { + chunk_record = + chunk_alloc_local_state + .chunk_stack[slab_sizeclass][(state.epoch - e) % NUM_EPOCHS] + .pop(); + } + + // Try global cache. + if (chunk_record == nullptr) + { + chunk_record = state.decommitted_chunk_stack[slab_sizeclass].pop(); + if (chunk_record != nullptr) + Pal::notify_using( + chunk_record->meta_common.chunk.unsafe_ptr(), slab_size); + } if (chunk_record != nullptr) { @@ -137,17 +253,21 @@ namespace snmalloc template SNMALLOC_SLOW_PATH static void dealloc( typename SharedStateHandle::LocalState& local_state, + ChunkAllocatorLocalState& chunk_alloc_local_state, ChunkRecord* p, size_t slab_sizeclass) { - auto& state = SharedStateHandle::get_chunk_allocator_state(&local_state); + ChunkAllocatorState& state = + SharedStateHandle::get_chunk_allocator_state(&local_state); #ifdef SNMALLOC_TRACING std::cout << "Return slab:" << p->meta_common.chunk.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " << slab_sizeclass_to_size(slab_sizeclass) << " memory in stacks " << state.memory_in_stacks << std::endl; #endif - state.chunk_stack[slab_sizeclass].push(p); + + chunk_alloc_local_state.chunk_stack[slab_sizeclass][state.epoch].push(p); + state.memory_in_stacks += slab_sizeclass_to_size(slab_sizeclass); } @@ -175,5 +295,41 @@ namespace snmalloc return new (p.unsafe_ptr()) U(std::forward(args)...); } + + template + static void register_local_state( + typename SharedStateHandle::LocalState& local_state, + ChunkAllocatorLocalState& chunk_alloc_local_state) + { + ChunkAllocatorState& state = + SharedStateHandle::get_chunk_allocator_state(&local_state); + + // Register with the Pal to receive notifications. + if (!state.register_decay.test_and_set()) + { + auto timer = alloc_meta_data< + DecayMemoryTimerObject, + SharedStateHandle>(&local_state, &state); + if (timer != nullptr) + { + SharedStateHandle::Pal::register_timer(timer); + } + else + { + // We failed to register the notification. + // This is not catarophic, but if we can't allocate this + // state something else will fail shortly. + state.register_decay.clear(); + } + } + + // Add to the list of local states. + auto* head = state.all_local.load(); + do + { + chunk_alloc_local_state.next = head; + } while (!state.all_local.compare_exchange_strong( + head, &chunk_alloc_local_state)); + } }; } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index ed97a6b..6443566 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -58,6 +58,11 @@ namespace snmalloc */ MetaslabCache alloc_classes[NUM_SIZECLASSES]; + /** + * Local cache for the Chunk allocator. + */ + ChunkAllocatorLocalState chunk_local_state; + /** * Local entropy source and current version of keys for * this thread @@ -376,6 +381,7 @@ namespace snmalloc auto chunk_record = clear_slab(meta, sizeclass); ChunkAllocator::dealloc( get_backend_local_state(), + chunk_local_state, chunk_record, sizeclass_to_slab_sizeclass(sizeclass)); @@ -548,6 +554,9 @@ namespace snmalloc message_queue().invariant(); } + ChunkAllocator::register_local_state( + get_backend_local_state(), chunk_local_state); + #ifndef NDEBUG for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) { @@ -746,6 +755,7 @@ namespace snmalloc auto [slab, meta] = snmalloc::ChunkAllocator::alloc_chunk( get_backend_local_state(), + chunk_local_state, sizeclass, slab_sizeclass, slab_size, diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 43423af..fa9d2c7 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -182,6 +182,7 @@ namespace snmalloc // Set remote as large allocator remote. auto [chunk, meta] = ChunkAllocator::alloc_chunk( core_alloc->get_backend_local_state(), + core_alloc->chunk_local_state, bits::next_pow2_bits(size), // TODO large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), @@ -547,6 +548,7 @@ namespace snmalloc size_t slab_sizeclass) { ChunkAllocator::dealloc( core_alloc->get_backend_local_state(), + core_alloc->chunk_local_state, slab_record, slab_sizeclass); return nullptr; diff --git a/src/pal/pal_timer_default.h b/src/pal/pal_timer_default.h index a38094a..c7761ef 100644 --- a/src/pal/pal_timer_default.h +++ b/src/pal/pal_timer_default.h @@ -29,4 +29,4 @@ namespace snmalloc timers.register_timer(timer); } }; -} \ No newline at end of file +} // namespace snmalloc