#pragma once #include "../backend/backend_concept.h" #include "../ds/mpmcstack.h" #include "../ds/spmcstack.h" #include "../mem/metaslab.h" #include "../mem/sizeclasstable.h" #include "../pal/pal_ds.h" #ifdef SNMALLOC_TRACING # include #endif #include namespace snmalloc { /** * Used to store slabs in the unused sizes. */ struct ChunkRecord { MetaCommon meta_common; std::atomic next; }; static_assert(std::is_standard_layout_v); static_assert( offsetof(ChunkRecord, meta_common) == 0, "ChunkRecord and Metaslab must share a common prefix"); /** * How many slab sizes that can be provided. */ constexpr size_t NUM_SLAB_SIZES = Pal::address_bits - MIN_CHUNK_BITS; /** * Used to ensure the per slab meta data is large enough for both use cases. */ static_assert( sizeof(Metaslab) >= sizeof(ChunkRecord), "We conflate these two types."); /** * Number of free stacks per chunk size that each allocator will use. * For performance ideally a power of 2. We will return to the central * pool anything that has not be used in the last NUM_EPOCHS - 1, where * each epoch is separated by DecayMemoryTimerObject::PERIOD. * I.e. if period is 500ms and num of epochs is 4, then we will return to * the central pool anything not used for the last 1500-2000ms. */ constexpr size_t NUM_EPOCHS = 4; static_assert(bits::is_pow2(NUM_EPOCHS), "Code assumes power of two."); class ChunkAllocatorLocalState { friend class ChunkAllocator; /** * Stack of slabs that have been returned for reuse. */ ModArray>> chunk_stack; /** * Used for list of all ChunkAllocatorLocalStates. */ std::atomic next{nullptr}; }; /** * This is the global state required for the chunk allocator. * It must be provided as a part of the shared state handle * to the chunk allocator. */ class ChunkAllocatorState { friend class ChunkAllocator; /** * Stack of slabs that have been returned for reuse. */ ModArray> decommitted_chunk_stack; /** * Which is the current epoch to place dealloced chunks, and the * first place we look for allocating chunks. */ alignas(CACHELINE_SIZE) std::atomic epoch{0}; /** * All memory issued by this address space manager */ std::atomic peak_memory_usage_{0}; std::atomic memory_in_stacks{0}; std::atomic all_local{nullptr}; // Flag to ensure one-shot registration with the PAL for notifications. std::atomic_flag register_decay{}; public: size_t unused_memory() { return memory_in_stacks; } size_t peak_memory_usage() { return peak_memory_usage_; } void add_peak_memory_usage(size_t size) { peak_memory_usage_ += size; #ifdef SNMALLOC_TRACING std::cout << "peak_memory_usage_: " << peak_memory_usage_ << std::endl; #endif } }; class ChunkAllocator { template class DecayMemoryTimerObject : public PalTimerObject { ChunkAllocatorState* state; /*** * Method for callback object to perform lazy decommit. */ static void process(PalTimerObject* p) { // Unsafe downcast here. Don't want vtable and RTTI. auto self = reinterpret_cast(p); ChunkAllocator::handle_decay_tick(self->state); } // Specify that we notify the ChunkAllocator every 500ms. static constexpr size_t PERIOD = 500; public: DecayMemoryTimerObject(ChunkAllocatorState* state) : PalTimerObject(&process, PERIOD), state(state) {} }; template static void handle_decay_tick(ChunkAllocatorState* state) { auto new_epoch = (state->epoch + 1) % NUM_EPOCHS; // Flush old index for all threads. ChunkAllocatorLocalState* curr = state->all_local; while (curr != nullptr) { for (size_t sc = 0; sc < NUM_SLAB_SIZES; sc++) { auto& old_stack = curr->chunk_stack[sc][new_epoch]; ChunkRecord* record = old_stack.pop_all(); while (record != nullptr) { auto next = record->next.load(); // Disable pages for this Pal::notify_not_using( record->meta_common.chunk.unsafe_ptr(), slab_sizeclass_to_size(sc)); // Add to global state state->decommitted_chunk_stack[sc].push(record); record = next; } } curr = curr->next; } // Advance current index state->epoch = new_epoch; } public: template static std::pair, Metaslab*> alloc_chunk( typename SharedStateHandle::LocalState& local_state, ChunkAllocatorLocalState& chunk_alloc_local_state, sizeclass_t sizeclass, chunksizeclass_t slab_sizeclass, size_t slab_size, RemoteAllocator* remote) { using PAL = typename SharedStateHandle::Pal; ChunkAllocatorState& state = SharedStateHandle::get_chunk_allocator_state(&local_state); if (slab_sizeclass >= NUM_SLAB_SIZES) { // Your address space is not big enough for this allocation! errno = ENOMEM; return {nullptr, nullptr}; } ChunkRecord* chunk_record = nullptr; if constexpr (pal_supports) { // Try local cache of chunks first for (size_t e = 0; e < NUM_EPOCHS && chunk_record == nullptr; e++) { chunk_record = chunk_alloc_local_state .chunk_stack[slab_sizeclass][(state.epoch - e) % NUM_EPOCHS] .pop(); } } // Try global cache. if (chunk_record == nullptr) { chunk_record = state.decommitted_chunk_stack[slab_sizeclass].pop(); if (chunk_record != nullptr) { PAL::template notify_using( chunk_record->meta_common.chunk.unsafe_ptr(), slab_size); } } if (chunk_record != nullptr) { auto slab = chunk_record->meta_common.chunk; state.memory_in_stacks -= slab_size; auto meta = reinterpret_cast(chunk_record); #ifdef SNMALLOC_TRACING std::cout << "Reuse slab:" << slab.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " << slab_size << " memory in stacks " << state.memory_in_stacks << std::endl; #endif MetaEntry entry{meta, remote, sizeclass}; SharedStateHandle::Pagemap::set_metaentry( address_cast(slab), slab_size, entry); return {slab, meta}; } // Allocate a fresh slab as there are no available ones. // First create meta-data auto [slab, meta] = SharedStateHandle::alloc_chunk( &local_state, slab_size, remote, sizeclass); #ifdef SNMALLOC_TRACING std::cout << "Create slab:" << slab.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " << slab_size << std::endl; #endif state.add_peak_memory_usage(slab_size); state.add_peak_memory_usage(sizeof(Metaslab)); // TODO handle bounded versus lazy pagemaps in stats state.add_peak_memory_usage( (slab_size / MIN_CHUNK_SIZE) * sizeof(MetaEntry)); return {slab, meta}; } template SNMALLOC_SLOW_PATH static void dealloc( typename SharedStateHandle::LocalState& local_state, ChunkAllocatorLocalState& chunk_alloc_local_state, ChunkRecord* p, size_t slab_sizeclass) { ChunkAllocatorState& state = SharedStateHandle::get_chunk_allocator_state(&local_state); if constexpr (pal_supports) { // If we have a time source use decay based local cache. #ifdef SNMALLOC_TRACING std::cout << "Return slab:" << p->meta_common.chunk.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " << slab_sizeclass_to_size(slab_sizeclass) << " memory in stacks " << state.memory_in_stacks << std::endl; #endif chunk_alloc_local_state.chunk_stack[slab_sizeclass][state.epoch].push( p); } else { // No time source share immediately with global state. // Disable pages for this chunk. SharedStateHandle::Pal::notify_not_using( p->meta_common.chunk.unsafe_ptr(), slab_sizeclass_to_size(slab_sizeclass)); // Add to global state state.decommitted_chunk_stack[slab_sizeclass].push(p); } state.memory_in_stacks += slab_sizeclass_to_size(slab_sizeclass); } /** * Provide a block of meta-data with size and align. * * Backend allocator may use guard pages and separate area of * address space to protect this from corruption. */ template< typename U, SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle, typename... Args> static U* alloc_meta_data( typename SharedStateHandle::LocalState* local_state, Args&&... args) { // Cache line align size_t size = bits::align_up(sizeof(U), 64); capptr::Chunk p = SharedStateHandle::template alloc_meta_data(local_state, size); if (p == nullptr) return nullptr; return new (p.unsafe_ptr()) U(std::forward(args)...); } template static void register_local_state( typename SharedStateHandle::LocalState& local_state, ChunkAllocatorLocalState& chunk_alloc_local_state) { if constexpr (pal_supports) { ChunkAllocatorState& state = SharedStateHandle::get_chunk_allocator_state(&local_state); // Register with the Pal to receive notifications. if (!state.register_decay.test_and_set()) { auto timer = alloc_meta_data< DecayMemoryTimerObject, SharedStateHandle>(&local_state, &state); if (timer != nullptr) { SharedStateHandle::Pal::register_timer(timer); } else { // We failed to register the notification. // This is not catarophic, but if we can't allocate this // state something else will fail shortly. state.register_decay.clear(); } } // Add to the list of local states. auto* head = state.all_local.load(); do { chunk_alloc_local_state.next = head; } while (!state.all_local.compare_exchange_strong( head, &chunk_alloc_local_state)); } else { UNUSED(local_state); UNUSED(chunk_alloc_local_state); } } }; } // namespace snmalloc