#pragma once #include "../ds/flaglock.h" #include "../ds/helpers.h" #include "../ds/mpmcstack.h" #include "../pal/pal.h" #include "allocstats.h" #include "baseslab.h" #include "sizeclass.h" #include namespace snmalloc { class Largeslab : public Baseslab { // This is the view of a contiguous memory area when it is being kept // in the global size-classed caches of available contiguous memory areas. private: template friend class MPMCStack; std::atomic next; public: void init() { kind = Large; } }; // This represents the state that the large allcoator needs to add to the // global state of the allocator. This is currently stored in the memory // provider, so we add this in. template class MemoryProviderStateMixin : public MemoryProviderState { std::atomic_flag lock = ATOMIC_FLAG_INIT; size_t bump; size_t remaining; std::pair reserve_block() noexcept { size_t size = SUPERSLAB_SIZE; void* r = ((MemoryProviderState*)this) ->template reserve(&size, SUPERSLAB_SIZE); if (size < SUPERSLAB_SIZE) error("out of memory"); ((MemoryProviderState*)this) ->template notify_using(r, OS_PAGE_SIZE); return std::make_pair(r, size); } void new_block() { auto r_size = reserve_block(); bump = (size_t)r_size.first; remaining = r_size.second; } public: /** * Stack of large allocations that have been returned for reuse. */ ModArray> large_stack; /** * Primitive allocator for structure that are required before * the allocator can be running. ***/ template void* alloc_chunk(size_t size) { // Cache line align size = bits::align_up(size, 64); void* p; { FlagLock f(lock); auto aligned_bump = bits::align_up(bump, alignment); if ((aligned_bump - bump) > remaining) { new_block(); } else { remaining -= aligned_bump - bump; bump = aligned_bump; } if (remaining < size) { new_block(); } p = (void*)bump; bump += size; remaining -= size; } auto page_start = bits::align_down((size_t)p, OS_PAGE_SIZE); auto page_end = bits::align_up((size_t)p + size, OS_PAGE_SIZE); ((MemoryProviderState*)this) ->template notify_using( (void*)page_start, page_end - page_start); return p; } }; using Stats = AllocStats; enum AllowReserve { NoReserve, YesReserve }; template class LargeAlloc { void* reserved_start = nullptr; void* reserved_end = nullptr; public: // This will be a zero-size structure if stats are not enabled. Stats stats; MemoryProvider& memory_provider; LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {} template bool reserve_memory(size_t need, size_t add) { if (((size_t)reserved_start + need) > (size_t)reserved_end) { if (allow_reserve == YesReserve) { stats.segment_create(); reserved_start = memory_provider.template reserve(&add, SUPERSLAB_SIZE); reserved_end = (void*)((size_t)reserved_start + add); reserved_start = (void*)bits::align_up((size_t)reserved_start, SUPERSLAB_SIZE); if (add < need) return false; } else { return false; } } return true; } template void* alloc(size_t large_class, size_t size) { size_t rsize = ((size_t)1 << SUPERSLAB_BITS) << large_class; if (size == 0) size = rsize; void* p = memory_provider.large_stack[large_class].pop(); if (p == nullptr) { assert(reserved_start <= reserved_end); size_t add; if ((rsize + SUPERSLAB_SIZE) < RESERVE_SIZE) add = RESERVE_SIZE; else add = rsize + SUPERSLAB_SIZE; if (!reserve_memory(rsize, add)) return nullptr; p = (void*)reserved_start; reserved_start = (void*)((size_t)p + rsize); // All memory is zeroed since it comes from reserved space. memory_provider.template notify_using(p, size); } else { if ((decommit_strategy != DecommitNone) || (large_class > 0)) { // Only the first page needs to be zeroed, as this was decommitted. if (zero_mem == YesZero) memory_provider.template zero(p, OS_PAGE_SIZE); memory_provider.template notify_using( (void*)((size_t)p + OS_PAGE_SIZE), size - OS_PAGE_SIZE); } else { // This is a superslab that has not been decommitted. if (zero_mem == YesZero) memory_provider.template zero(p, size); } } return p; } void dealloc(void* p, size_t large_class) { memory_provider.large_stack[large_class].push((Largeslab*)p); } }; using GlobalVirtual = MemoryProviderStateMixin; /** * The memory provider that will be used if no other provider is explicitly * passed as an argument. */ HEADER_GLOBAL GlobalVirtual default_memory_provider; }