#pragma once #include "../ds/helpers.h" #include "allocslab.h" #include "metaslab.h" #include namespace snmalloc { class Superslab : public Allocslab { // This is the view of a 16 mb superslab when it is being used to allocate // 64 kb slabs. private: friend DLList; // Keep the allocator pointer on a separate cache line. It is read by // other threads, and does not change, so we avoid false sharing. alignas(CACHELINE_SIZE) // The superslab is kept on a doubly linked list of superslabs which // have some space. Superslab* next; Superslab* prev; // This is a reference to the first unused slab in the free slab list // It is does not contain the short slab, which is handled using a bit // in the "used" field below. The list is terminated by pointing to // the short slab. // The head linked list has an absolute pointer for head, but the next // pointers stores in the metaslabs are relative pointers, that is they // are the relative offset to the next entry minus 1. This means that // all zeros is a list that chains through all the blocks, so the zero // initialised memory requires no more work. Mod head; // Represents twice the number of full size slabs used // plus 1 for the short slab. i.e. using 3 slabs and the // short slab would be 6 + 1 = 7 uint16_t used; ModArray meta; // Used size_t as results in better code in MSVC size_t slab_to_index(Slab* slab) { auto res = ((address_cast(slab) - address_cast(this)) >> SLAB_BITS); assert(res == (uint8_t)res); return res; } public: enum Status { Full, Available, OnlyShortSlabAvailable, Empty }; enum Action { NoSlabReturn = 0, NoStatusChange = 1, StatusChange = 2 }; static Superslab* get(void* p) { return pointer_cast(address_cast(p) & SUPERSLAB_MASK); } static bool is_short_sizeclass(sizeclass_t sizeclass) { constexpr sizeclass_t h = size_to_sizeclass_const(sizeof(Superslab)); return sizeclass <= h; } void init(RemoteAllocator* alloc) { allocator = alloc; if (kind != Super) { // If this wasn't previously a Superslab, we need to set up the // header. kind = Super; // Point head at the first non-short slab. head = 1; if (kind != Fresh) { // If this wasn't previously Fresh, we need to zero some things. used = 0; for (size_t i = 0; i < SLAB_COUNT; i++) { new (&(meta[i])) Metaslab(); } } #ifndef NDEBUG auto curr = head; for (size_t i = 0; i < SLAB_COUNT - used - 1; i++) { curr = (curr + meta[curr].next + 1) & (SLAB_COUNT - 1); } assert(curr == 0); for (size_t i = 0; i < SLAB_COUNT; i++) { assert(meta[i].is_unused()); } #endif } } bool is_empty() { return used == 0; } bool is_full() { return (used == (((SLAB_COUNT - 1) << 1) + 1)); } bool is_almost_full() { return (used >= ((SLAB_COUNT - 1) << 1)); } Status get_status() { if (!is_almost_full()) { if (!is_empty()) { return Available; } return Empty; } if (!is_full()) { return OnlyShortSlabAvailable; } return Full; } Metaslab& get_meta(Slab* slab) { return meta[slab_to_index(slab)]; } template Slab* alloc_short_slab(sizeclass_t sizeclass, MemoryProvider& memory_provider) { if ((used & 1) == 1) return alloc_slab(sizeclass, memory_provider); meta[0].allocated = 1; meta[0].head = 1; meta[0].sizeclass = static_cast(sizeclass); meta[0].link = get_initial_offset(sizeclass, true); { memory_provider.template notify_using( pointer_offset(this, OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE); } used++; return (Slab*)this; } template Slab* alloc_slab(sizeclass_t sizeclass, MemoryProvider& memory_provider) { uint8_t h = head; Slab* slab = pointer_cast( address_cast(this) + (static_cast(h) << SLAB_BITS)); uint8_t n = meta[h].next; meta[h].head = 1; meta[h].allocated = 1; meta[h].sizeclass = static_cast(sizeclass); meta[h].link = get_initial_offset(sizeclass, false); head = h + n + 1; used += 2; if constexpr (decommit_strategy == DecommitAll) { memory_provider.template notify_using(slab, SLAB_SIZE); } return slab; } // Returns true, if this alters the value of get_status template Action dealloc_slab(Slab* slab, MemoryProvider& memory_provider) { // This is not the short slab. uint8_t index = static_cast(slab_to_index(slab)); uint8_t n = head - index - 1; meta[index].sizeclass = 0; meta[index].next = n; head = index; bool was_almost_full = is_almost_full(); used -= 2; if constexpr (decommit_strategy == DecommitAll) memory_provider.notify_not_using(slab, SLAB_SIZE); assert(meta[index].is_unused()); if (was_almost_full || is_empty()) return StatusChange; return NoStatusChange; } // Returns true, if this alters the value of get_status template Action dealloc_short_slab(MemoryProvider& memory_provider) { // This is the short slab. if constexpr (decommit_strategy == DecommitAll) { memory_provider.notify_not_using( pointer_offset(this, OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE); } bool was_full = is_full(); used--; assert(meta[0].is_unused()); if (was_full || is_empty()) return StatusChange; return NoStatusChange; } }; } // namespace snmalloc