Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Aggressively optimise fast path for allocation

Browse Source 
This change introduces a per small sizeclass free list.  That can be
used to access the free objects for that sizeclass with minimal
calculations being required.

It changes to a partial bump ptr.  We bump allocate a whole OS
page worth of objects at a go, so we don't switch as frequently
between bump and free list allocation.

The code for the fast paths has been restructured to minimise the
work required on the common case, and also it is all inlined for the
common case.

Allocating a zero sized object is moved off the fast path.  Ask for 1
byte if you want to be fast.
This commit is contained in:
Matthew Parkinson
2019-06-28 11:50:17 +01:00
parent b0c1531221
commit 7a198cbda5
7 changed files with 299 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
src/mem/alloc.h
View File

@@ -226,6 +226,17 @@ namespace snmalloc
# define SNMALLOC_DEFAULT_PAGEMAP snmalloc::SuperslabMap<>
#endif
// This class is just used so that the free lists are the first entry
// in the allocator and hence has better code gen.
// It contains a free list per small size class. These are used for allocation
// on the fast path.
// This part of the code is inspired by mimalloc.
class FastFreeLists
{
protected:
FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES];
};
/**
* Allocator. This class is parameterised on three template parameters. The
* `MemoryProvider` defines the source of memory for this allocator.
@@ -247,7 +258,7 @@ namespace snmalloc
class PageMap = SNMALLOC_DEFAULT_PAGEMAP,
bool IsQueueInline = true>
class Allocator
: public Pooled<Allocator<MemoryProvider, PageMap, IsQueueInline>>
: public FastFreeLists, public Pooled<Allocator<MemoryProvider, PageMap, IsQueueInline>>
{
LargeAlloc<MemoryProvider> large_allocator;
PageMap page_map;
@@ -281,28 +292,27 @@ namespace snmalloc
stats().alloc_request(size);
handle_message_queue();
// Allocate memory of a statically known size.
if constexpr (sizeclass < NUM_SMALL_CLASSES)
{
constexpr size_t rsize = sizeclass_to_size(sizeclass);
return small_alloc<zero_mem, allow_reserve>(sizeclass, rsize);
return small_alloc<zero_mem, allow_reserve>(size);
}
else if constexpr (sizeclass < NUM_SIZECLASSES)
{
handle_message_queue();
constexpr size_t rsize = sizeclass_to_size(sizeclass);
return medium_alloc<zero_mem, allow_reserve>(sizeclass, rsize, size);
}
else
{
{
handle_message_queue();
return large_alloc<zero_mem, allow_reserve>(size);
}
#endif
}
template<ZeroMem zero_mem = NoZero, AllowReserve allow_reserve = YesReserve>
ALLOCATOR void* alloc(size_t size)
inline ALLOCATOR void* alloc(size_t size)
{
#ifdef USE_MALLOC
static_assert(
@@ -315,18 +325,30 @@ namespace snmalloc
#else
stats().alloc_request(size);
handle_message_queue();
sizeclass_t sizeclass = size_to_sizeclass(size);
// Allocate memory of a dynamically known size.
if (sizeclass < NUM_SMALL_CLASSES)
// Perform the - 1 on size, so that zero wraps around and ends up on
// slow path.
if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1)))
{
// Allocations smaller than the slab size are more likely. Improve
// branch prediction by placing this case first.
size_t rsize = sizeclass_to_size(sizeclass);
return small_alloc<zero_mem, allow_reserve>(sizeclass, rsize);
return small_alloc<zero_mem, allow_reserve>(size);
}
return alloc_not_small<zero_mem, allow_reserve>(size);
}
template<ZeroMem zero_mem = NoZero, AllowReserve allow_reserve = YesReserve>
NOINLINE ALLOCATOR void* alloc_not_small(size_t size)
{
handle_message_queue();
if (size == 0)
{
return small_alloc<zero_mem, allow_reserve>(1);
}
sizeclass_t sizeclass = size_to_sizeclass(size);
if (sizeclass < NUM_SIZECLASSES)
{
size_t rsize = sizeclass_to_size(sizeclass);
@@ -418,7 +440,7 @@ namespace snmalloc
#endif
}
void dealloc(void* p)
ALWAYSINLINE void dealloc(void* p)
{
#ifdef USE_MALLOC
return free(p);
@@ -429,14 +451,10 @@ namespace snmalloc
// pointer.
uint8_t size = pagemap().get(address_cast(p));
if (size == 0)
{
error("Not allocated by this allocator");
}
Superslab* super = Superslab::get(p);
if (size == PMSuperslab)
if (likely(size == PMSuperslab))
{
RemoteAllocator* target = super->get_allocator();
Slab* slab = Slab::get(p);
@@ -447,12 +465,17 @@ namespace snmalloc
// pointer.
sizeclass_t sizeclass = meta.sizeclass;
if (super->get_allocator() == public_state())
if (likely(super->get_allocator() == public_state()))
small_dealloc(super, p, sizeclass);
else
remote_dealloc(target, p, sizeclass);
return;
}
dealloc_not_small(p, size);
}
NOINLINE void dealloc_not_small(void* p, uint8_t size)
{
if (size == PMMediumslab)
{
Mediumslab* slab = Mediumslab::get(p);
@@ -469,7 +492,13 @@ namespace snmalloc
return;
}
if (size == 0)
{
error("Not allocated by this allocator");
}
# ifdef CHECK_CLIENT
Superslab* super = Superslab::get(p);
if (size > 64 || address_cast(super) != address_cast(p))
{
error("Not deallocating start of an object");
@@ -821,11 +850,11 @@ namespace snmalloc
if (p->target_id() != super->get_allocator()->id())
error("Detected memory corruption. Potential use-after-free");
#endif
if (super->get_kind() == Super)
if (likely(super->get_kind() == Super))
{
Slab* slab = Slab::get(p);
Metaslab& meta = super->get_meta(slab);
if (p->target_id() == id())
if (likely(p->target_id() == id()))
{
small_dealloc_offseted(super, p, meta.sizeclass);
}
@@ -978,7 +1007,7 @@ namespace snmalloc
}
template<ZeroMem zero_mem, AllowReserve allow_reserve>
void* small_alloc(sizeclass_t sizeclass, size_t rsize)
inline void* small_alloc(size_t size)
{
MEASURE_TIME_MARKERS(
small_alloc,
@@ -988,14 +1017,42 @@ namespace snmalloc
zero_mem == YesZero ? "zeromem" : "nozeromem",
allow_reserve == NoReserve ? "noreserve" : "reserve"));
sizeclass_t sizeclass = size_to_sizeclass(size);
stats().sizeclass_alloc(sizeclass);
SlabList* sc = &small_classes[sizeclass];
auto& fl = small_fast_free_lists[sizeclass];
auto head = fl.value;
if (likely((reinterpret_cast<size_t>(head) & 1) == 0))
{
void * p = head;
// Read the next slot from the memory that's about to be allocated.
fl.value = Metaslab::follow_next(p);
if constexpr (zero_mem == YesZero)
{
large_allocator.memory_provider.zero(p, size);
}
return p;
}
return small_alloc_slow<zero_mem, allow_reserve>(size);
}
template<ZeroMem zero_mem, AllowReserve allow_reserve>
NOINLINE
void* small_alloc_slow(size_t size)
{
handle_message_queue();
sizeclass_t sizeclass = size_to_sizeclass(size);
size_t rsize = sizeclass_to_size(sizeclass);
auto& sl = small_classes[sizeclass];
Slab* slab;
if (!sc->is_empty())
if (!sl.is_empty())
{
SlabLink* link = sc->get_head();
SlabLink* link = sl.get_head();
slab = link->get_slab();
}
else
@@ -1005,13 +1062,13 @@ namespace snmalloc
if ((allow_reserve == NoReserve) && (slab == nullptr))
return nullptr;
sc->insert(slab->get_link());
sl.insert(slab->get_link());
}
return slab->alloc<zero_mem>(sc, rsize, large_allocator.memory_provider);
auto& ffl = small_fast_free_lists[sizeclass];
return slab->alloc<zero_mem>(sl, ffl, rsize, large_allocator.memory_provider);
}
void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass)
ALWAYSINLINE void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass)
{
#ifdef CHECK_CLIENT
Slab* slab = Slab::get(p);
@@ -1025,20 +1082,27 @@ namespace snmalloc
small_dealloc_offseted(super, offseted, sizeclass);
}
void
small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass)
ALWAYSINLINE void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass)
{
MEASURE_TIME(small_dealloc, 4, 16);
stats().sizeclass_dealloc(sizeclass);
bool was_full = super->is_full();
SlabList* sc = &small_classes[sizeclass];
Slab* slab = Slab::get(p);
Superslab::Action a =
slab->dealloc(sc, super, p, large_allocator.memory_provider);
if (a == Superslab::NoSlabReturn)
if (likely(slab->dealloc_fast(super, p)))
return;
small_dealloc_offseted_slow(super, p, sizeclass);
}
NOINLINE void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass)
{
bool was_full = super->is_full();
SlabList* sl = &small_classes[sizeclass];
Slab* slab = Slab::get(p);
Superslab::Action a =
slab->dealloc_slow(sl, super, p, large_allocator.memory_provider);
if (likely(a == Superslab::NoSlabReturn))
return;
stats().sizeclass_dealloc_slab(sizeclass);
if (a == Superslab::NoStatusChange)

108
src/mem/metaslab.h
View File

@@ -29,22 +29,24 @@ namespace snmalloc
// This can be either a short or a standard slab.
class Metaslab
{
private:
// How many entries are used in this slab.
public:
// How many entries are not in the free list of slab.
uint16_t used = 0;
public:
// Bump free list of unused entries in this sizeclass.
// If the bottom bit is 1, then this represents a bump_ptr
// of where we have allocated up to in this slab. Otherwise,
// it represents the location of the first block in the free
// list. The free list is chained through deallocated blocks.
// It is terminated with a bump ptr.
//
// Note that, the first entry in a slab is never bump allocated
// but is used for the link. This means that 1 represents the fully
// bump allocated slab.
// How many entries have been allocated from this slab.
uint16_t allocated;
// Index of first entry in this slab that forms the free
// list. The list entries are stored as the first pointer
// in each unused object. The terminator is a pointer or
// offset into the block with the bottom bit set. This means
// I.e.
// * an empty list has a head of 1.
// * a one element list has an head contains an offset to this
// this block, and then contains a pointer with the bottom
// bit set.
Mod<SLAB_SIZE, uint16_t> head;
// When a slab has free space it will be on the has space list for
// that size class. We use an empty block in this slab to be the
// doubly linked node into that size class's free list.
@@ -93,35 +95,38 @@ namespace snmalloc
/// Value used to check for corruptions in a block
static constexpr size_t POISON =
static_cast<size_t>(bits::is64() ? 0xDEADBEEFDEAD0000 : 0xDEAD0000);
static_cast<size_t>(bits::is64() ? 0xDEADBEEFDEADBEEF : 0xDEADBEEF);
/// Store next pointer in a block. In Debug using magic value to detect some
/// simple corruptions.
static void store_next(void* p, uint16_t head)
static void store_next(void* p, void* head)
{
#ifndef CHECK_CLIENT
*static_cast<size_t*>(p) = head;
*static_cast<void**>(p) = head;
#else
*static_cast<size_t*>(p) =
head ^ POISON ^ (static_cast<size_t>(head) << (bits::BITS - 16));
*static_cast<void**>(p) = head;
*(static_cast<uintptr_t*>(p) + 1) =
address_cast(head) ^ POISON;
#endif
}
/// Accessor function for the next pointer in a block.
/// In Debug checks for simple corruptions.
static uint16_t follow_next(void* node)
static void* follow_next(void* node)
{
size_t next = *static_cast<size_t*>(node);
#ifdef CHECK_CLIENT
if (((next ^ POISON) ^ (next << (bits::BITS - 16))) > 0xFFFF)
uintptr_t next = *static_cast<uintptr_t*>(node);
uintptr_t chk = *(static_cast<uintptr_t*>(node) + 1);
if ((next ^ chk) != POISON)
error("Detected memory corruption. Use-after-free.");
#endif
return static_cast<uint16_t>(next);
return *static_cast<void**>(node);
}
bool valid_head(bool is_short)
{
size_t size = sizeclass_to_size(sizeclass);
size_t slab_start = get_initial_link(sizeclass, is_short);
size_t slab_start = get_initial_offset(sizeclass, is_short);
size_t all_high_bits = ~static_cast<size_t>(1);
size_t head_start =
@@ -138,18 +143,19 @@ namespace snmalloc
* We don't expect a cycle, so worst case is only followed by a crash, so
* slow doesn't mater.
**/
void debug_slab_acyclic_free_list(Slab* slab)
size_t debug_slab_acyclic_free_list(Slab* slab)
{
#ifndef NDEBUG
uint16_t curr = head;
uint16_t curr_slow = head;
size_t length = 0;
void* curr = pointer_offset(slab, head);
void* curr_slow = pointer_offset(slab, head);
bool both = false;
while ((curr & 1) != 1)
while ((reinterpret_cast<size_t>(curr) & 1) == 0)
{
curr = follow_next(pointer_offset(slab, curr));
curr = follow_next(curr);
if (both)
{
curr_slow = follow_next(pointer_offset(slab, curr_slow));
curr_slow = follow_next(curr_slow);
}
if (curr == curr_slow)
@@ -158,9 +164,12 @@ namespace snmalloc
}
both = !both;
length ++;
}
return length;
#else
UNUSED(slab);
return 0;
#endif
}
@@ -168,7 +177,10 @@ namespace snmalloc
{
#if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS)
size_t size = sizeclass_to_size(sizeclass);
size_t offset = get_initial_link(sizeclass, is_short);
size_t offset = get_initial_offset(sizeclass, is_short);
if (is_unused())
return;
size_t accounted_for = used * size + offset;
@@ -184,40 +196,40 @@ namespace snmalloc
// Block is not full
assert(SLAB_SIZE > accounted_for);
debug_slab_acyclic_free_list(slab);
size_t length = debug_slab_acyclic_free_list(slab);
UNUSED(length);
// Walk bump-free-list-segment accounting for unused space
uint16_t curr = head;
while ((curr & 1) != 1)
void* curr = pointer_offset(slab, head);
while ((reinterpret_cast<size_t>(curr) & 1) == 0)
{
// Check we are looking at a correctly aligned block
uint16_t start = remove_cache_friendly_offset(curr, sizeclass);
assert((start - offset) % size == 0);
void* start = curr;
assert(((address_cast(start) - address_cast(slab) - offset) % size) == 0);
// Account for free elements in free list
accounted_for += size;
assert(SLAB_SIZE >= accounted_for);
// We should never reach the link node in the free list.
assert(curr != link);
assert(curr != pointer_offset(slab, link));
// Iterate bump/free list segment
curr = follow_next(pointer_offset(slab, curr));
curr = follow_next(curr);
}
if (curr != 1)
auto bumpptr = (allocated * size) + offset;
// Check we haven't allocaated more than gits in a slab
assert(bumpptr <= SLAB_SIZE);
// Account for to be bump allocated space
accounted_for += SLAB_SIZE - bumpptr;
if (bumpptr != SLAB_SIZE)
{
// Check we terminated traversal on a correctly aligned block
uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass);
assert((start - offset) % size == 0);
// Account for to be bump allocated space
accounted_for += SLAB_SIZE - (curr - 1);
// The link should be the first allocation as we
// haven't completely filled this block at any point.
assert(link == get_initial_link(sizeclass, is_short));
assert(link == get_initial_offset(sizeclass, is_short));
}
assert(!is_full());
// Add the link node.
accounted_for += size;

3
src/mem/sizeclass.h
View File

@@ -8,8 +8,7 @@ namespace snmalloc
using sizeclass_t = size_t;
// using sizeclass_t = uint8_t;
constexpr static uint16_t get_initial_bumpptr(sizeclass_t sc, bool is_short);
constexpr static uint16_t get_initial_link(sizeclass_t sc, bool is_short);
constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short);
constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass);
constexpr static size_t sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass);
constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc);

41
src/mem/sizeclasstable.h
View File

@@ -23,10 +23,8 @@ namespace snmalloc
ModArray<NUM_SIZECLASSES, size_t> size;
ModArray<NUM_SIZECLASSES, size_t> cache_friendly_mask;
ModArray<NUM_SIZECLASSES, size_t> inverse_cache_friendly_mask;
ModArray<NUM_SMALL_CLASSES, uint16_t> bump_ptr_start;
ModArray<NUM_SMALL_CLASSES, uint16_t> short_bump_ptr_start;
ModArray<NUM_SMALL_CLASSES, uint16_t> initial_link_ptr;
ModArray<NUM_SMALL_CLASSES, uint16_t> short_initial_link_ptr;
ModArray<NUM_SMALL_CLASSES, uint16_t> initial_offset_ptr;
ModArray<NUM_SMALL_CLASSES, uint16_t> short_initial_offset_ptr;
ModArray<NUM_MEDIUM_CLASSES, uint16_t> medium_slab_slots;
@@ -35,10 +33,8 @@ namespace snmalloc
size(),
cache_friendly_mask(),
inverse_cache_friendly_mask(),
bump_ptr_start(),
short_bump_ptr_start(),
initial_link_ptr(),
short_initial_link_ptr(),
initial_offset_ptr(),
short_initial_offset_ptr(),
medium_slab_slots()
{
size_t curr = 1;
@@ -71,19 +67,9 @@ namespace snmalloc
size_t correction = SLAB_SIZE % size[i];
// First element in the block is the link
initial_link_ptr[i] = static_cast<uint16_t>(correction);
short_initial_link_ptr[i] =
initial_offset_ptr[i] = static_cast<uint16_t>(correction);
short_initial_offset_ptr[i] =
static_cast<uint16_t>(header_size + short_correction);
// Move to object after link.
auto short_after_link = short_initial_link_ptr[i] + size[i];
size_t after_link = initial_link_ptr[i] + size[i];
// Bump ptr has bottom bit set.
// In case we only have one object on this slab check for wrap around.
short_bump_ptr_start[i] =
static_cast<uint16_t>((short_after_link + 1) % SLAB_SIZE);
bump_ptr_start[i] = static_cast<uint16_t>((after_link + 1) % SLAB_SIZE);
}
for (sizeclass_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++)
@@ -97,21 +83,12 @@ namespace snmalloc
static constexpr SizeClassTable sizeclass_metadata = SizeClassTable();
static inline constexpr uint16_t
get_initial_bumpptr(sizeclass_t sc, bool is_short)
get_initial_offset(sizeclass_t sc, bool is_short)
{
if (is_short)
return sizeclass_metadata.short_bump_ptr_start[sc];
return sizeclass_metadata.short_initial_offset_ptr[sc];
return sizeclass_metadata.bump_ptr_start[sc];
}
static inline constexpr uint16_t
get_initial_link(sizeclass_t sc, bool is_short)
{
if (is_short)
return sizeclass_metadata.short_initial_link_ptr[sc];
return sizeclass_metadata.initial_link_ptr[sc];
return sizeclass_metadata.initial_offset_ptr[sc];
}
constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass)

182
src/mem/slab.h
View File

@@ -4,6 +4,12 @@
namespace snmalloc
{
struct FreeListHead
{
// Use a value with bottom bit set for empty list.
void* value = pointer_offset<void*>(nullptr, 1);
};
class Slab
{
private:
@@ -31,7 +37,7 @@ namespace snmalloc
}
template<ZeroMem zero_mem, typename MemoryProvider>
void* alloc(SlabList* sc, size_t rsize, MemoryProvider& memory_provider)
inline void* alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize, MemoryProvider& memory_provider)
{
// Read the head from the metadata stored in the superslab.
Metaslab& meta = get_meta();
@@ -39,38 +45,81 @@ namespace snmalloc
assert(rsize == sizeclass_to_size(meta.sizeclass));
meta.debug_slab_invariant(is_short(), this);
assert(sc->get_head() == (SlabLink*)((size_t)this + meta.link));
assert(sl.get_head() == (SlabLink*)((size_t)this + meta.link));
assert(!meta.is_full());
meta.add_use();
void* p;
if ((head & 1) == 0)
if (head == 1)
{
void* node = pointer_offset(this, head);
// Read the next slot from the memory that's about to be allocated.
meta.head = Metaslab::follow_next(node);
p = remove_cache_friendly_offset(node, meta.sizeclass);
}
else
{
if (meta.head == 1)
size_t bumpptr = get_initial_offset(meta.sizeclass, is_short());
bumpptr += meta.allocated * rsize;
if (bumpptr == SLAB_SIZE)
{
meta.add_use();
assert(meta.used == meta.allocated);
p = pointer_offset(this, meta.link);
sc->pop();
meta.set_full();
sl.pop();
goto finish1;
}
else
{
// This slab is being bump allocated.
p = pointer_offset(this, head - 1);
meta.head = (head + static_cast<uint16_t>(rsize)) & (SLAB_SIZE - 1);
void* curr = nullptr;
bool commit = false;
while (true)
{
size_t newbumpptr = bumpptr + rsize;
auto alignedbumpptr =
bits::align_up(bumpptr - 1, OS_PAGE_SIZE);
auto alignednewbumpptr = bits::align_up(newbumpptr, OS_PAGE_SIZE);
if (alignedbumpptr != alignednewbumpptr)
{
// We have committed once already.
if (commit) break;
memory_provider.template notify_using<NoZero>(pointer_offset(this, alignedbumpptr), alignednewbumpptr - alignedbumpptr);
commit = true;
}
if (curr == nullptr)
{
meta.head = static_cast<uint16_t>(bumpptr);
}
else
{
Metaslab::store_next(curr, pointer_offset(this, bumpptr));
}
curr = pointer_offset(this, bumpptr);
bumpptr = newbumpptr;
meta.allocated = meta.allocated + 1;
}
assert(curr != nullptr);
Metaslab::store_next(curr, pointer_offset<void*>(nullptr, 1));
}
}
{
p = pointer_offset(this, meta.head);
// Read the next slot from the memory that's about to be allocated.
void* next = Metaslab::follow_next(p);
// Put everything in allocators small_class free list.
meta.head = 1;
fast_free_list.value = next;
// Treat stealing the free list as allocating it all.
// Link is not in use, i.e. - 1 is required.
meta.used = meta.allocated - 1;
}
assert (is_start_of_object(Superslab::get(p), p));
finish1:
meta.debug_slab_invariant(is_short(), this);
if constexpr (zero_mem == YesZero)
@@ -92,38 +141,56 @@ namespace snmalloc
address_cast(this) + SLAB_SIZE - address_cast(p));
}
// Returns true, if it alters get_status.
template<typename MemoryProvider>
inline typename Superslab::Action dealloc(
SlabList* sc, Superslab* super, void* p, MemoryProvider& memory_provider)
// Returns true, if it deallocation can proceed without changing any status bits.
// Note that this does remove the use from the meta slab, so it doesn't need doing
// on the slow path.
ALWAYSINLINE bool dealloc_fast(Superslab* super, void* p)
{
Metaslab& meta = super->get_meta(this);
bool was_full = meta.is_full();
#ifdef CHECK_CLIENT
if (meta.is_unused())
error("Detected potential double free.");
#endif
meta.debug_slab_invariant(is_short(), this);
meta.sub_use();
bool was_full = meta.is_full();
if (unlikely(was_full)) return false;
bool is_unused = meta.is_unused();
if (unlikely(is_unused)) return false;
// Update the head and the next pointer in the free list.
uint16_t head = meta.head;
uint16_t current = pointer_to_index(p);
// Set the head to the memory being deallocated.
meta.head = current;
assert(meta.valid_head(is_short()));
// Set the next pointer to the previous head.
Metaslab::store_next(p, pointer_offset(this, head));
meta.debug_slab_invariant(is_short(), this);
return true;
}
// If dealloc fast returns false, then call this.
// This does not need to remove the "use" as done by the fast path.
// Returns a complex return code for managing the superslab meta data.
// i.e. This deallocation could make an entire superslab free.
template<typename MemoryProvider>
NOINLINE typename Superslab::Action dealloc_slow(
SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider)
{
Metaslab& meta = super->get_meta(this);
bool was_full = meta.is_full();
bool is_unused = meta.is_unused();
if (was_full)
{
// We are not on the sizeclass list.
if (!meta.is_unused())
{
// Update the head and the sizeclass link.
uint16_t index = pointer_to_index(p);
assert(meta.head == 1);
meta.link = index;
// Push on the list of slabs for this sizeclass.
sc->insert(meta.get_link(this));
meta.debug_slab_invariant(is_short(), this);
}
else
if (is_unused)
{
// Dealloc on the superslab.
if (is_short())
@@ -131,37 +198,30 @@ namespace snmalloc
return super->dealloc_slab(this, memory_provider);
}
// Update the head and the sizeclass link.
uint16_t index = pointer_to_index(p);
assert(meta.head == 1);
// assert(meta.fully_allocated(is_short()));
meta.link = index;
// Push on the list of slabs for this sizeclass.
sl->insert(meta.get_link(this));
meta.debug_slab_invariant(is_short(), this);
return Superslab::NoSlabReturn;
}
else if (meta.is_unused())
if (is_unused)
{
// Remove from the sizeclass list and dealloc on the superslab.
sc->remove(meta.get_link(this));
sl->remove(meta.get_link(this));
if (is_short())
return super->dealloc_short_slab(memory_provider);
return super->dealloc_slab(this, memory_provider);
}
else
{
#ifndef NDEBUG
sc->debug_check_contains(meta.get_link(this));
#endif
// Update the head and the next pointer in the free list.
uint16_t head = meta.head;
uint16_t current = pointer_to_index(p);
// Set the head to the memory being deallocated.
meta.head = current;
assert(meta.valid_head(is_short()));
// Set the next pointer to the previous head.
Metaslab::store_next(p, head);
meta.debug_slab_invariant(is_short(), this);
}
return Superslab::NoSlabReturn;
abort();
}
bool is_short()

11
src/mem/superslab.h
View File

@@ -160,11 +160,11 @@ namespace snmalloc
if ((used & 1) == 1)
return alloc_slab(sizeclass, memory_provider);
meta[0].head = get_initial_bumpptr(sizeclass, true);
meta[0].allocated = 1;
meta[0].head = 1;
meta[0].sizeclass = static_cast<uint8_t>(sizeclass);
meta[0].link = get_initial_link(sizeclass, true);
meta[0].link = get_initial_offset(sizeclass, true);
if constexpr (decommit_strategy == DecommitAll)
{
memory_provider.template notify_using<NoZero>(
pointer_offset(this, OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE);
@@ -183,9 +183,10 @@ namespace snmalloc
uint8_t n = meta[h].next;
meta[h].head = get_initial_bumpptr(sizeclass, false);
meta[h].head = 1;
meta[h].allocated = 1;
meta[h].sizeclass = static_cast<uint8_t>(sizeclass);
meta[h].link = get_initial_link(sizeclass, false);
meta[h].link = get_initial_offset(sizeclass, false);
head = h + n + 1;
used += 2;

5
src/override/malloc.cc
View File

@@ -23,9 +23,6 @@ extern "C"
SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size)
{
// Include size 0 in the first sizeclass.
size = ((size - 1) >> (bits::BITS - 1)) + size;
return ThreadAlloc::get()->alloc(size);
}
@@ -46,8 +43,6 @@ extern "C"
errno = ENOMEM;
return nullptr;
}
// Include size 0 in the first sizeclass.
sz = ((sz - 1) >> (bits::BITS - 1)) + sz;
return ThreadAlloc::get()->alloc<ZeroMem::YesZero>(sz);
}