From 7a198cbda5e823370fd24b7b145c76ecb87e31b4 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 28 Jun 2019 11:50:17 +0100 Subject: [PATCH] Aggressively optimise fast path for allocation This change introduces a per small sizeclass free list. That can be used to access the free objects for that sizeclass with minimal calculations being required. It changes to a partial bump ptr. We bump allocate a whole OS page worth of objects at a go, so we don't switch as frequently between bump and free list allocation. The code for the fast paths has been restructured to minimise the work required on the common case, and also it is all inlined for the common case. Allocating a zero sized object is moved off the fast path. Ask for 1 byte if you want to be fast. --- src/mem/alloc.h | 140 ++++++++++++++++++++++-------- src/mem/metaslab.h | 108 ++++++++++++----------- src/mem/sizeclass.h | 3 +- src/mem/sizeclasstable.h | 41 ++------- src/mem/slab.h | 182 ++++++++++++++++++++++++++------------- src/mem/superslab.h | 11 +-- src/override/malloc.cc | 5 -- 7 files changed, 299 insertions(+), 191 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index bf104c8..dbccb96 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -226,6 +226,17 @@ namespace snmalloc # define SNMALLOC_DEFAULT_PAGEMAP snmalloc::SuperslabMap<> #endif + // This class is just used so that the free lists are the first entry + // in the allocator and hence has better code gen. + // It contains a free list per small size class. These are used for allocation + // on the fast path. + // This part of the code is inspired by mimalloc. + class FastFreeLists + { + protected: + FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES]; + }; + /** * Allocator. This class is parameterised on three template parameters. The * `MemoryProvider` defines the source of memory for this allocator. @@ -247,7 +258,7 @@ namespace snmalloc class PageMap = SNMALLOC_DEFAULT_PAGEMAP, bool IsQueueInline = true> class Allocator - : public Pooled> + : public FastFreeLists, public Pooled> { LargeAlloc large_allocator; PageMap page_map; @@ -281,28 +292,27 @@ namespace snmalloc stats().alloc_request(size); - handle_message_queue(); - // Allocate memory of a statically known size. if constexpr (sizeclass < NUM_SMALL_CLASSES) { - constexpr size_t rsize = sizeclass_to_size(sizeclass); - return small_alloc(sizeclass, rsize); + return small_alloc(size); } else if constexpr (sizeclass < NUM_SIZECLASSES) { + handle_message_queue(); constexpr size_t rsize = sizeclass_to_size(sizeclass); return medium_alloc(sizeclass, rsize, size); } else - { + { + handle_message_queue(); return large_alloc(size); } #endif } template - ALLOCATOR void* alloc(size_t size) + inline ALLOCATOR void* alloc(size_t size) { #ifdef USE_MALLOC static_assert( @@ -315,18 +325,30 @@ namespace snmalloc #else stats().alloc_request(size); - handle_message_queue(); - - sizeclass_t sizeclass = size_to_sizeclass(size); - // Allocate memory of a dynamically known size. - if (sizeclass < NUM_SMALL_CLASSES) + // Perform the - 1 on size, so that zero wraps around and ends up on + // slow path. + if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1))) { // Allocations smaller than the slab size are more likely. Improve // branch prediction by placing this case first. - size_t rsize = sizeclass_to_size(sizeclass); - return small_alloc(sizeclass, rsize); + return small_alloc(size); } + + return alloc_not_small(size); + } + + template + NOINLINE ALLOCATOR void* alloc_not_small(size_t size) + { + handle_message_queue(); + + if (size == 0) + { + return small_alloc(1); + } + + sizeclass_t sizeclass = size_to_sizeclass(size); if (sizeclass < NUM_SIZECLASSES) { size_t rsize = sizeclass_to_size(sizeclass); @@ -418,7 +440,7 @@ namespace snmalloc #endif } - void dealloc(void* p) + ALWAYSINLINE void dealloc(void* p) { #ifdef USE_MALLOC return free(p); @@ -429,14 +451,10 @@ namespace snmalloc // pointer. uint8_t size = pagemap().get(address_cast(p)); - if (size == 0) - { - error("Not allocated by this allocator"); - } Superslab* super = Superslab::get(p); - if (size == PMSuperslab) + if (likely(size == PMSuperslab)) { RemoteAllocator* target = super->get_allocator(); Slab* slab = Slab::get(p); @@ -447,12 +465,17 @@ namespace snmalloc // pointer. sizeclass_t sizeclass = meta.sizeclass; - if (super->get_allocator() == public_state()) + if (likely(super->get_allocator() == public_state())) small_dealloc(super, p, sizeclass); else remote_dealloc(target, p, sizeclass); return; } + dealloc_not_small(p, size); + } + + NOINLINE void dealloc_not_small(void* p, uint8_t size) + { if (size == PMMediumslab) { Mediumslab* slab = Mediumslab::get(p); @@ -469,7 +492,13 @@ namespace snmalloc return; } + if (size == 0) + { + error("Not allocated by this allocator"); + } + # ifdef CHECK_CLIENT + Superslab* super = Superslab::get(p); if (size > 64 || address_cast(super) != address_cast(p)) { error("Not deallocating start of an object"); @@ -821,11 +850,11 @@ namespace snmalloc if (p->target_id() != super->get_allocator()->id()) error("Detected memory corruption. Potential use-after-free"); #endif - if (super->get_kind() == Super) + if (likely(super->get_kind() == Super)) { Slab* slab = Slab::get(p); Metaslab& meta = super->get_meta(slab); - if (p->target_id() == id()) + if (likely(p->target_id() == id())) { small_dealloc_offseted(super, p, meta.sizeclass); } @@ -978,7 +1007,7 @@ namespace snmalloc } template - void* small_alloc(sizeclass_t sizeclass, size_t rsize) + inline void* small_alloc(size_t size) { MEASURE_TIME_MARKERS( small_alloc, @@ -988,14 +1017,42 @@ namespace snmalloc zero_mem == YesZero ? "zeromem" : "nozeromem", allow_reserve == NoReserve ? "noreserve" : "reserve")); + sizeclass_t sizeclass = size_to_sizeclass(size); stats().sizeclass_alloc(sizeclass); - SlabList* sc = &small_classes[sizeclass]; + + auto& fl = small_fast_free_lists[sizeclass]; + auto head = fl.value; + if (likely((reinterpret_cast(head) & 1) == 0)) + { + void * p = head; + // Read the next slot from the memory that's about to be allocated. + fl.value = Metaslab::follow_next(p); + + if constexpr (zero_mem == YesZero) + { + large_allocator.memory_provider.zero(p, size); + } + return p; + } + + return small_alloc_slow(size); + } + + template + NOINLINE + void* small_alloc_slow(size_t size) + { + handle_message_queue(); + sizeclass_t sizeclass = size_to_sizeclass(size); + size_t rsize = sizeclass_to_size(sizeclass); + auto& sl = small_classes[sizeclass]; + Slab* slab; - if (!sc->is_empty()) + if (!sl.is_empty()) { - SlabLink* link = sc->get_head(); + SlabLink* link = sl.get_head(); slab = link->get_slab(); } else @@ -1005,13 +1062,13 @@ namespace snmalloc if ((allow_reserve == NoReserve) && (slab == nullptr)) return nullptr; - sc->insert(slab->get_link()); + sl.insert(slab->get_link()); } - - return slab->alloc(sc, rsize, large_allocator.memory_provider); + auto& ffl = small_fast_free_lists[sizeclass]; + return slab->alloc(sl, ffl, rsize, large_allocator.memory_provider); } - void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) + ALWAYSINLINE void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) { #ifdef CHECK_CLIENT Slab* slab = Slab::get(p); @@ -1025,20 +1082,27 @@ namespace snmalloc small_dealloc_offseted(super, offseted, sizeclass); } - void - small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) + ALWAYSINLINE void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) { MEASURE_TIME(small_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); - bool was_full = super->is_full(); - SlabList* sc = &small_classes[sizeclass]; Slab* slab = Slab::get(p); - Superslab::Action a = - slab->dealloc(sc, super, p, large_allocator.memory_provider); - if (a == Superslab::NoSlabReturn) + if (likely(slab->dealloc_fast(super, p))) return; + small_dealloc_offseted_slow(super, p, sizeclass); + } + + NOINLINE void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass) + { + bool was_full = super->is_full(); + SlabList* sl = &small_classes[sizeclass]; + Slab* slab = Slab::get(p); + Superslab::Action a = + slab->dealloc_slow(sl, super, p, large_allocator.memory_provider); + if (likely(a == Superslab::NoSlabReturn)) + return; stats().sizeclass_dealloc_slab(sizeclass); if (a == Superslab::NoStatusChange) diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 7bb2a32..df03708 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -29,22 +29,24 @@ namespace snmalloc // This can be either a short or a standard slab. class Metaslab { - private: - // How many entries are used in this slab. + public: + // How many entries are not in the free list of slab. uint16_t used = 0; - public: - // Bump free list of unused entries in this sizeclass. - // If the bottom bit is 1, then this represents a bump_ptr - // of where we have allocated up to in this slab. Otherwise, - // it represents the location of the first block in the free - // list. The free list is chained through deallocated blocks. - // It is terminated with a bump ptr. - // - // Note that, the first entry in a slab is never bump allocated - // but is used for the link. This means that 1 represents the fully - // bump allocated slab. + // How many entries have been allocated from this slab. + uint16_t allocated; + + // Index of first entry in this slab that forms the free + // list. The list entries are stored as the first pointer + // in each unused object. The terminator is a pointer or + // offset into the block with the bottom bit set. This means + // I.e. + // * an empty list has a head of 1. + // * a one element list has an head contains an offset to this + // this block, and then contains a pointer with the bottom + // bit set. Mod head; + // When a slab has free space it will be on the has space list for // that size class. We use an empty block in this slab to be the // doubly linked node into that size class's free list. @@ -93,35 +95,38 @@ namespace snmalloc /// Value used to check for corruptions in a block static constexpr size_t POISON = - static_cast(bits::is64() ? 0xDEADBEEFDEAD0000 : 0xDEAD0000); + static_cast(bits::is64() ? 0xDEADBEEFDEADBEEF : 0xDEADBEEF); /// Store next pointer in a block. In Debug using magic value to detect some /// simple corruptions. - static void store_next(void* p, uint16_t head) + static void store_next(void* p, void* head) { #ifndef CHECK_CLIENT - *static_cast(p) = head; + *static_cast(p) = head; #else - *static_cast(p) = - head ^ POISON ^ (static_cast(head) << (bits::BITS - 16)); + *static_cast(p) = head; + *(static_cast(p) + 1) = + address_cast(head) ^ POISON; #endif } /// Accessor function for the next pointer in a block. /// In Debug checks for simple corruptions. - static uint16_t follow_next(void* node) + static void* follow_next(void* node) { - size_t next = *static_cast(node); #ifdef CHECK_CLIENT - if (((next ^ POISON) ^ (next << (bits::BITS - 16))) > 0xFFFF) + uintptr_t next = *static_cast(node); + uintptr_t chk = *(static_cast(node) + 1); + if ((next ^ chk) != POISON) error("Detected memory corruption. Use-after-free."); #endif - return static_cast(next); + return *static_cast(node); } + bool valid_head(bool is_short) { size_t size = sizeclass_to_size(sizeclass); - size_t slab_start = get_initial_link(sizeclass, is_short); + size_t slab_start = get_initial_offset(sizeclass, is_short); size_t all_high_bits = ~static_cast(1); size_t head_start = @@ -138,18 +143,19 @@ namespace snmalloc * We don't expect a cycle, so worst case is only followed by a crash, so * slow doesn't mater. **/ - void debug_slab_acyclic_free_list(Slab* slab) + size_t debug_slab_acyclic_free_list(Slab* slab) { #ifndef NDEBUG - uint16_t curr = head; - uint16_t curr_slow = head; + size_t length = 0; + void* curr = pointer_offset(slab, head); + void* curr_slow = pointer_offset(slab, head); bool both = false; - while ((curr & 1) != 1) + while ((reinterpret_cast(curr) & 1) == 0) { - curr = follow_next(pointer_offset(slab, curr)); + curr = follow_next(curr); if (both) { - curr_slow = follow_next(pointer_offset(slab, curr_slow)); + curr_slow = follow_next(curr_slow); } if (curr == curr_slow) @@ -158,9 +164,12 @@ namespace snmalloc } both = !both; + length ++; } + return length; #else UNUSED(slab); + return 0; #endif } @@ -168,7 +177,10 @@ namespace snmalloc { #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) size_t size = sizeclass_to_size(sizeclass); - size_t offset = get_initial_link(sizeclass, is_short); + size_t offset = get_initial_offset(sizeclass, is_short); + + if (is_unused()) + return; size_t accounted_for = used * size + offset; @@ -184,40 +196,40 @@ namespace snmalloc // Block is not full assert(SLAB_SIZE > accounted_for); - debug_slab_acyclic_free_list(slab); - + size_t length = debug_slab_acyclic_free_list(slab); + UNUSED(length); // Walk bump-free-list-segment accounting for unused space - uint16_t curr = head; - while ((curr & 1) != 1) + void* curr = pointer_offset(slab, head); + while ((reinterpret_cast(curr) & 1) == 0) { // Check we are looking at a correctly aligned block - uint16_t start = remove_cache_friendly_offset(curr, sizeclass); - assert((start - offset) % size == 0); + void* start = curr; + assert(((address_cast(start) - address_cast(slab) - offset) % size) == 0); // Account for free elements in free list accounted_for += size; assert(SLAB_SIZE >= accounted_for); // We should never reach the link node in the free list. - assert(curr != link); + assert(curr != pointer_offset(slab, link)); // Iterate bump/free list segment - curr = follow_next(pointer_offset(slab, curr)); + curr = follow_next(curr); } - if (curr != 1) + auto bumpptr = (allocated * size) + offset; + // Check we haven't allocaated more than gits in a slab + assert(bumpptr <= SLAB_SIZE); + + // Account for to be bump allocated space + accounted_for += SLAB_SIZE - bumpptr; + + if (bumpptr != SLAB_SIZE) { - // Check we terminated traversal on a correctly aligned block - uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass); - assert((start - offset) % size == 0); - - // Account for to be bump allocated space - accounted_for += SLAB_SIZE - (curr - 1); - // The link should be the first allocation as we // haven't completely filled this block at any point. - assert(link == get_initial_link(sizeclass, is_short)); + assert(link == get_initial_offset(sizeclass, is_short)); } - + assert(!is_full()); // Add the link node. accounted_for += size; diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index cb5d3b8..735adab 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -8,8 +8,7 @@ namespace snmalloc using sizeclass_t = size_t; // using sizeclass_t = uint8_t; - constexpr static uint16_t get_initial_bumpptr(sizeclass_t sc, bool is_short); - constexpr static uint16_t get_initial_link(sizeclass_t sc, bool is_short); + constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short); constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); constexpr static size_t sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index cab6595..6ecab99 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -23,10 +23,8 @@ namespace snmalloc ModArray size; ModArray cache_friendly_mask; ModArray inverse_cache_friendly_mask; - ModArray bump_ptr_start; - ModArray short_bump_ptr_start; - ModArray initial_link_ptr; - ModArray short_initial_link_ptr; + ModArray initial_offset_ptr; + ModArray short_initial_offset_ptr; ModArray medium_slab_slots; @@ -35,10 +33,8 @@ namespace snmalloc size(), cache_friendly_mask(), inverse_cache_friendly_mask(), - bump_ptr_start(), - short_bump_ptr_start(), - initial_link_ptr(), - short_initial_link_ptr(), + initial_offset_ptr(), + short_initial_offset_ptr(), medium_slab_slots() { size_t curr = 1; @@ -71,19 +67,9 @@ namespace snmalloc size_t correction = SLAB_SIZE % size[i]; // First element in the block is the link - initial_link_ptr[i] = static_cast(correction); - short_initial_link_ptr[i] = + initial_offset_ptr[i] = static_cast(correction); + short_initial_offset_ptr[i] = static_cast(header_size + short_correction); - - // Move to object after link. - auto short_after_link = short_initial_link_ptr[i] + size[i]; - size_t after_link = initial_link_ptr[i] + size[i]; - - // Bump ptr has bottom bit set. - // In case we only have one object on this slab check for wrap around. - short_bump_ptr_start[i] = - static_cast((short_after_link + 1) % SLAB_SIZE); - bump_ptr_start[i] = static_cast((after_link + 1) % SLAB_SIZE); } for (sizeclass_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) @@ -97,21 +83,12 @@ namespace snmalloc static constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); static inline constexpr uint16_t - get_initial_bumpptr(sizeclass_t sc, bool is_short) + get_initial_offset(sizeclass_t sc, bool is_short) { if (is_short) - return sizeclass_metadata.short_bump_ptr_start[sc]; + return sizeclass_metadata.short_initial_offset_ptr[sc]; - return sizeclass_metadata.bump_ptr_start[sc]; - } - - static inline constexpr uint16_t - get_initial_link(sizeclass_t sc, bool is_short) - { - if (is_short) - return sizeclass_metadata.short_initial_link_ptr[sc]; - - return sizeclass_metadata.initial_link_ptr[sc]; + return sizeclass_metadata.initial_offset_ptr[sc]; } constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) diff --git a/src/mem/slab.h b/src/mem/slab.h index 0442740..141b6e8 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -4,6 +4,12 @@ namespace snmalloc { + struct FreeListHead + { + // Use a value with bottom bit set for empty list. + void* value = pointer_offset(nullptr, 1); + }; + class Slab { private: @@ -31,7 +37,7 @@ namespace snmalloc } template - void* alloc(SlabList* sc, size_t rsize, MemoryProvider& memory_provider) + inline void* alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize, MemoryProvider& memory_provider) { // Read the head from the metadata stored in the superslab. Metaslab& meta = get_meta(); @@ -39,38 +45,81 @@ namespace snmalloc assert(rsize == sizeclass_to_size(meta.sizeclass)); meta.debug_slab_invariant(is_short(), this); - assert(sc->get_head() == (SlabLink*)((size_t)this + meta.link)); + assert(sl.get_head() == (SlabLink*)((size_t)this + meta.link)); assert(!meta.is_full()); - meta.add_use(); - void* p; - if ((head & 1) == 0) + if (head == 1) { - void* node = pointer_offset(this, head); - - // Read the next slot from the memory that's about to be allocated. - meta.head = Metaslab::follow_next(node); - - p = remove_cache_friendly_offset(node, meta.sizeclass); - } - else - { - if (meta.head == 1) + size_t bumpptr = get_initial_offset(meta.sizeclass, is_short()); + bumpptr += meta.allocated * rsize; + if (bumpptr == SLAB_SIZE) { + meta.add_use(); + assert(meta.used == meta.allocated); + p = pointer_offset(this, meta.link); - sc->pop(); meta.set_full(); + sl.pop(); + goto finish1; } else { - // This slab is being bump allocated. - p = pointer_offset(this, head - 1); - meta.head = (head + static_cast(rsize)) & (SLAB_SIZE - 1); + + void* curr = nullptr; + bool commit = false; + while (true) + { + size_t newbumpptr = bumpptr + rsize; + auto alignedbumpptr = + bits::align_up(bumpptr - 1, OS_PAGE_SIZE); + auto alignednewbumpptr = bits::align_up(newbumpptr, OS_PAGE_SIZE); + + if (alignedbumpptr != alignednewbumpptr) + { + // We have committed once already. + if (commit) break; + + memory_provider.template notify_using(pointer_offset(this, alignedbumpptr), alignednewbumpptr - alignedbumpptr); + + commit = true; + } + + if (curr == nullptr) + { + meta.head = static_cast(bumpptr); + } + else + { + Metaslab::store_next(curr, pointer_offset(this, bumpptr)); + } + curr = pointer_offset(this, bumpptr); + bumpptr = newbumpptr; + meta.allocated = meta.allocated + 1; + } + + assert(curr != nullptr); + Metaslab::store_next(curr, pointer_offset(nullptr, 1)); } } + { + p = pointer_offset(this, meta.head); + + // Read the next slot from the memory that's about to be allocated. + void* next = Metaslab::follow_next(p); + // Put everything in allocators small_class free list. + meta.head = 1; + fast_free_list.value = next; + // Treat stealing the free list as allocating it all. + // Link is not in use, i.e. - 1 is required. + meta.used = meta.allocated - 1; + } + + assert (is_start_of_object(Superslab::get(p), p)); + + finish1: meta.debug_slab_invariant(is_short(), this); if constexpr (zero_mem == YesZero) @@ -92,38 +141,56 @@ namespace snmalloc address_cast(this) + SLAB_SIZE - address_cast(p)); } - // Returns true, if it alters get_status. - template - inline typename Superslab::Action dealloc( - SlabList* sc, Superslab* super, void* p, MemoryProvider& memory_provider) + // Returns true, if it deallocation can proceed without changing any status bits. + // Note that this does remove the use from the meta slab, so it doesn't need doing + // on the slow path. + ALWAYSINLINE bool dealloc_fast(Superslab* super, void* p) { Metaslab& meta = super->get_meta(this); - - bool was_full = meta.is_full(); - #ifdef CHECK_CLIENT if (meta.is_unused()) error("Detected potential double free."); #endif - meta.debug_slab_invariant(is_short(), this); meta.sub_use(); + bool was_full = meta.is_full(); + if (unlikely(was_full)) return false; + + bool is_unused = meta.is_unused(); + if (unlikely(is_unused)) return false; + + // Update the head and the next pointer in the free list. + uint16_t head = meta.head; + uint16_t current = pointer_to_index(p); + + // Set the head to the memory being deallocated. + meta.head = current; + assert(meta.valid_head(is_short())); + + // Set the next pointer to the previous head. + Metaslab::store_next(p, pointer_offset(this, head)); + meta.debug_slab_invariant(is_short(), this); + return true; + } + + // If dealloc fast returns false, then call this. + // This does not need to remove the "use" as done by the fast path. + // Returns a complex return code for managing the superslab meta data. + // i.e. This deallocation could make an entire superslab free. + template + NOINLINE typename Superslab::Action dealloc_slow( + SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider) + { + Metaslab& meta = super->get_meta(this); + + bool was_full = meta.is_full(); + bool is_unused = meta.is_unused(); + if (was_full) { // We are not on the sizeclass list. - if (!meta.is_unused()) - { - // Update the head and the sizeclass link. - uint16_t index = pointer_to_index(p); - assert(meta.head == 1); - meta.link = index; - - // Push on the list of slabs for this sizeclass. - sc->insert(meta.get_link(this)); - meta.debug_slab_invariant(is_short(), this); - } - else + if (is_unused) { // Dealloc on the superslab. if (is_short()) @@ -131,37 +198,30 @@ namespace snmalloc return super->dealloc_slab(this, memory_provider); } + // Update the head and the sizeclass link. + uint16_t index = pointer_to_index(p); + assert(meta.head == 1); +// assert(meta.fully_allocated(is_short())); + meta.link = index; + + // Push on the list of slabs for this sizeclass. + sl->insert(meta.get_link(this)); + meta.debug_slab_invariant(is_short(), this); + return Superslab::NoSlabReturn; } - else if (meta.is_unused()) + + if (is_unused) { // Remove from the sizeclass list and dealloc on the superslab. - sc->remove(meta.get_link(this)); + sl->remove(meta.get_link(this)); if (is_short()) return super->dealloc_short_slab(memory_provider); return super->dealloc_slab(this, memory_provider); } - else - { -#ifndef NDEBUG - sc->debug_check_contains(meta.get_link(this)); -#endif - - // Update the head and the next pointer in the free list. - uint16_t head = meta.head; - uint16_t current = pointer_to_index(p); - - // Set the head to the memory being deallocated. - meta.head = current; - assert(meta.valid_head(is_short())); - - // Set the next pointer to the previous head. - Metaslab::store_next(p, head); - - meta.debug_slab_invariant(is_short(), this); - } - return Superslab::NoSlabReturn; + + abort(); } bool is_short() diff --git a/src/mem/superslab.h b/src/mem/superslab.h index 2c8d8b0..66d65a5 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -160,11 +160,11 @@ namespace snmalloc if ((used & 1) == 1) return alloc_slab(sizeclass, memory_provider); - meta[0].head = get_initial_bumpptr(sizeclass, true); + meta[0].allocated = 1; + meta[0].head = 1; meta[0].sizeclass = static_cast(sizeclass); - meta[0].link = get_initial_link(sizeclass, true); + meta[0].link = get_initial_offset(sizeclass, true); - if constexpr (decommit_strategy == DecommitAll) { memory_provider.template notify_using( pointer_offset(this, OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE); @@ -183,9 +183,10 @@ namespace snmalloc uint8_t n = meta[h].next; - meta[h].head = get_initial_bumpptr(sizeclass, false); + meta[h].head = 1; + meta[h].allocated = 1; meta[h].sizeclass = static_cast(sizeclass); - meta[h].link = get_initial_link(sizeclass, false); + meta[h].link = get_initial_offset(sizeclass, false); head = h + n + 1; used += 2; diff --git a/src/override/malloc.cc b/src/override/malloc.cc index aba9ac3..1e55b84 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -23,9 +23,6 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) { - // Include size 0 in the first sizeclass. - size = ((size - 1) >> (bits::BITS - 1)) + size; - return ThreadAlloc::get()->alloc(size); } @@ -46,8 +43,6 @@ extern "C" errno = ENOMEM; return nullptr; } - // Include size 0 in the first sizeclass. - sz = ((sz - 1) >> (bits::BITS - 1)) + sz; return ThreadAlloc::get()->alloc(sz); }