diff --git a/src/mem/alloc.h b/src/mem/alloc.h index bf104c8..dbccb96 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -226,6 +226,17 @@ namespace snmalloc # define SNMALLOC_DEFAULT_PAGEMAP snmalloc::SuperslabMap<> #endif + // This class is just used so that the free lists are the first entry + // in the allocator and hence has better code gen. + // It contains a free list per small size class. These are used for allocation + // on the fast path. + // This part of the code is inspired by mimalloc. + class FastFreeLists + { + protected: + FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES]; + }; + /** * Allocator. This class is parameterised on three template parameters. The * `MemoryProvider` defines the source of memory for this allocator. @@ -247,7 +258,7 @@ namespace snmalloc class PageMap = SNMALLOC_DEFAULT_PAGEMAP, bool IsQueueInline = true> class Allocator - : public Pooled> + : public FastFreeLists, public Pooled> { LargeAlloc large_allocator; PageMap page_map; @@ -281,28 +292,27 @@ namespace snmalloc stats().alloc_request(size); - handle_message_queue(); - // Allocate memory of a statically known size. if constexpr (sizeclass < NUM_SMALL_CLASSES) { - constexpr size_t rsize = sizeclass_to_size(sizeclass); - return small_alloc(sizeclass, rsize); + return small_alloc(size); } else if constexpr (sizeclass < NUM_SIZECLASSES) { + handle_message_queue(); constexpr size_t rsize = sizeclass_to_size(sizeclass); return medium_alloc(sizeclass, rsize, size); } else - { + { + handle_message_queue(); return large_alloc(size); } #endif } template - ALLOCATOR void* alloc(size_t size) + inline ALLOCATOR void* alloc(size_t size) { #ifdef USE_MALLOC static_assert( @@ -315,18 +325,30 @@ namespace snmalloc #else stats().alloc_request(size); - handle_message_queue(); - - sizeclass_t sizeclass = size_to_sizeclass(size); - // Allocate memory of a dynamically known size. - if (sizeclass < NUM_SMALL_CLASSES) + // Perform the - 1 on size, so that zero wraps around and ends up on + // slow path. + if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1))) { // Allocations smaller than the slab size are more likely. Improve // branch prediction by placing this case first. - size_t rsize = sizeclass_to_size(sizeclass); - return small_alloc(sizeclass, rsize); + return small_alloc(size); } + + return alloc_not_small(size); + } + + template + NOINLINE ALLOCATOR void* alloc_not_small(size_t size) + { + handle_message_queue(); + + if (size == 0) + { + return small_alloc(1); + } + + sizeclass_t sizeclass = size_to_sizeclass(size); if (sizeclass < NUM_SIZECLASSES) { size_t rsize = sizeclass_to_size(sizeclass); @@ -418,7 +440,7 @@ namespace snmalloc #endif } - void dealloc(void* p) + ALWAYSINLINE void dealloc(void* p) { #ifdef USE_MALLOC return free(p); @@ -429,14 +451,10 @@ namespace snmalloc // pointer. uint8_t size = pagemap().get(address_cast(p)); - if (size == 0) - { - error("Not allocated by this allocator"); - } Superslab* super = Superslab::get(p); - if (size == PMSuperslab) + if (likely(size == PMSuperslab)) { RemoteAllocator* target = super->get_allocator(); Slab* slab = Slab::get(p); @@ -447,12 +465,17 @@ namespace snmalloc // pointer. sizeclass_t sizeclass = meta.sizeclass; - if (super->get_allocator() == public_state()) + if (likely(super->get_allocator() == public_state())) small_dealloc(super, p, sizeclass); else remote_dealloc(target, p, sizeclass); return; } + dealloc_not_small(p, size); + } + + NOINLINE void dealloc_not_small(void* p, uint8_t size) + { if (size == PMMediumslab) { Mediumslab* slab = Mediumslab::get(p); @@ -469,7 +492,13 @@ namespace snmalloc return; } + if (size == 0) + { + error("Not allocated by this allocator"); + } + # ifdef CHECK_CLIENT + Superslab* super = Superslab::get(p); if (size > 64 || address_cast(super) != address_cast(p)) { error("Not deallocating start of an object"); @@ -821,11 +850,11 @@ namespace snmalloc if (p->target_id() != super->get_allocator()->id()) error("Detected memory corruption. Potential use-after-free"); #endif - if (super->get_kind() == Super) + if (likely(super->get_kind() == Super)) { Slab* slab = Slab::get(p); Metaslab& meta = super->get_meta(slab); - if (p->target_id() == id()) + if (likely(p->target_id() == id())) { small_dealloc_offseted(super, p, meta.sizeclass); } @@ -978,7 +1007,7 @@ namespace snmalloc } template - void* small_alloc(sizeclass_t sizeclass, size_t rsize) + inline void* small_alloc(size_t size) { MEASURE_TIME_MARKERS( small_alloc, @@ -988,14 +1017,42 @@ namespace snmalloc zero_mem == YesZero ? "zeromem" : "nozeromem", allow_reserve == NoReserve ? "noreserve" : "reserve")); + sizeclass_t sizeclass = size_to_sizeclass(size); stats().sizeclass_alloc(sizeclass); - SlabList* sc = &small_classes[sizeclass]; + + auto& fl = small_fast_free_lists[sizeclass]; + auto head = fl.value; + if (likely((reinterpret_cast(head) & 1) == 0)) + { + void * p = head; + // Read the next slot from the memory that's about to be allocated. + fl.value = Metaslab::follow_next(p); + + if constexpr (zero_mem == YesZero) + { + large_allocator.memory_provider.zero(p, size); + } + return p; + } + + return small_alloc_slow(size); + } + + template + NOINLINE + void* small_alloc_slow(size_t size) + { + handle_message_queue(); + sizeclass_t sizeclass = size_to_sizeclass(size); + size_t rsize = sizeclass_to_size(sizeclass); + auto& sl = small_classes[sizeclass]; + Slab* slab; - if (!sc->is_empty()) + if (!sl.is_empty()) { - SlabLink* link = sc->get_head(); + SlabLink* link = sl.get_head(); slab = link->get_slab(); } else @@ -1005,13 +1062,13 @@ namespace snmalloc if ((allow_reserve == NoReserve) && (slab == nullptr)) return nullptr; - sc->insert(slab->get_link()); + sl.insert(slab->get_link()); } - - return slab->alloc(sc, rsize, large_allocator.memory_provider); + auto& ffl = small_fast_free_lists[sizeclass]; + return slab->alloc(sl, ffl, rsize, large_allocator.memory_provider); } - void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) + ALWAYSINLINE void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) { #ifdef CHECK_CLIENT Slab* slab = Slab::get(p); @@ -1025,20 +1082,27 @@ namespace snmalloc small_dealloc_offseted(super, offseted, sizeclass); } - void - small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) + ALWAYSINLINE void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) { MEASURE_TIME(small_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); - bool was_full = super->is_full(); - SlabList* sc = &small_classes[sizeclass]; Slab* slab = Slab::get(p); - Superslab::Action a = - slab->dealloc(sc, super, p, large_allocator.memory_provider); - if (a == Superslab::NoSlabReturn) + if (likely(slab->dealloc_fast(super, p))) return; + small_dealloc_offseted_slow(super, p, sizeclass); + } + + NOINLINE void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass) + { + bool was_full = super->is_full(); + SlabList* sl = &small_classes[sizeclass]; + Slab* slab = Slab::get(p); + Superslab::Action a = + slab->dealloc_slow(sl, super, p, large_allocator.memory_provider); + if (likely(a == Superslab::NoSlabReturn)) + return; stats().sizeclass_dealloc_slab(sizeclass); if (a == Superslab::NoStatusChange) diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 7bb2a32..df03708 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -29,22 +29,24 @@ namespace snmalloc // This can be either a short or a standard slab. class Metaslab { - private: - // How many entries are used in this slab. + public: + // How many entries are not in the free list of slab. uint16_t used = 0; - public: - // Bump free list of unused entries in this sizeclass. - // If the bottom bit is 1, then this represents a bump_ptr - // of where we have allocated up to in this slab. Otherwise, - // it represents the location of the first block in the free - // list. The free list is chained through deallocated blocks. - // It is terminated with a bump ptr. - // - // Note that, the first entry in a slab is never bump allocated - // but is used for the link. This means that 1 represents the fully - // bump allocated slab. + // How many entries have been allocated from this slab. + uint16_t allocated; + + // Index of first entry in this slab that forms the free + // list. The list entries are stored as the first pointer + // in each unused object. The terminator is a pointer or + // offset into the block with the bottom bit set. This means + // I.e. + // * an empty list has a head of 1. + // * a one element list has an head contains an offset to this + // this block, and then contains a pointer with the bottom + // bit set. Mod head; + // When a slab has free space it will be on the has space list for // that size class. We use an empty block in this slab to be the // doubly linked node into that size class's free list. @@ -93,35 +95,38 @@ namespace snmalloc /// Value used to check for corruptions in a block static constexpr size_t POISON = - static_cast(bits::is64() ? 0xDEADBEEFDEAD0000 : 0xDEAD0000); + static_cast(bits::is64() ? 0xDEADBEEFDEADBEEF : 0xDEADBEEF); /// Store next pointer in a block. In Debug using magic value to detect some /// simple corruptions. - static void store_next(void* p, uint16_t head) + static void store_next(void* p, void* head) { #ifndef CHECK_CLIENT - *static_cast(p) = head; + *static_cast(p) = head; #else - *static_cast(p) = - head ^ POISON ^ (static_cast(head) << (bits::BITS - 16)); + *static_cast(p) = head; + *(static_cast(p) + 1) = + address_cast(head) ^ POISON; #endif } /// Accessor function for the next pointer in a block. /// In Debug checks for simple corruptions. - static uint16_t follow_next(void* node) + static void* follow_next(void* node) { - size_t next = *static_cast(node); #ifdef CHECK_CLIENT - if (((next ^ POISON) ^ (next << (bits::BITS - 16))) > 0xFFFF) + uintptr_t next = *static_cast(node); + uintptr_t chk = *(static_cast(node) + 1); + if ((next ^ chk) != POISON) error("Detected memory corruption. Use-after-free."); #endif - return static_cast(next); + return *static_cast(node); } + bool valid_head(bool is_short) { size_t size = sizeclass_to_size(sizeclass); - size_t slab_start = get_initial_link(sizeclass, is_short); + size_t slab_start = get_initial_offset(sizeclass, is_short); size_t all_high_bits = ~static_cast(1); size_t head_start = @@ -138,18 +143,19 @@ namespace snmalloc * We don't expect a cycle, so worst case is only followed by a crash, so * slow doesn't mater. **/ - void debug_slab_acyclic_free_list(Slab* slab) + size_t debug_slab_acyclic_free_list(Slab* slab) { #ifndef NDEBUG - uint16_t curr = head; - uint16_t curr_slow = head; + size_t length = 0; + void* curr = pointer_offset(slab, head); + void* curr_slow = pointer_offset(slab, head); bool both = false; - while ((curr & 1) != 1) + while ((reinterpret_cast(curr) & 1) == 0) { - curr = follow_next(pointer_offset(slab, curr)); + curr = follow_next(curr); if (both) { - curr_slow = follow_next(pointer_offset(slab, curr_slow)); + curr_slow = follow_next(curr_slow); } if (curr == curr_slow) @@ -158,9 +164,12 @@ namespace snmalloc } both = !both; + length ++; } + return length; #else UNUSED(slab); + return 0; #endif } @@ -168,7 +177,10 @@ namespace snmalloc { #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) size_t size = sizeclass_to_size(sizeclass); - size_t offset = get_initial_link(sizeclass, is_short); + size_t offset = get_initial_offset(sizeclass, is_short); + + if (is_unused()) + return; size_t accounted_for = used * size + offset; @@ -184,40 +196,40 @@ namespace snmalloc // Block is not full assert(SLAB_SIZE > accounted_for); - debug_slab_acyclic_free_list(slab); - + size_t length = debug_slab_acyclic_free_list(slab); + UNUSED(length); // Walk bump-free-list-segment accounting for unused space - uint16_t curr = head; - while ((curr & 1) != 1) + void* curr = pointer_offset(slab, head); + while ((reinterpret_cast(curr) & 1) == 0) { // Check we are looking at a correctly aligned block - uint16_t start = remove_cache_friendly_offset(curr, sizeclass); - assert((start - offset) % size == 0); + void* start = curr; + assert(((address_cast(start) - address_cast(slab) - offset) % size) == 0); // Account for free elements in free list accounted_for += size; assert(SLAB_SIZE >= accounted_for); // We should never reach the link node in the free list. - assert(curr != link); + assert(curr != pointer_offset(slab, link)); // Iterate bump/free list segment - curr = follow_next(pointer_offset(slab, curr)); + curr = follow_next(curr); } - if (curr != 1) + auto bumpptr = (allocated * size) + offset; + // Check we haven't allocaated more than gits in a slab + assert(bumpptr <= SLAB_SIZE); + + // Account for to be bump allocated space + accounted_for += SLAB_SIZE - bumpptr; + + if (bumpptr != SLAB_SIZE) { - // Check we terminated traversal on a correctly aligned block - uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass); - assert((start - offset) % size == 0); - - // Account for to be bump allocated space - accounted_for += SLAB_SIZE - (curr - 1); - // The link should be the first allocation as we // haven't completely filled this block at any point. - assert(link == get_initial_link(sizeclass, is_short)); + assert(link == get_initial_offset(sizeclass, is_short)); } - + assert(!is_full()); // Add the link node. accounted_for += size; diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index cb5d3b8..735adab 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -8,8 +8,7 @@ namespace snmalloc using sizeclass_t = size_t; // using sizeclass_t = uint8_t; - constexpr static uint16_t get_initial_bumpptr(sizeclass_t sc, bool is_short); - constexpr static uint16_t get_initial_link(sizeclass_t sc, bool is_short); + constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short); constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); constexpr static size_t sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index cab6595..6ecab99 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -23,10 +23,8 @@ namespace snmalloc ModArray size; ModArray cache_friendly_mask; ModArray inverse_cache_friendly_mask; - ModArray bump_ptr_start; - ModArray short_bump_ptr_start; - ModArray initial_link_ptr; - ModArray short_initial_link_ptr; + ModArray initial_offset_ptr; + ModArray short_initial_offset_ptr; ModArray medium_slab_slots; @@ -35,10 +33,8 @@ namespace snmalloc size(), cache_friendly_mask(), inverse_cache_friendly_mask(), - bump_ptr_start(), - short_bump_ptr_start(), - initial_link_ptr(), - short_initial_link_ptr(), + initial_offset_ptr(), + short_initial_offset_ptr(), medium_slab_slots() { size_t curr = 1; @@ -71,19 +67,9 @@ namespace snmalloc size_t correction = SLAB_SIZE % size[i]; // First element in the block is the link - initial_link_ptr[i] = static_cast(correction); - short_initial_link_ptr[i] = + initial_offset_ptr[i] = static_cast(correction); + short_initial_offset_ptr[i] = static_cast(header_size + short_correction); - - // Move to object after link. - auto short_after_link = short_initial_link_ptr[i] + size[i]; - size_t after_link = initial_link_ptr[i] + size[i]; - - // Bump ptr has bottom bit set. - // In case we only have one object on this slab check for wrap around. - short_bump_ptr_start[i] = - static_cast((short_after_link + 1) % SLAB_SIZE); - bump_ptr_start[i] = static_cast((after_link + 1) % SLAB_SIZE); } for (sizeclass_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) @@ -97,21 +83,12 @@ namespace snmalloc static constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); static inline constexpr uint16_t - get_initial_bumpptr(sizeclass_t sc, bool is_short) + get_initial_offset(sizeclass_t sc, bool is_short) { if (is_short) - return sizeclass_metadata.short_bump_ptr_start[sc]; + return sizeclass_metadata.short_initial_offset_ptr[sc]; - return sizeclass_metadata.bump_ptr_start[sc]; - } - - static inline constexpr uint16_t - get_initial_link(sizeclass_t sc, bool is_short) - { - if (is_short) - return sizeclass_metadata.short_initial_link_ptr[sc]; - - return sizeclass_metadata.initial_link_ptr[sc]; + return sizeclass_metadata.initial_offset_ptr[sc]; } constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) diff --git a/src/mem/slab.h b/src/mem/slab.h index 0442740..141b6e8 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -4,6 +4,12 @@ namespace snmalloc { + struct FreeListHead + { + // Use a value with bottom bit set for empty list. + void* value = pointer_offset(nullptr, 1); + }; + class Slab { private: @@ -31,7 +37,7 @@ namespace snmalloc } template - void* alloc(SlabList* sc, size_t rsize, MemoryProvider& memory_provider) + inline void* alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize, MemoryProvider& memory_provider) { // Read the head from the metadata stored in the superslab. Metaslab& meta = get_meta(); @@ -39,38 +45,81 @@ namespace snmalloc assert(rsize == sizeclass_to_size(meta.sizeclass)); meta.debug_slab_invariant(is_short(), this); - assert(sc->get_head() == (SlabLink*)((size_t)this + meta.link)); + assert(sl.get_head() == (SlabLink*)((size_t)this + meta.link)); assert(!meta.is_full()); - meta.add_use(); - void* p; - if ((head & 1) == 0) + if (head == 1) { - void* node = pointer_offset(this, head); - - // Read the next slot from the memory that's about to be allocated. - meta.head = Metaslab::follow_next(node); - - p = remove_cache_friendly_offset(node, meta.sizeclass); - } - else - { - if (meta.head == 1) + size_t bumpptr = get_initial_offset(meta.sizeclass, is_short()); + bumpptr += meta.allocated * rsize; + if (bumpptr == SLAB_SIZE) { + meta.add_use(); + assert(meta.used == meta.allocated); + p = pointer_offset(this, meta.link); - sc->pop(); meta.set_full(); + sl.pop(); + goto finish1; } else { - // This slab is being bump allocated. - p = pointer_offset(this, head - 1); - meta.head = (head + static_cast(rsize)) & (SLAB_SIZE - 1); + + void* curr = nullptr; + bool commit = false; + while (true) + { + size_t newbumpptr = bumpptr + rsize; + auto alignedbumpptr = + bits::align_up(bumpptr - 1, OS_PAGE_SIZE); + auto alignednewbumpptr = bits::align_up(newbumpptr, OS_PAGE_SIZE); + + if (alignedbumpptr != alignednewbumpptr) + { + // We have committed once already. + if (commit) break; + + memory_provider.template notify_using(pointer_offset(this, alignedbumpptr), alignednewbumpptr - alignedbumpptr); + + commit = true; + } + + if (curr == nullptr) + { + meta.head = static_cast(bumpptr); + } + else + { + Metaslab::store_next(curr, pointer_offset(this, bumpptr)); + } + curr = pointer_offset(this, bumpptr); + bumpptr = newbumpptr; + meta.allocated = meta.allocated + 1; + } + + assert(curr != nullptr); + Metaslab::store_next(curr, pointer_offset(nullptr, 1)); } } + { + p = pointer_offset(this, meta.head); + + // Read the next slot from the memory that's about to be allocated. + void* next = Metaslab::follow_next(p); + // Put everything in allocators small_class free list. + meta.head = 1; + fast_free_list.value = next; + // Treat stealing the free list as allocating it all. + // Link is not in use, i.e. - 1 is required. + meta.used = meta.allocated - 1; + } + + assert (is_start_of_object(Superslab::get(p), p)); + + finish1: meta.debug_slab_invariant(is_short(), this); if constexpr (zero_mem == YesZero) @@ -92,38 +141,56 @@ namespace snmalloc address_cast(this) + SLAB_SIZE - address_cast(p)); } - // Returns true, if it alters get_status. - template - inline typename Superslab::Action dealloc( - SlabList* sc, Superslab* super, void* p, MemoryProvider& memory_provider) + // Returns true, if it deallocation can proceed without changing any status bits. + // Note that this does remove the use from the meta slab, so it doesn't need doing + // on the slow path. + ALWAYSINLINE bool dealloc_fast(Superslab* super, void* p) { Metaslab& meta = super->get_meta(this); - - bool was_full = meta.is_full(); - #ifdef CHECK_CLIENT if (meta.is_unused()) error("Detected potential double free."); #endif - meta.debug_slab_invariant(is_short(), this); meta.sub_use(); + bool was_full = meta.is_full(); + if (unlikely(was_full)) return false; + + bool is_unused = meta.is_unused(); + if (unlikely(is_unused)) return false; + + // Update the head and the next pointer in the free list. + uint16_t head = meta.head; + uint16_t current = pointer_to_index(p); + + // Set the head to the memory being deallocated. + meta.head = current; + assert(meta.valid_head(is_short())); + + // Set the next pointer to the previous head. + Metaslab::store_next(p, pointer_offset(this, head)); + meta.debug_slab_invariant(is_short(), this); + return true; + } + + // If dealloc fast returns false, then call this. + // This does not need to remove the "use" as done by the fast path. + // Returns a complex return code for managing the superslab meta data. + // i.e. This deallocation could make an entire superslab free. + template + NOINLINE typename Superslab::Action dealloc_slow( + SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider) + { + Metaslab& meta = super->get_meta(this); + + bool was_full = meta.is_full(); + bool is_unused = meta.is_unused(); + if (was_full) { // We are not on the sizeclass list. - if (!meta.is_unused()) - { - // Update the head and the sizeclass link. - uint16_t index = pointer_to_index(p); - assert(meta.head == 1); - meta.link = index; - - // Push on the list of slabs for this sizeclass. - sc->insert(meta.get_link(this)); - meta.debug_slab_invariant(is_short(), this); - } - else + if (is_unused) { // Dealloc on the superslab. if (is_short()) @@ -131,37 +198,30 @@ namespace snmalloc return super->dealloc_slab(this, memory_provider); } + // Update the head and the sizeclass link. + uint16_t index = pointer_to_index(p); + assert(meta.head == 1); +// assert(meta.fully_allocated(is_short())); + meta.link = index; + + // Push on the list of slabs for this sizeclass. + sl->insert(meta.get_link(this)); + meta.debug_slab_invariant(is_short(), this); + return Superslab::NoSlabReturn; } - else if (meta.is_unused()) + + if (is_unused) { // Remove from the sizeclass list and dealloc on the superslab. - sc->remove(meta.get_link(this)); + sl->remove(meta.get_link(this)); if (is_short()) return super->dealloc_short_slab(memory_provider); return super->dealloc_slab(this, memory_provider); } - else - { -#ifndef NDEBUG - sc->debug_check_contains(meta.get_link(this)); -#endif - - // Update the head and the next pointer in the free list. - uint16_t head = meta.head; - uint16_t current = pointer_to_index(p); - - // Set the head to the memory being deallocated. - meta.head = current; - assert(meta.valid_head(is_short())); - - // Set the next pointer to the previous head. - Metaslab::store_next(p, head); - - meta.debug_slab_invariant(is_short(), this); - } - return Superslab::NoSlabReturn; + + abort(); } bool is_short() diff --git a/src/mem/superslab.h b/src/mem/superslab.h index 2c8d8b0..66d65a5 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -160,11 +160,11 @@ namespace snmalloc if ((used & 1) == 1) return alloc_slab(sizeclass, memory_provider); - meta[0].head = get_initial_bumpptr(sizeclass, true); + meta[0].allocated = 1; + meta[0].head = 1; meta[0].sizeclass = static_cast(sizeclass); - meta[0].link = get_initial_link(sizeclass, true); + meta[0].link = get_initial_offset(sizeclass, true); - if constexpr (decommit_strategy == DecommitAll) { memory_provider.template notify_using( pointer_offset(this, OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE); @@ -183,9 +183,10 @@ namespace snmalloc uint8_t n = meta[h].next; - meta[h].head = get_initial_bumpptr(sizeclass, false); + meta[h].head = 1; + meta[h].allocated = 1; meta[h].sizeclass = static_cast(sizeclass); - meta[h].link = get_initial_link(sizeclass, false); + meta[h].link = get_initial_offset(sizeclass, false); head = h + n + 1; used += 2; diff --git a/src/override/malloc.cc b/src/override/malloc.cc index aba9ac3..1e55b84 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -23,9 +23,6 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) { - // Include size 0 in the first sizeclass. - size = ((size - 1) >> (bits::BITS - 1)) + size; - return ThreadAlloc::get()->alloc(size); } @@ -46,8 +43,6 @@ extern "C" errno = ENOMEM; return nullptr; } - // Include size 0 in the first sizeclass. - sz = ((sz - 1) >> (bits::BITS - 1)) + sz; return ThreadAlloc::get()->alloc(sz); }