From 59edf294d035b2edb4dc8bf6704628c4e4c05d3a Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 12 Mar 2021 21:11:43 +0000 Subject: [PATCH] Move link out of object space. The link object was previously stored in a disused object. This is good for reducing meta-data, but if we want to reduce the meta-data corruption potential, then this is not a good design choice. This commit moves it into the Metaslab. --- src/ds/cdllist.h | 11 ++++++ src/mem/alloc.h | 8 ++--- src/mem/metaslab.h | 85 +++++++++++++++++++++++++++++---------------- src/mem/slab.h | 72 +++----------------------------------- src/mem/superslab.h | 6 ++-- 5 files changed, 75 insertions(+), 107 deletions(-) diff --git a/src/ds/cdllist.h b/src/ds/cdllist.h index 7fe4795..b1e79f7 100644 --- a/src/ds/cdllist.h +++ b/src/ds/cdllist.h @@ -110,6 +110,17 @@ namespace snmalloc #endif } + /** + * Nulls the previous pointer + * + * The Meta-slab uses nullptr in prev to mean that it is not part of a + * size class list. + **/ + void null_prev() + { + prev = nullptr; + } + SNMALLOC_FAST_PATH CDLLNode* get_prev() { return prev; diff --git a/src/mem/alloc.h b/src/mem/alloc.h index c336a1f..e245d47 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1074,18 +1074,14 @@ namespace snmalloc size_t rsize = sizeclass_to_size(sizeclass); auto& sl = small_classes[sizeclass]; - Slab* slab; - if (likely(!sl.is_empty())) { stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); - SlabLink* link = sl.get_next(); - slab = get_slab(link); + auto meta = reinterpret_cast(sl.get_next()); auto& ffl = small_fast_free_lists[sizeclass]; - return slab->alloc( - sl, ffl, rsize); + return meta->alloc(ffl, rsize); } return small_alloc_rare(sizeclass, size); } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 0e36c03..6751231 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -9,6 +9,12 @@ namespace snmalloc { class Slab; + struct FreeListHead + { + // Use a value with bottom bit set for empty list. + void* value = nullptr; + }; + using SlabList = CDLLNode; using SlabLink = CDLLNode; @@ -23,14 +29,13 @@ namespace snmalloc // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. - class Metaslab + class Metaslab : public SlabLink { public: /** * Pointer to first free entry in this slab * - * The list will be (allocated - needed - 1) long. The -1 is - * for the `link` element which is not in the free list. + * The list will be (allocated - needed) long. */ void* head = nullptr; @@ -38,7 +43,7 @@ namespace snmalloc * How many entries are not in the free list of slab, i.e. * how many entries are needed to fully free this slab. * - * In the case of a fully allocated slab, where link==1 needed + * In the case of a fully allocated slab, where prev==0 needed * will be 1. This enables 'return_object' to detect the slow path * case with a single operation subtract and test. */ @@ -49,11 +54,6 @@ namespace snmalloc */ uint16_t allocated = 0; - // When a slab has free space it will be on the has space list for - // that size class. We use an empty block in this slab to be the - // doubly linked node into that size class's free list. - Mod link; - uint8_t sizeclass; // Initially zero to encode the superslabs relative list of slabs. uint8_t next = 0; @@ -77,7 +77,7 @@ namespace snmalloc bool is_full() { - auto result = link == 1; + auto result = get_prev() == nullptr; SNMALLOC_ASSERT(!result || head == nullptr); return result; } @@ -85,16 +85,10 @@ namespace snmalloc SNMALLOC_FAST_PATH void set_full() { SNMALLOC_ASSERT(head == nullptr); - SNMALLOC_ASSERT(link != 1); - link = 1; // Set needed to 1, so that "return_object" will return true after calling // set_full needed = 1; - } - - SlabLink* get_link(Slab* slab) - { - return reinterpret_cast(pointer_offset(slab, link)); + null_prev(); } /// Value used to check for corruptions in a block @@ -157,6 +151,51 @@ namespace snmalloc pointer_diff(p, pointer_align_up(pointer_offset(p, 1)))); } + /** + * Takes a free list out of a slabs meta data. + * Returns the link as the allocation, and places the free list into the + * `fast_free_list` for further allocations. + */ + template + SNMALLOC_FAST_PATH void* alloc(FreeListHead& fast_free_list, size_t rsize) + { + SNMALLOC_ASSERT(rsize == sizeclass_to_size(sizeclass)); + SNMALLOC_ASSERT(!is_full()); + + auto slab = get_slab(head); + debug_slab_invariant(slab); + + // Use first element as the allocation + void* p = head; + // Put the rest in allocators small_class fast free list. + fast_free_list.value = Metaslab::follow_next(p); + head = nullptr; + + // Treat stealing the free list as allocating it all. + needed = allocated; + remove(); + set_full(); + + p = remove_cache_friendly_offset(p, sizeclass); + SNMALLOC_ASSERT(is_start_of_object(p)); + + debug_slab_invariant(slab); + + if constexpr (zero_mem == YesZero) + { + if (rsize < PAGE_ALIGNED_SIZE) + PAL::zero(p, rsize); + else + PAL::template zero(p, rsize); + } + else + { + UNUSED(rsize); + } + + return p; + } + /** * Check bump-free-list-segment for cycles * @@ -207,7 +246,6 @@ namespace snmalloc if (is_full()) { // There is no free list to validate - // 'link' value is not important if full. return; } @@ -236,8 +274,6 @@ namespace snmalloc // Account for free elements in free list accounted_for += size; SNMALLOC_ASSERT(SLAB_SIZE >= accounted_for); - // We should never reach the link node in the free list. - SNMALLOC_ASSERT(curr != pointer_offset(slab, link)); // Iterate bump/free list segment curr = follow_next(curr); @@ -250,16 +286,7 @@ namespace snmalloc // Account for to be bump allocated space accounted_for += SLAB_SIZE - bumpptr; - if (bumpptr != SLAB_SIZE) - { - // The link should be the first allocation as we - // haven't completely filled this block at any point. - SNMALLOC_ASSERT(link == get_initial_offset(sizeclass, is_short)); - } - SNMALLOC_ASSERT(!is_full()); - // Add the link node. - accounted_for += size; // All space accounted for SNMALLOC_ASSERT(SLAB_SIZE == accounted_for); diff --git a/src/mem/slab.h b/src/mem/slab.h index 025feb6..5f6efd6 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -4,12 +4,6 @@ namespace snmalloc { - struct FreeListHead - { - // Use a value with bottom bit set for empty list. - void* value = nullptr; - }; - class Slab { private: @@ -26,62 +20,6 @@ namespace snmalloc return super->get_meta(this); } - SlabLink* get_link() - { - return get_meta().get_link(this); - } - - /** - * Takes a free list out of a slabs meta data. - * Returns the link as the allocation, and places the free list into the - * `fast_free_list` for further allocations. - */ - template - SNMALLOC_FAST_PATH void* - alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize) - { - // Read the head from the metadata stored in the superslab. - Metaslab& meta = get_meta(); - SNMALLOC_ASSERT(meta.link != 1); - - SNMALLOC_ASSERT(rsize == sizeclass_to_size(meta.sizeclass)); - SNMALLOC_ASSERT( - sl.get_next() == (SlabLink*)pointer_offset(this, meta.link)); - SNMALLOC_ASSERT(!meta.is_full()); - meta.debug_slab_invariant(this); - - // Put everything in allocators small_class free list. - fast_free_list.value = meta.head; - meta.head = nullptr; - - // Return the link as the node for this allocation. - void* link = pointer_offset(this, meta.link); - void* p = remove_cache_friendly_offset(link, meta.sizeclass); - - // Treat stealing the free list as allocating it all. - meta.needed = meta.allocated; - meta.set_full(); - sl.get_next()->remove(); - - SNMALLOC_ASSERT(meta.is_start_of_object(p)); - - meta.debug_slab_invariant(this); - - if constexpr (zero_mem == YesZero) - { - if (rsize < PAGE_ALIGNED_SIZE) - PAL::zero(p, rsize); - else - PAL::template zero(p, rsize); - } - else - { - UNUSED(rsize); - } - - return p; - } - /** * Given a bumpptr and a fast_free_list head reference, builds a new free * list, and stores it in the fast_free_list. It will only create a page @@ -158,21 +96,19 @@ namespace snmalloc return super->dealloc_slab(this); } - // Update the head and the sizeclass link. - uint16_t index = pointer_to_index(p); SNMALLOC_ASSERT(meta.head == nullptr); - // SNMALLOC_ASSERT(meta.fully_allocated(is_short())); - meta.link = index; + meta.head = p; + Metaslab::store_next(p, nullptr); meta.needed = meta.allocated - 1; // Push on the list of slabs for this sizeclass. - sl->insert_prev(meta.get_link(this)); + sl->insert_prev(&meta); meta.debug_slab_invariant(this); return Superslab::NoSlabReturn; } // Remove from the sizeclass list and dealloc on the superslab. - meta.get_link(this)->remove(); + meta.remove(); if (is_short()) return super->dealloc_short_slab(); diff --git a/src/mem/superslab.h b/src/mem/superslab.h index 93aabb9..cfe1e8e 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -173,8 +173,7 @@ namespace snmalloc meta[0].allocated = static_cast( (SLAB_SIZE - get_initial_offset(sizeclass, true)) / sizeclass_to_size(sizeclass)); - meta[0].link = 1; - meta[0].needed = 1; + meta[0].set_full(); meta[0].sizeclass = static_cast(sizeclass); used++; @@ -198,8 +197,7 @@ namespace snmalloc meta[h].allocated = static_cast( (SLAB_SIZE - get_initial_offset(sizeclass, false)) / sizeclass_to_size(sizeclass)); - meta[h].needed = 1; - meta[h].link = 1; + meta[h].set_full(); meta[h].sizeclass = static_cast(sizeclass); head = h + n + 1;