From 49fefc3f835dcea6781a44b1ac088689c82f7e2c Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Fri, 5 Mar 2021 00:12:18 +0000 Subject: [PATCH] NFC: introduce SlabNext type for intra-small-slab free pointers Mostly cosmetic, but eliminates some void*-s and makes intention clearer. --- src/mem/alloc.h | 11 ++++++----- src/mem/metaslab.h | 43 +++++++++++++++++++++++++++---------------- src/mem/slab.h | 23 +++++++++++++++-------- 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index e245d47..2f897c3 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -743,7 +743,7 @@ namespace snmalloc while (pointer_align_up(bp, SLAB_SIZE) != bp) { Slab::alloc_new_list(bp, ffl, rsize); - void* prev = ffl.value; + SlabNext* prev = ffl.value; while (prev != nullptr) { auto n = Metaslab::follow_next(prev); @@ -1027,7 +1027,7 @@ namespace snmalloc { SNMALLOC_ASSUME(sizeclass < NUM_SMALL_CLASSES); auto& fl = small_fast_free_lists[sizeclass]; - void* head = fl.value; + SlabNext* head = fl.value; if (likely(head != nullptr)) { stats().alloc_request(size); @@ -1148,7 +1148,8 @@ namespace snmalloc SNMALLOC_ASSERT(ffl.value == nullptr); Slab::alloc_new_list(bp, ffl, rsize); - void* p = remove_cache_friendly_offset(ffl.value, sizeclass); + SlabNext* p = static_cast( + remove_cache_friendly_offset(ffl.value, sizeclass)); ffl.value = Metaslab::follow_next(p); if constexpr (zero_mem == YesZero) @@ -1171,8 +1172,8 @@ namespace snmalloc Slab* slab = alloc_slab(sizeclass); if (slab == nullptr) return nullptr; - bp = - pointer_offset(slab, get_initial_offset(sizeclass, slab->is_short())); + bp = reinterpret_cast( + pointer_offset(slab, get_initial_offset(sizeclass, slab->is_short()))); return small_alloc_build_free_list(sizeclass); } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 6751231..86ea3d2 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -9,10 +9,22 @@ namespace snmalloc { class Slab; + /** + * Free objects within each slab point directly to the next (contrast + * SlabLink, which chain different Slabs of the same sizeclass together). + */ + struct SlabNext + { + struct SlabNext* next; +#ifdef CHECK_CLIENT + uintptr_t guard; +#endif + }; + struct FreeListHead { // Use a value with bottom bit set for empty list. - void* value = nullptr; + SlabNext* value = nullptr; }; using SlabList = CDLLNode; @@ -37,7 +49,7 @@ namespace snmalloc * * The list will be (allocated - needed) long. */ - void* head = nullptr; + SlabNext* head = nullptr; /** * How many entries are not in the free list of slab, i.e. @@ -97,31 +109,30 @@ namespace snmalloc /// Store next pointer in a block. In Debug using magic value to detect some /// simple corruptions. - static SNMALLOC_FAST_PATH void store_next(void* p, void* head) + static SNMALLOC_FAST_PATH void store_next(SlabNext* p, SlabNext* head) { - *static_cast(p) = head; + p->next = head; #if defined(CHECK_CLIENT) if constexpr (aal_supports) { - *(static_cast(p) + 1) = address_cast(head) ^ POISON; + p->guard = address_cast(head) ^ POISON; } #endif } /// Accessor function for the next pointer in a block. /// In Debug checks for simple corruptions. - static SNMALLOC_FAST_PATH void* follow_next(void* node) + static SNMALLOC_FAST_PATH SlabNext* follow_next(SlabNext* node) { #if defined(CHECK_CLIENT) if constexpr (aal_supports) { - uintptr_t next = *static_cast(node); - uintptr_t chk = *(static_cast(node) + 1); - if ((next ^ chk) != POISON) + uintptr_t next = address_cast(node->next); + if ((next ^ node->guard) != POISON) error("Detected memory corruption. Use-after-free."); } #endif - return *static_cast(node); + return node->next; } bool valid_head() @@ -166,9 +177,9 @@ namespace snmalloc debug_slab_invariant(slab); // Use first element as the allocation - void* p = head; + SlabNext* h = head; // Put the rest in allocators small_class fast free list. - fast_free_list.value = Metaslab::follow_next(p); + fast_free_list.value = Metaslab::follow_next(h); head = nullptr; // Treat stealing the free list as allocating it all. @@ -176,7 +187,7 @@ namespace snmalloc remove(); set_full(); - p = remove_cache_friendly_offset(p, sizeclass); + void* p = remove_cache_friendly_offset(h, sizeclass); SNMALLOC_ASSERT(is_start_of_object(p)); debug_slab_invariant(slab); @@ -208,8 +219,8 @@ namespace snmalloc { #ifndef NDEBUG size_t length = 0; - void* curr = head; - void* curr_slow = head; + SlabNext* curr = head; + SlabNext* curr_slow = head; bool both = false; while (curr != nullptr) { @@ -264,7 +275,7 @@ namespace snmalloc UNUSED(length); // Walk bump-free-list-segment accounting for unused space - void* curr = head; + SlabNext* curr = head; while (curr != nullptr) { // Check we are looking at a correctly aligned block diff --git a/src/mem/slab.h b/src/mem/slab.h index 5f6efd6..3d54d28 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -29,7 +29,9 @@ namespace snmalloc static SNMALLOC_FAST_PATH void alloc_new_list(void*& bumpptr, FreeListHead& fast_free_list, size_t rsize) { - fast_free_list.value = bumpptr; + auto snbumpptr = static_cast(bumpptr); + fast_free_list.value = snbumpptr; + void* newbumpptr = pointer_offset(bumpptr, rsize); void* slab_end = pointer_align_up(newbumpptr); void* slab_end2 = @@ -39,12 +41,14 @@ namespace snmalloc while (newbumpptr < slab_end) { - Metaslab::store_next(bumpptr, newbumpptr); + auto newsnbumpptr = static_cast(newbumpptr); + Metaslab::store_next(snbumpptr, newsnbumpptr); + snbumpptr = newsnbumpptr; bumpptr = newbumpptr; newbumpptr = pointer_offset(bumpptr, rsize); } - Metaslab::store_next(bumpptr, nullptr); + Metaslab::store_next(snbumpptr, nullptr); bumpptr = newbumpptr; } @@ -63,14 +67,16 @@ namespace snmalloc return false; // Update the head and the next pointer in the free list. - void* head = meta.head; + SlabNext* head = meta.head; + + SlabNext* psn = static_cast(p); // Set the head to the memory being deallocated. - meta.head = p; + meta.head = psn; SNMALLOC_ASSERT(meta.valid_head()); // Set the next pointer to the previous head. - Metaslab::store_next(p, head); + Metaslab::store_next(psn, head); return true; } @@ -97,8 +103,9 @@ namespace snmalloc return super->dealloc_slab(this); } SNMALLOC_ASSERT(meta.head == nullptr); - meta.head = p; - Metaslab::store_next(p, nullptr); + SlabNext* psn = static_cast(p); + meta.head = psn; + Metaslab::store_next(psn, nullptr); meta.needed = meta.allocated - 1; // Push on the list of slabs for this sizeclass.