diff --git a/src/ds/bits.h b/src/ds/bits.h index e6721cd..2503606 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -21,7 +21,7 @@ # include # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) -# define SNMALLOC_SLOW_PATH NOINLINE +# define SNMALLOC_SLOW_PATH NOINLINE __attribute__((section(".text,slow"))) # define SNMALLOC_FAST_PATH inline ALWAYSINLINE # define SNMALLOC_PURE __attribute__((const)) # ifdef __clang__ diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 94d209d..43e6ff7 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1072,7 +1072,7 @@ namespace snmalloc assert(sizeclass < NUM_SMALL_CLASSES); auto& fl = small_fast_free_lists[sizeclass]; auto head = fl.value; - if (likely((reinterpret_cast(head) & 1) == 0)) + if (likely(head != nullptr)) { void* p = head; // Read the next slot from the memory that's about to be allocated. diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index a7a60b9..7295324 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -146,10 +146,10 @@ namespace snmalloc { #ifndef NDEBUG size_t length = 0; - void* curr = pointer_offset(slab, head); - void* curr_slow = pointer_offset(slab, head); + void* curr = (head == 1) ? nullptr : pointer_offset(slab, head); + void* curr_slow = (head == 1) ? nullptr : pointer_offset(slab, head); bool both = false; - while ((reinterpret_cast(curr) & 1) == 0) + while (curr != nullptr) { curr = follow_next(curr); if (both) @@ -200,8 +200,8 @@ namespace snmalloc UNUSED(length); // Walk bump-free-list-segment accounting for unused space - void* curr = pointer_offset(slab, head); - while ((address_cast(curr) & 1) == 0) + void* curr = (head == 1) ? nullptr : pointer_offset(slab, head); + while (curr != nullptr) { // Check we are looking at a correctly aligned block void* start = curr; diff --git a/src/mem/slab.h b/src/mem/slab.h index 24c4fe2..0845d7c 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -7,7 +7,7 @@ namespace snmalloc struct FreeListHead { // Use a value with bottom bit set for empty list. - void* value = pointer_offset(nullptr, 1); + void* value = nullptr; }; class Slab @@ -98,7 +98,8 @@ namespace snmalloc } else { - Metaslab::store_next(curr, pointer_offset(this, bumpptr)); + Metaslab::store_next( + curr, (bumpptr == 1) ? nullptr : pointer_offset(this, bumpptr)); } curr = pointer_offset(this, bumpptr); bumpptr = newbumpptr; @@ -106,7 +107,7 @@ namespace snmalloc } assert(curr != nullptr); - Metaslab::store_next(curr, pointer_offset(nullptr, 1)); + Metaslab::store_next(curr, nullptr); } } @@ -177,7 +178,8 @@ namespace snmalloc assert(meta.valid_head(is_short())); // Set the next pointer to the previous head. - Metaslab::store_next(p, pointer_offset(this, head)); + Metaslab::store_next( + p, (head == 1) ? nullptr : pointer_offset(this, head)); meta.debug_slab_invariant(is_short(), this); return true; }