diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index e258543..e933b19 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -94,12 +94,11 @@ namespace snmalloc bool valid_head(bool is_short) { size_t size = sizeclass_to_size(sizeclass); - size_t offset = get_slab_offset(sizeclass, is_short); + size_t slab_start = get_initial_link(sizeclass, is_short); size_t all_high_bits = ~static_cast(1); size_t head_start = remove_cache_friendly_offset(head & all_high_bits, sizeclass); - size_t slab_start = offset & all_high_bits; return ((head_start - slab_start) % size) == 0; } @@ -108,7 +107,7 @@ namespace snmalloc { #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) size_t size = sizeclass_to_size(sizeclass); - size_t offset = get_slab_offset(sizeclass, is_short) - 1; + size_t offset = get_initial_link(sizeclass, is_short); size_t accounted_for = used * size + offset; @@ -152,7 +151,7 @@ namespace snmalloc // The link should be the first allocation as we // haven't completely filled this block at any point. - assert(link == (get_slab_offset(sizeclass, is_short) - 1)); + assert(link == get_initial_link(sizeclass, is_short)); } assert(!is_full()); diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 77ce364..724ac28 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -4,7 +4,8 @@ namespace snmalloc { - constexpr static uint16_t get_slab_offset(uint8_t sc, bool is_short); + constexpr static uint16_t get_initial_bumpptr(uint8_t sc, bool is_short); + constexpr static uint16_t get_initial_link(uint8_t sc, bool is_short); constexpr static size_t sizeclass_to_size(uint8_t sizeclass); constexpr static size_t sizeclass_to_cache_friendly_mask(uint8_t sizeclass); constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(uint8_t sc); diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index a9a80c6..06c80aa 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -12,7 +12,8 @@ namespace snmalloc ModArray inverse_cache_friendly_mask; ModArray bump_ptr_start; ModArray short_bump_ptr_start; - ModArray count_per_slab; + ModArray initial_link_ptr; + ModArray short_initial_link_ptr; ModArray medium_slab_slots; constexpr SizeClassTable() @@ -21,7 +22,8 @@ namespace snmalloc inverse_cache_friendly_mask(), bump_ptr_start(), short_bump_ptr_start(), - count_per_slab(), + initial_link_ptr(), + short_initial_link_ptr(), medium_slab_slots() { for (uint8_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) @@ -40,10 +42,25 @@ namespace snmalloc for (uint8_t i = 0; i < NUM_SMALL_CLASSES; i++) { + // We align to the end of the block to remove special cases for the + // short block. Calculate remainders + size_t short_correction = short_slab_size % size[i]; + size_t correction = SLAB_SIZE % size[i]; + + // First element in the block is the link + initial_link_ptr[i] = static_cast(correction); + short_initial_link_ptr[i] = + static_cast(header_size + short_correction); + + // Move to object after link. + auto short_after_link = short_initial_link_ptr[i] + size[i]; + size_t after_link = initial_link_ptr[i] + size[i]; + + // Bump ptr has bottom bit set. + // In case we only have one object on this slab check for wrap around. short_bump_ptr_start[i] = - static_cast(1 + (short_slab_size % size[i]) + header_size); - bump_ptr_start[i] = static_cast(1 + (SLAB_SIZE % size[i])); - count_per_slab[i] = static_cast(SLAB_SIZE / size[i]); + static_cast((short_after_link + 1) % SLAB_SIZE); + bump_ptr_start[i] = static_cast((after_link + 1) % SLAB_SIZE); } for (uint8_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) @@ -56,7 +73,8 @@ namespace snmalloc static constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); - static inline constexpr uint16_t get_slab_offset(uint8_t sc, bool is_short) + static inline constexpr uint16_t + get_initial_bumpptr(uint8_t sc, bool is_short) { if (is_short) return sizeclass_metadata.short_bump_ptr_start[sc]; @@ -64,6 +82,14 @@ namespace snmalloc return sizeclass_metadata.bump_ptr_start[sc]; } + static inline constexpr uint16_t get_initial_link(uint8_t sc, bool is_short) + { + if (is_short) + return sizeclass_metadata.short_initial_link_ptr[sc]; + + return sizeclass_metadata.initial_link_ptr[sc]; + } + constexpr static inline size_t sizeclass_to_size(uint8_t sizeclass) { return sizeclass_metadata.size[sizeclass]; @@ -81,11 +107,6 @@ namespace snmalloc return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass]; } - constexpr static inline size_t sizeclass_to_count(uint8_t sizeclass) - { - return sizeclass_metadata.count_per_slab[sizeclass]; - } - constexpr static inline uint16_t medium_slab_free(uint8_t sizeclass) { return sizeclass_metadata diff --git a/src/mem/superslab.h b/src/mem/superslab.h index afa9e98..c131d78 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -159,12 +159,9 @@ namespace snmalloc if ((used & 1) == 1) return alloc_slab(sizeclass, memory_provider); - auto rsize = sizeclass_to_size(sizeclass); - - meta[0].head = - (get_slab_offset(sizeclass, true) + rsize) & (SLAB_SIZE - 1); + meta[0].head = get_initial_bumpptr(sizeclass, true); meta[0].sizeclass = sizeclass; - meta[0].link = get_slab_offset(sizeclass, true) - 1; + meta[0].link = get_initial_link(sizeclass, true); if constexpr (decommit_strategy == DecommitAll) { @@ -185,11 +182,9 @@ namespace snmalloc uint8_t n = meta[h].next; - auto rsize = sizeclass_to_size(sizeclass); - meta[h].head = - (get_slab_offset(sizeclass, false) + rsize) & (SLAB_SIZE - 1); + meta[h].head = get_initial_bumpptr(sizeclass, false); meta[h].sizeclass = sizeclass; - meta[h].link = get_slab_offset(sizeclass, false) - 1; + meta[h].link = get_initial_link(sizeclass, false); head = h + n + 1; used += 2;