From 44416ed70e5179b3c2c58c9e3b358992a26fcd60 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 25 Aug 2021 14:49:06 +0100 Subject: [PATCH] Factor sizeclass meta-data for cache locality This commit splits the sizeclass meta-data to generate better cache locality for various lookups for checking for size and start of sizeclasses. Also, contains some tidying including removing sizeclasses covering large range. This is left over from an alternative design for large classes that is no longer in use. --- src/mem/localalloc.h | 4 +- src/mem/sizeclasstable.h | 213 ++++++++++++++------------- src/test/func/memory/memory.cc | 2 +- src/test/func/sizeclass/sizeclass.cc | 3 +- 4 files changed, 115 insertions(+), 107 deletions(-) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index ae25905..8e0978a 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -189,8 +189,8 @@ namespace snmalloc // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING - std::cout << "size " << size << " sizeclass " << size_to_sizeclass(size) - << std::endl; + std::cout << "size " << size << " pow2 size " + << bits::next_pow2_bits(size) << std::endl; #endif // Note that meta data is not currently used for large allocs. diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 97f224d..3e08c44 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -30,6 +30,7 @@ namespace snmalloc static inline size_t large_sizeclass_to_size(uint8_t large_class) { + // TODO. Remove UNUSED(large_class); abort(); // return bits::one_at_bit(large_class + SUPERSLAB_BITS); @@ -53,98 +54,77 @@ namespace snmalloc return ((alignment - 1) | (size - 1)) + 1; } - constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s) + /** + * This structure contains the fields required for fast paths for sizeclasses. + */ + struct sizeclass_data_fast { - // We subtract and shift to reduce the size of the table, i.e. we don't have - // to store a value for every size. - return (s - 1) >> MIN_ALLOC_BITS; - } + size_t size; + // We store the mask as it is used more on the fast path, and the size of + // the slab. + size_t slab_mask; + // Table of constants for reciprocal division for each sizeclass. + size_t div_mult; + // Table of constants for reciprocal modulus for each sizeclass. + size_t mod_mult; + }; - constexpr static size_t NUM_SIZECLASSES_EXTENDED = - size_to_sizeclass_const(bits::one_at_bit(bits::ADDRESS_BITS - 1)); - - constexpr static size_t sizeclass_lookup_size = - sizeclass_lookup_index(MAX_SIZECLASS_SIZE); + /** + * This structure contains the remaining fields required for slow paths for + * sizeclasses. + */ + struct sizeclass_data_slow + { + uint16_t capacity; + uint16_t waking; + }; struct SizeClassTable { - sizeclass_compress_t sizeclass_lookup[sizeclass_lookup_size] = {{}}; - ModArray size; + ModArray fast; + ModArray slow; - ModArray capacity; - ModArray waking; - // We store the mask as it is used more on the fast path, and the size of - // the slab. - ModArray slab_mask; - - // Table of constants for reciprocal division for each sizeclass. - ModArray div_mult; - // Table of constants for reciprocal modulus for each sizeclass. - ModArray mod_mult; - - constexpr SizeClassTable() - : size(), capacity(), waking(), slab_mask(), div_mult(), mod_mult() + constexpr SizeClassTable() : fast(), slow() { for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { size_t rsize = bits::from_exp_mant(sizeclass); - size[sizeclass] = rsize; + fast[sizeclass].size = rsize; size_t slab_bits = bits::max( bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS); - slab_mask[sizeclass] = bits::one_at_bit(slab_bits) - 1; + fast[sizeclass].slab_mask = bits::one_at_bit(slab_bits) - 1; - capacity[sizeclass] = - static_cast((slab_mask[sizeclass] + 1) / rsize); + slow[sizeclass].capacity = + static_cast((fast[sizeclass].slab_mask + 1) / rsize); - waking[sizeclass] = + slow[sizeclass].waking = #ifdef SNMALLOC_CHECK_CLIENT - static_cast(capacity[sizeclass] / 4); + static_cast(slow[sizeclass].capacity / 4); #else - static_cast(bits::min((capacity[sizeclass] / 4), 32)); + static_cast(bits::min((slow[sizeclass].capacity / 4), 32)); #endif } - for (sizeclass_compress_t sizeclass = NUM_SIZECLASSES; - sizeclass < NUM_SIZECLASSES_EXTENDED; - sizeclass++) - { - size[sizeclass] = bits::prev_pow2_const( - bits::from_exp_mant(sizeclass)); - } - for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { - div_mult[sizeclass] = // TODO is MAX_SIZECLASS_BITS right? + fast[sizeclass].div_mult = // TODO is MAX_SIZECLASS_BITS right? (bits::one_at_bit(bits::BITS - 24) / - (size[sizeclass] / MIN_ALLOC_SIZE)); - if (!bits::is_pow2(size[sizeclass])) - div_mult[sizeclass]++; + (fast[sizeclass].size / MIN_ALLOC_SIZE)); + if (!bits::is_pow2(fast[sizeclass].size)) + fast[sizeclass].div_mult++; - mod_mult[sizeclass] = - (bits::one_at_bit(bits::BITS - 1) / size[sizeclass]); - if (!bits::is_pow2(size[sizeclass])) - mod_mult[sizeclass]++; + fast[sizeclass].mod_mult = + (bits::one_at_bit(bits::BITS - 1) / fast[sizeclass].size); + if (!bits::is_pow2(fast[sizeclass].size)) + fast[sizeclass].mod_mult++; // Shift multiplier, so that the result of division completely // overflows, and thus the top SUPERSLAB_BITS will be zero if the mod is // zero. - mod_mult[sizeclass] *= 2; - } - - size_t curr = 1; - for (sizeclass_compress_t sizeclass = 0; sizeclass <= NUM_SIZECLASSES; - sizeclass++) - { - for (; curr <= size[sizeclass]; curr += 1 << MIN_ALLOC_BITS) - { - auto i = sizeclass_lookup_index(curr); - if (i == sizeclass_lookup_size) - break; - sizeclass_lookup[i] = sizeclass; - } + fast[sizeclass].mod_mult *= 2; } } }; @@ -153,12 +133,12 @@ namespace snmalloc constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) { - return sizeclass_metadata.size[sizeclass]; + return sizeclass_metadata.fast[sizeclass].size; } inline static size_t sizeclass_to_slab_size(sizeclass_t sizeclass) { - return sizeclass_metadata.slab_mask[sizeclass] + 1; + return sizeclass_metadata.fast[sizeclass].slab_mask + 1; } /** @@ -170,12 +150,7 @@ namespace snmalloc */ inline uint16_t threshold_for_waking_slab(sizeclass_t sizeclass) { - // #ifdef SNMALLOC_CHECK_CLIENT - return sizeclass_metadata.waking[sizeclass]; - // #else - // UNUSED(sizeclass); - // return 1; - // #endif + return sizeclass_metadata.slow[sizeclass].waking; } inline static size_t sizeclass_to_slab_sizeclass(sizeclass_t sizeclass) @@ -193,26 +168,7 @@ namespace snmalloc inline constexpr static uint16_t sizeclass_to_slab_object_count(sizeclass_t sizeclass) { - return sizeclass_metadata.capacity[sizeclass]; - } - - static inline sizeclass_t size_to_sizeclass(size_t size) - { - auto index = sizeclass_lookup_index(size); - if (index < sizeclass_lookup_size) - { - return sizeclass_metadata.sizeclass_lookup[index]; - } - - // Don't use sizeclasses that are not a multiple of the alignment. - // For example, 24 byte allocations can be - // problematic for some data due to alignment issues. - - // TODO hack to power of 2 for large sizes - size = bits::next_pow2(size); - - return static_cast( - bits::to_exp_mant(size)); + return sizeclass_metadata.slow[sizeclass].capacity; } inline static size_t round_by_sizeclass(sizeclass_t sc, size_t offset) @@ -236,7 +192,8 @@ namespace snmalloc // SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); // TODO 24 hack - return (((offset >> MIN_ALLOC_BITS) * sizeclass_metadata.div_mult[sc]) >> + return (((offset >> MIN_ALLOC_BITS) * + sizeclass_metadata.fast[sc].div_mult) >> (bits::BITS - 24)) * rsize; } @@ -265,7 +222,7 @@ namespace snmalloc static constexpr size_t MASK = ~(bits::one_at_bit(bits::BITS - 1 - 24) - 1); - return ((offset * sizeclass_metadata.mod_mult[sc]) & MASK) == 0; + return ((offset * sizeclass_metadata.fast[sc].mod_mult) & MASK) == 0; } else // Use 32-bit division as considerably faster than 64-bit, and @@ -273,6 +230,68 @@ namespace snmalloc return static_cast(offset % sizeclass_to_size(sc)) == 0; } + inline static size_t large_size_to_chunk_size(size_t size) + { + return bits::next_pow2(size); + } + + inline static size_t large_size_to_chunk_sizeclass(size_t size) + { + return bits::next_pow2_bits(size) - MIN_CHUNK_BITS; + } + + constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s) + { + // We subtract and shift to reduce the size of the table, i.e. we don't have + // to store a value for every size. + return (s - 1) >> MIN_ALLOC_BITS; + } + + static inline sizeclass_t size_to_sizeclass(size_t size) + { + constexpr static size_t sizeclass_lookup_size = + sizeclass_lookup_index(MAX_SIZECLASS_SIZE); + + /** + * This struct is used to statically initialise a table for looking up + * the correct sizeclass. + */ + struct SizeClassLookup + { + sizeclass_compress_t table[sizeclass_lookup_size] = {{}}; + + constexpr SizeClassLookup() + { + size_t curr = 1; + for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + sizeclass++) + { + for (; curr <= sizeclass_metadata.fast[sizeclass].size; + curr += 1 << MIN_ALLOC_BITS) + { + auto i = sizeclass_lookup_index(curr); + if (i == sizeclass_lookup_size) + break; + table[i] = sizeclass; + } + } + } + }; + + static constexpr SizeClassLookup sizeclass_lookup = SizeClassLookup(); + + auto index = sizeclass_lookup_index(size); + if (index < sizeclass_lookup_size) + { + return sizeclass_lookup.table[index]; + } + + // Check this is not called on large sizes. + SNMALLOC_ASSERT(size == 0); + // Map size == 0 to the first sizeclass. + return 0; + } + inline SNMALLOC_FAST_PATH static size_t round_size(size_t size) { if (size > sizeclass_to_size(NUM_SIZECLASSES - 1)) @@ -295,14 +314,4 @@ namespace snmalloc return 1; return bits::one_at_bit(bits::ctz(rsize)); } - - inline static size_t large_size_to_chunk_size(size_t size) - { - return bits::next_pow2(size); - } - - inline static size_t large_size_to_chunk_sizeclass(size_t size) - { - return bits::next_pow2_bits(size) - MIN_CHUNK_BITS; - } } // namespace snmalloc diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index cc2787c..cf65096 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -415,7 +415,7 @@ void test_static_sized_allocs() { // For each small, medium, and large class, do each kind dealloc. This is // mostly to ensure that all of these forms compile. - for (size_t sc = 0; sc < NUM_SIZECLASSES_EXTENDED; sc++) + for (size_t sc = 0; sc < NUM_SIZECLASSES; sc++) { // test_static_sized_alloc(); // test_static_sized_alloc(); diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 3d4709e..8b2faf3 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -78,9 +78,8 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; size_t slab_size = 0; - for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES + 20; sz++) + for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++) { - // Separate printing for small and medium sizeclasses if ( sz < snmalloc::NUM_SIZECLASSES && slab_size != snmalloc::sizeclass_to_slab_size(sz))