diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index ae25905..8e0978a 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -189,8 +189,8 @@ namespace snmalloc // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING - std::cout << "size " << size << " sizeclass " << size_to_sizeclass(size) - << std::endl; + std::cout << "size " << size << " pow2 size " + << bits::next_pow2_bits(size) << std::endl; #endif // Note that meta data is not currently used for large allocs. diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 97f224d..3e08c44 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -30,6 +30,7 @@ namespace snmalloc static inline size_t large_sizeclass_to_size(uint8_t large_class) { + // TODO. Remove UNUSED(large_class); abort(); // return bits::one_at_bit(large_class + SUPERSLAB_BITS); @@ -53,98 +54,77 @@ namespace snmalloc return ((alignment - 1) | (size - 1)) + 1; } - constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s) + /** + * This structure contains the fields required for fast paths for sizeclasses. + */ + struct sizeclass_data_fast { - // We subtract and shift to reduce the size of the table, i.e. we don't have - // to store a value for every size. - return (s - 1) >> MIN_ALLOC_BITS; - } + size_t size; + // We store the mask as it is used more on the fast path, and the size of + // the slab. + size_t slab_mask; + // Table of constants for reciprocal division for each sizeclass. + size_t div_mult; + // Table of constants for reciprocal modulus for each sizeclass. + size_t mod_mult; + }; - constexpr static size_t NUM_SIZECLASSES_EXTENDED = - size_to_sizeclass_const(bits::one_at_bit(bits::ADDRESS_BITS - 1)); - - constexpr static size_t sizeclass_lookup_size = - sizeclass_lookup_index(MAX_SIZECLASS_SIZE); + /** + * This structure contains the remaining fields required for slow paths for + * sizeclasses. + */ + struct sizeclass_data_slow + { + uint16_t capacity; + uint16_t waking; + }; struct SizeClassTable { - sizeclass_compress_t sizeclass_lookup[sizeclass_lookup_size] = {{}}; - ModArray size; + ModArray fast; + ModArray slow; - ModArray capacity; - ModArray waking; - // We store the mask as it is used more on the fast path, and the size of - // the slab. - ModArray slab_mask; - - // Table of constants for reciprocal division for each sizeclass. - ModArray div_mult; - // Table of constants for reciprocal modulus for each sizeclass. - ModArray mod_mult; - - constexpr SizeClassTable() - : size(), capacity(), waking(), slab_mask(), div_mult(), mod_mult() + constexpr SizeClassTable() : fast(), slow() { for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { size_t rsize = bits::from_exp_mant(sizeclass); - size[sizeclass] = rsize; + fast[sizeclass].size = rsize; size_t slab_bits = bits::max( bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS); - slab_mask[sizeclass] = bits::one_at_bit(slab_bits) - 1; + fast[sizeclass].slab_mask = bits::one_at_bit(slab_bits) - 1; - capacity[sizeclass] = - static_cast((slab_mask[sizeclass] + 1) / rsize); + slow[sizeclass].capacity = + static_cast((fast[sizeclass].slab_mask + 1) / rsize); - waking[sizeclass] = + slow[sizeclass].waking = #ifdef SNMALLOC_CHECK_CLIENT - static_cast(capacity[sizeclass] / 4); + static_cast(slow[sizeclass].capacity / 4); #else - static_cast(bits::min((capacity[sizeclass] / 4), 32)); + static_cast(bits::min((slow[sizeclass].capacity / 4), 32)); #endif } - for (sizeclass_compress_t sizeclass = NUM_SIZECLASSES; - sizeclass < NUM_SIZECLASSES_EXTENDED; - sizeclass++) - { - size[sizeclass] = bits::prev_pow2_const( - bits::from_exp_mant(sizeclass)); - } - for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { - div_mult[sizeclass] = // TODO is MAX_SIZECLASS_BITS right? + fast[sizeclass].div_mult = // TODO is MAX_SIZECLASS_BITS right? (bits::one_at_bit(bits::BITS - 24) / - (size[sizeclass] / MIN_ALLOC_SIZE)); - if (!bits::is_pow2(size[sizeclass])) - div_mult[sizeclass]++; + (fast[sizeclass].size / MIN_ALLOC_SIZE)); + if (!bits::is_pow2(fast[sizeclass].size)) + fast[sizeclass].div_mult++; - mod_mult[sizeclass] = - (bits::one_at_bit(bits::BITS - 1) / size[sizeclass]); - if (!bits::is_pow2(size[sizeclass])) - mod_mult[sizeclass]++; + fast[sizeclass].mod_mult = + (bits::one_at_bit(bits::BITS - 1) / fast[sizeclass].size); + if (!bits::is_pow2(fast[sizeclass].size)) + fast[sizeclass].mod_mult++; // Shift multiplier, so that the result of division completely // overflows, and thus the top SUPERSLAB_BITS will be zero if the mod is // zero. - mod_mult[sizeclass] *= 2; - } - - size_t curr = 1; - for (sizeclass_compress_t sizeclass = 0; sizeclass <= NUM_SIZECLASSES; - sizeclass++) - { - for (; curr <= size[sizeclass]; curr += 1 << MIN_ALLOC_BITS) - { - auto i = sizeclass_lookup_index(curr); - if (i == sizeclass_lookup_size) - break; - sizeclass_lookup[i] = sizeclass; - } + fast[sizeclass].mod_mult *= 2; } } }; @@ -153,12 +133,12 @@ namespace snmalloc constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) { - return sizeclass_metadata.size[sizeclass]; + return sizeclass_metadata.fast[sizeclass].size; } inline static size_t sizeclass_to_slab_size(sizeclass_t sizeclass) { - return sizeclass_metadata.slab_mask[sizeclass] + 1; + return sizeclass_metadata.fast[sizeclass].slab_mask + 1; } /** @@ -170,12 +150,7 @@ namespace snmalloc */ inline uint16_t threshold_for_waking_slab(sizeclass_t sizeclass) { - // #ifdef SNMALLOC_CHECK_CLIENT - return sizeclass_metadata.waking[sizeclass]; - // #else - // UNUSED(sizeclass); - // return 1; - // #endif + return sizeclass_metadata.slow[sizeclass].waking; } inline static size_t sizeclass_to_slab_sizeclass(sizeclass_t sizeclass) @@ -193,26 +168,7 @@ namespace snmalloc inline constexpr static uint16_t sizeclass_to_slab_object_count(sizeclass_t sizeclass) { - return sizeclass_metadata.capacity[sizeclass]; - } - - static inline sizeclass_t size_to_sizeclass(size_t size) - { - auto index = sizeclass_lookup_index(size); - if (index < sizeclass_lookup_size) - { - return sizeclass_metadata.sizeclass_lookup[index]; - } - - // Don't use sizeclasses that are not a multiple of the alignment. - // For example, 24 byte allocations can be - // problematic for some data due to alignment issues. - - // TODO hack to power of 2 for large sizes - size = bits::next_pow2(size); - - return static_cast( - bits::to_exp_mant(size)); + return sizeclass_metadata.slow[sizeclass].capacity; } inline static size_t round_by_sizeclass(sizeclass_t sc, size_t offset) @@ -236,7 +192,8 @@ namespace snmalloc // SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); // TODO 24 hack - return (((offset >> MIN_ALLOC_BITS) * sizeclass_metadata.div_mult[sc]) >> + return (((offset >> MIN_ALLOC_BITS) * + sizeclass_metadata.fast[sc].div_mult) >> (bits::BITS - 24)) * rsize; } @@ -265,7 +222,7 @@ namespace snmalloc static constexpr size_t MASK = ~(bits::one_at_bit(bits::BITS - 1 - 24) - 1); - return ((offset * sizeclass_metadata.mod_mult[sc]) & MASK) == 0; + return ((offset * sizeclass_metadata.fast[sc].mod_mult) & MASK) == 0; } else // Use 32-bit division as considerably faster than 64-bit, and @@ -273,6 +230,68 @@ namespace snmalloc return static_cast(offset % sizeclass_to_size(sc)) == 0; } + inline static size_t large_size_to_chunk_size(size_t size) + { + return bits::next_pow2(size); + } + + inline static size_t large_size_to_chunk_sizeclass(size_t size) + { + return bits::next_pow2_bits(size) - MIN_CHUNK_BITS; + } + + constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s) + { + // We subtract and shift to reduce the size of the table, i.e. we don't have + // to store a value for every size. + return (s - 1) >> MIN_ALLOC_BITS; + } + + static inline sizeclass_t size_to_sizeclass(size_t size) + { + constexpr static size_t sizeclass_lookup_size = + sizeclass_lookup_index(MAX_SIZECLASS_SIZE); + + /** + * This struct is used to statically initialise a table for looking up + * the correct sizeclass. + */ + struct SizeClassLookup + { + sizeclass_compress_t table[sizeclass_lookup_size] = {{}}; + + constexpr SizeClassLookup() + { + size_t curr = 1; + for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + sizeclass++) + { + for (; curr <= sizeclass_metadata.fast[sizeclass].size; + curr += 1 << MIN_ALLOC_BITS) + { + auto i = sizeclass_lookup_index(curr); + if (i == sizeclass_lookup_size) + break; + table[i] = sizeclass; + } + } + } + }; + + static constexpr SizeClassLookup sizeclass_lookup = SizeClassLookup(); + + auto index = sizeclass_lookup_index(size); + if (index < sizeclass_lookup_size) + { + return sizeclass_lookup.table[index]; + } + + // Check this is not called on large sizes. + SNMALLOC_ASSERT(size == 0); + // Map size == 0 to the first sizeclass. + return 0; + } + inline SNMALLOC_FAST_PATH static size_t round_size(size_t size) { if (size > sizeclass_to_size(NUM_SIZECLASSES - 1)) @@ -295,14 +314,4 @@ namespace snmalloc return 1; return bits::one_at_bit(bits::ctz(rsize)); } - - inline static size_t large_size_to_chunk_size(size_t size) - { - return bits::next_pow2(size); - } - - inline static size_t large_size_to_chunk_sizeclass(size_t size) - { - return bits::next_pow2_bits(size) - MIN_CHUNK_BITS; - } } // namespace snmalloc diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index cc2787c..cf65096 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -415,7 +415,7 @@ void test_static_sized_allocs() { // For each small, medium, and large class, do each kind dealloc. This is // mostly to ensure that all of these forms compile. - for (size_t sc = 0; sc < NUM_SIZECLASSES_EXTENDED; sc++) + for (size_t sc = 0; sc < NUM_SIZECLASSES; sc++) { // test_static_sized_alloc(); // test_static_sized_alloc(); diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 3d4709e..8b2faf3 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -78,9 +78,8 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; size_t slab_size = 0; - for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES + 20; sz++) + for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++) { - // Separate printing for small and medium sizeclasses if ( sz < snmalloc::NUM_SIZECLASSES && slab_size != snmalloc::sizeclass_to_slab_size(sz))