diff --git a/src/mem/alloc.h b/src/mem/alloc.h index dce3f81..7f0b3a3 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -807,7 +807,7 @@ namespace snmalloc size_t offset_from_end = pointer_diff(p, pointer_offset_signed(end_point, -1)); - size_t end_to_end = round_by_sizeclass(rsize, offset_from_end); + size_t end_to_end = round_by_sizeclass(sizeclass, offset_from_end); return pointer_offset_signed( end_point_correction, -static_cast(end_to_end)); @@ -1380,8 +1380,7 @@ namespace snmalloc { check_client( is_multiple_of_sizeclass( - sizeclass_to_size(sizeclass), - pointer_diff(p, pointer_offset(slab, SUPERSLAB_SIZE))), + sizeclass, pointer_diff(p, pointer_offset(slab, SUPERSLAB_SIZE))), "Not deallocating start of an object"); medium_dealloc_start(slab, p, sizeclass); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 509604c..7121105 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -109,8 +109,8 @@ namespace snmalloc SNMALLOC_FAST_PATH static bool is_start_of_object(Metaslab* self, void* p) { return is_multiple_of_sizeclass( - sizeclass_to_size(self->sizeclass), - pointer_diff(p, pointer_align_up(pointer_offset(p, 1)))); + self->sizeclass, + SLAB_SIZE - pointer_diff(pointer_align_down(p), p)); } /** diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index f9c996f..35e3d74 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -55,99 +55,6 @@ namespace snmalloc static constexpr size_t NUM_LARGE_CLASSES = bits::ADDRESS_BITS - SUPERSLAB_BITS; - inline static size_t round_by_sizeclass(size_t rsize, size_t offset) - { - // check_same(); - // Must be called with a rounded size. - SNMALLOC_ASSERT(sizeclass_to_size(size_to_sizeclass(rsize)) == rsize); - // Only works up to certain offsets, exhaustively tested upto - // SUPERSLAB_SIZE. - SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); - - size_t align = bits::ctz(rsize); - size_t divider = rsize >> align; - // Maximum of 24 bits for 16MiB super/medium slab - if (INTERMEDIATE_BITS == 0 || divider == 1) - { - SNMALLOC_ASSERT(divider == 1); - return offset & ~(rsize - 1); - } - - if constexpr (bits::is64() && INTERMEDIATE_BITS <= 2) - { - // Only works for 64 bit multiplication, as the following will overflow in - // 32bit. - // The code is using reciprocal division, with a shift of 26 bits, this - // is considerably more bits than we need in the result. If SUPERSLABS - // get larger then we should review this code. - static_assert(SUPERSLAB_BITS <= 24, "The following code assumes 24 bits"); - static constexpr size_t shift = 26; - size_t back_shift = shift + align; - static constexpr size_t mul_shift = 1ULL << shift; - static constexpr uint32_t constants[8] = {0, - mul_shift, - 0, - (mul_shift / 3) + 1, - 0, - (mul_shift / 5) + 1, - 0, - (mul_shift / 7) + 1}; - return ((constants[divider] * offset) >> back_shift) * rsize; - } - else - // Use 32-bit division as considerably faster than 64-bit, and - // everything fits into 32bits here. - return static_cast(offset / rsize) * rsize; - } - - inline static bool is_multiple_of_sizeclass(size_t rsize, size_t offset) - { - // Must be called with a rounded size. - SNMALLOC_ASSERT(sizeclass_to_size(size_to_sizeclass(rsize)) == rsize); - // Only works up to certain offsets, exhaustively tested upto - // SUPERSLAB_SIZE. - SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); - - size_t align = bits::ctz(rsize); - size_t divider = rsize >> align; - // Maximum of 24 bits for 16MiB super/medium slab - if (INTERMEDIATE_BITS == 0 || divider == 1) - { - SNMALLOC_ASSERT(divider == 1); - return (offset & (rsize - 1)) == 0; - } - - if constexpr (bits::is64() && INTERMEDIATE_BITS <= 2) - { - // Only works for 64 bit multiplication, as the following will overflow in - // 32bit. - // The code is using reciprocal division, with a shift of 26 bits, this - // is considerably more bits than we need in the result. If SUPERSLABS - // get larger then we should review this code. - static_assert(SUPERSLAB_BITS <= 24, "The following code assumes 24 bits"); - static constexpr size_t shift = 31; - static constexpr size_t mul_shift = 1ULL << shift; - static constexpr uint32_t constants[8] = {0, - mul_shift, - 0, - (mul_shift / 3) + 1, - 0, - (mul_shift / 5) + 1, - 0, - (mul_shift / 7) + 1}; - - // There is a long chain of zeros after the backshift - // However, not all zero so just check a range. - // This is exhaustively tested for the current use case - return (((constants[divider] * offset)) & - (((1ULL << (align + 3)) - 1) << (shift - 3))) == 0; - } - else - // Use 32-bit division as considerably faster than 64-bit, and - // everything fits into 32bits here. - return static_cast(offset % rsize) == 0; - } - #ifdef CACHE_FRIENDLY_OFFSET SNMALLOC_FAST_PATH static void* remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) @@ -201,6 +108,9 @@ namespace snmalloc return sizeclass_to_size(size_to_sizeclass(size)); } + // Uses table for reciprocal division, so provide forward reference. + static bool is_multiple_of_sizeclass(sizeclass_t sc, size_t offset); + /// Returns the alignment that this size naturally has, that is /// all allocations of size `size` will be aligned to the returned value. SNMALLOC_FAST_PATH static size_t natural_alignment(size_t size) diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 5e067db..6db06aa 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -30,6 +30,10 @@ namespace snmalloc ModArray initial_offset_ptr; ModArray short_initial_offset_ptr; ModArray medium_slab_slots; + // Table of constants for reciprocal division for each sizeclass. + ModArray div_mult; + // Table of constants for reciprocal modulus for each sizeclass. + ModArray mod_mult; constexpr SizeClassTable() : size(), @@ -37,13 +41,27 @@ namespace snmalloc inverse_cache_friendly_mask(), initial_offset_ptr(), short_initial_offset_ptr(), - medium_slab_slots() + medium_slab_slots(), + div_mult(), + mod_mult() { size_t curr = 1; for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { size[sizeclass] = bits::from_exp_mant(sizeclass); + + div_mult[sizeclass] = + (bits::one_at_bit(bits::BITS - SUPERSLAB_BITS) / + (size[sizeclass] / MIN_ALLOC_SIZE)); + if (!bits::is_pow2(size[sizeclass])) + div_mult[sizeclass]++; + + mod_mult[sizeclass] = + (bits::one_at_bit(bits::BITS - 1) / size[sizeclass]); + if (!bits::is_pow2(size[sizeclass])) + mod_mult[sizeclass]++; + if (sizeclass < NUM_SMALL_CLASSES) { for (; curr <= size[sizeclass]; curr += 1 << PTR_BITS) @@ -131,4 +149,62 @@ namespace snmalloc return sizeclass_metadata .medium_slab_slots[(sizeclass - NUM_SMALL_CLASSES)]; } + + inline static size_t round_by_sizeclass(sizeclass_t sc, size_t offset) + { + // Only works up to certain offsets, exhaustively tested upto + // SUPERSLAB_SIZE. + SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); + + auto rsize = sizeclass_to_size(sc); + + if constexpr (bits::is64()) + { + // Only works for 64 bit multiplication, as the following will overflow in + // 32bit. + // The code is using reciprocal division. If SUPERSLABS + // get larger then we should review this code. For 24 bits, there are in + // sufficient bits to do this completely efficiently as 24 * 3 is larger + // than 64 bits. But we can pre-round by MIN_ALLOC_SIZE which gets us an + // extra 4 * 3 bits, and thus achievable in 64bit multiplication. + static_assert( + SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); + + return (((offset >> MIN_ALLOC_BITS) * sizeclass_metadata.div_mult[sc]) >> + (bits::BITS - SUPERSLAB_BITS)) * + rsize; + } + else + // Use 32-bit division as considerably faster than 64-bit, and + // everything fits into 32bits here. + return static_cast(offset / rsize) * rsize; + } + + inline static bool is_multiple_of_sizeclass(sizeclass_t sc, size_t offset) + { + // Only works up to certain offsets, exhaustively tested upto + // SUPERSLAB_SIZE. + SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); + + if constexpr (bits::is64()) + { + // Only works for 64 bit multiplication, as the following will overflow in + // 32bit. + // The code is using reciprocal division. If SUPERSLABS + // get larger then we should review this code. The modulus code + // has fewer restrictions than division, as it only requires the + // square of the offset to be representable. + static_assert( + SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); + static constexpr size_t MASK = (bits::one_at_bit(bits::BITS - 1) - 1) ^ + (bits::one_at_bit(bits::BITS - 1 - SUPERSLAB_BITS) - 1); + + return ((offset * sizeclass_metadata.mod_mult[sc]) & MASK) == 0; + } + else + // Use 32-bit division as considerably faster than 64-bit, and + // everything fits into 32bits here. + return static_cast(offset % sizeclass_to_size(sc)) == 0; + } + } // namespace snmalloc diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index b702cf3..1f5cb92 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -243,8 +243,11 @@ void test_external_pointer() void* p2 = pointer_offset(p1, offset); void* p3 = alloc->external_pointer(p2); void* p4 = alloc->external_pointer(p2); - UNUSED(p3); - UNUSED(p4); + if (p1 != p3) + { + std::cout << "size: " << size << " offset: " << offset << " p1: " << p1 + << " p3: " << p3 << std::endl; + } SNMALLOC_CHECK(p1 == p3); SNMALLOC_CHECK((size_t)p4 == (size_t)p1 + size - 1); } diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index 0836319..b0c391c 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -18,18 +18,28 @@ int main(int argc, char** argv) for (size_t size_class = 0; size_class < NUM_SIZECLASSES; size_class++) { size_t rsize = sizeclass_to_size((uint8_t)size_class); - for (size_t offset = 0; offset < SUPERSLAB_SIZE; offset++) + size_t max_offset = + size_class < NUM_SMALL_CLASSES ? SLAB_SIZE : SUPERSLAB_SIZE; + for (size_t offset = 0; offset < max_offset; offset++) { size_t rounded = (offset / rsize) * rsize; bool mod_0 = (offset % rsize) == 0; - size_t opt_rounded = round_by_sizeclass(rsize, offset); + size_t opt_rounded = round_by_sizeclass(size_class, offset); if (rounded != opt_rounded) + { + std::cout << "rsize " << rsize << " offset " << offset << " opt " + << opt_rounded << std::endl; abort(); + } - bool opt_mod_0 = is_multiple_of_sizeclass(rsize, offset); + bool opt_mod_0 = is_multiple_of_sizeclass(size_class, offset); if (opt_mod_0 != mod_0) + { + std::cout << "rsize " << rsize << " offset " << offset << " opt_mod " + << opt_mod_0 << std::endl; abort(); + } } } return 0;