Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Better reciprocal division and modulus.

Browse Source 
The previous reciprocal division branch on the prime that the sizeclass
was constructed from. All sizeclasses can be represented as
  2^n * {1,3,5,7}
This lead to a very small table, but some work to calculate the
appropriate shifts and multiplications to implement reciprocal division.

This commit uses a completely uniform representation for every
sizeclass using a lookup table.  Due to the precise ranges that we query
the modulus and rounding on, we can do this much more efficiently.

The func-release-rounding exhaustively tests all the queries we are
interested in.
This commit is contained in:
Matthew Parkinson
2021-03-19 08:14:12 +00:00
committed by Matthew Parkinson
parent 486ef10c21
commit 8824622136
6 changed files with 102 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/mem/alloc.h
View File

@@ -807,7 +807,7 @@ namespace snmalloc
size_t offset_from_end =
pointer_diff(p, pointer_offset_signed(end_point, -1));
size_t end_to_end = round_by_sizeclass(rsize, offset_from_end);
size_t end_to_end = round_by_sizeclass(sizeclass, offset_from_end);
return pointer_offset_signed(
end_point_correction, -static_cast<ptrdiff_t>(end_to_end));
@@ -1380,8 +1380,7 @@ namespace snmalloc
{
check_client(
is_multiple_of_sizeclass(
sizeclass_to_size(sizeclass),
pointer_diff(p, pointer_offset(slab, SUPERSLAB_SIZE))),
sizeclass, pointer_diff(p, pointer_offset(slab, SUPERSLAB_SIZE))),
"Not deallocating start of an object");
medium_dealloc_start(slab, p, sizeclass);

4
src/mem/metaslab.h
View File

@@ -109,8 +109,8 @@ namespace snmalloc
SNMALLOC_FAST_PATH static bool is_start_of_object(Metaslab* self, void* p)
{
return is_multiple_of_sizeclass(
sizeclass_to_size(self->sizeclass),
pointer_diff(p, pointer_align_up<SLAB_SIZE>(pointer_offset(p, 1))));
self->sizeclass,
SLAB_SIZE - pointer_diff(pointer_align_down<SLAB_SIZE>(p), p));
}
/**

96
src/mem/sizeclass.h
View File

@@ -55,99 +55,6 @@ namespace snmalloc
static constexpr size_t NUM_LARGE_CLASSES =
bits::ADDRESS_BITS - SUPERSLAB_BITS;
inline static size_t round_by_sizeclass(size_t rsize, size_t offset)
{
// check_same<NUM_LARGE_CLASSES, Globals::num_large_classes>();
// Must be called with a rounded size.
SNMALLOC_ASSERT(sizeclass_to_size(size_to_sizeclass(rsize)) == rsize);
// Only works up to certain offsets, exhaustively tested upto
// SUPERSLAB_SIZE.
SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE);
size_t align = bits::ctz(rsize);
size_t divider = rsize >> align;
// Maximum of 24 bits for 16MiB super/medium slab
if (INTERMEDIATE_BITS == 0 || divider == 1)
{
SNMALLOC_ASSERT(divider == 1);
return offset & ~(rsize - 1);
}
if constexpr (bits::is64() && INTERMEDIATE_BITS <= 2)
{
// Only works for 64 bit multiplication, as the following will overflow in
// 32bit.
// The code is using reciprocal division, with a shift of 26 bits, this
// is considerably more bits than we need in the result. If SUPERSLABS
// get larger then we should review this code.
static_assert(SUPERSLAB_BITS <= 24, "The following code assumes 24 bits");
static constexpr size_t shift = 26;
size_t back_shift = shift + align;
static constexpr size_t mul_shift = 1ULL << shift;
static constexpr uint32_t constants[8] = {0,
mul_shift,
0,
(mul_shift / 3) + 1,
0,
(mul_shift / 5) + 1,
0,
(mul_shift / 7) + 1};
return ((constants[divider] * offset) >> back_shift) * rsize;
}
else
// Use 32-bit division as considerably faster than 64-bit, and
// everything fits into 32bits here.
return static_cast<uint32_t>(offset / rsize) * rsize;
}
inline static bool is_multiple_of_sizeclass(size_t rsize, size_t offset)
{
// Must be called with a rounded size.
SNMALLOC_ASSERT(sizeclass_to_size(size_to_sizeclass(rsize)) == rsize);
// Only works up to certain offsets, exhaustively tested upto
// SUPERSLAB_SIZE.
SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE);
size_t align = bits::ctz(rsize);
size_t divider = rsize >> align;
// Maximum of 24 bits for 16MiB super/medium slab
if (INTERMEDIATE_BITS == 0 || divider == 1)
{
SNMALLOC_ASSERT(divider == 1);
return (offset & (rsize - 1)) == 0;
}
if constexpr (bits::is64() && INTERMEDIATE_BITS <= 2)
{
// Only works for 64 bit multiplication, as the following will overflow in
// 32bit.
// The code is using reciprocal division, with a shift of 26 bits, this
// is considerably more bits than we need in the result. If SUPERSLABS
// get larger then we should review this code.
static_assert(SUPERSLAB_BITS <= 24, "The following code assumes 24 bits");
static constexpr size_t shift = 31;
static constexpr size_t mul_shift = 1ULL << shift;
static constexpr uint32_t constants[8] = {0,
mul_shift,
0,
(mul_shift / 3) + 1,
0,
(mul_shift / 5) + 1,
0,
(mul_shift / 7) + 1};
// There is a long chain of zeros after the backshift
// However, not all zero so just check a range.
// This is exhaustively tested for the current use case
return (((constants[divider] * offset)) &
(((1ULL << (align + 3)) - 1) << (shift - 3))) == 0;
}
else
// Use 32-bit division as considerably faster than 64-bit, and
// everything fits into 32bits here.
return static_cast<uint32_t>(offset % rsize) == 0;
}
#ifdef CACHE_FRIENDLY_OFFSET
SNMALLOC_FAST_PATH static void*
remove_cache_friendly_offset(void* p, sizeclass_t sizeclass)
@@ -201,6 +108,9 @@ namespace snmalloc
return sizeclass_to_size(size_to_sizeclass(size));
}
// Uses table for reciprocal division, so provide forward reference.
static bool is_multiple_of_sizeclass(sizeclass_t sc, size_t offset);
/// Returns the alignment that this size naturally has, that is
/// all allocations of size `size` will be aligned to the returned value.
SNMALLOC_FAST_PATH static size_t natural_alignment(size_t size)

78
src/mem/sizeclasstable.h
View File

@@ -30,6 +30,10 @@ namespace snmalloc
ModArray<NUM_SMALL_CLASSES, uint16_t> initial_offset_ptr;
ModArray<NUM_SMALL_CLASSES, uint16_t> short_initial_offset_ptr;
ModArray<NUM_MEDIUM_CLASSES, uint16_t> medium_slab_slots;
// Table of constants for reciprocal division for each sizeclass.
ModArray<NUM_SIZECLASSES, size_t> div_mult;
// Table of constants for reciprocal modulus for each sizeclass.
ModArray<NUM_SIZECLASSES, size_t> mod_mult;
constexpr SizeClassTable()
: size(),
@@ -37,13 +41,27 @@ namespace snmalloc
inverse_cache_friendly_mask(),
initial_offset_ptr(),
short_initial_offset_ptr(),
medium_slab_slots()
medium_slab_slots(),
div_mult(),
mod_mult()
{
size_t curr = 1;
for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++)
{
size[sizeclass] =
bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(sizeclass);
div_mult[sizeclass] =
(bits::one_at_bit(bits::BITS - SUPERSLAB_BITS) /
(size[sizeclass] / MIN_ALLOC_SIZE));
if (!bits::is_pow2(size[sizeclass]))
div_mult[sizeclass]++;
mod_mult[sizeclass] =
(bits::one_at_bit(bits::BITS - 1) / size[sizeclass]);
if (!bits::is_pow2(size[sizeclass]))
mod_mult[sizeclass]++;
if (sizeclass < NUM_SMALL_CLASSES)
{
for (; curr <= size[sizeclass]; curr += 1 << PTR_BITS)
@@ -131,4 +149,62 @@ namespace snmalloc
return sizeclass_metadata
.medium_slab_slots[(sizeclass - NUM_SMALL_CLASSES)];
}
inline static size_t round_by_sizeclass(sizeclass_t sc, size_t offset)
{
// Only works up to certain offsets, exhaustively tested upto
// SUPERSLAB_SIZE.
SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE);
auto rsize = sizeclass_to_size(sc);
if constexpr (bits::is64())
{
// Only works for 64 bit multiplication, as the following will overflow in
// 32bit.
// The code is using reciprocal division. If SUPERSLABS
// get larger then we should review this code. For 24 bits, there are in
// sufficient bits to do this completely efficiently as 24 * 3 is larger
// than 64 bits. But we can pre-round by MIN_ALLOC_SIZE which gets us an
// extra 4 * 3 bits, and thus achievable in 64bit multiplication.
static_assert(
SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits");
return (((offset >> MIN_ALLOC_BITS) * sizeclass_metadata.div_mult[sc]) >>
(bits::BITS - SUPERSLAB_BITS)) *
rsize;
}
else
// Use 32-bit division as considerably faster than 64-bit, and
// everything fits into 32bits here.
return static_cast<uint32_t>(offset / rsize) * rsize;
}
inline static bool is_multiple_of_sizeclass(sizeclass_t sc, size_t offset)
{
// Only works up to certain offsets, exhaustively tested upto
// SUPERSLAB_SIZE.
SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE);
if constexpr (bits::is64())
{
// Only works for 64 bit multiplication, as the following will overflow in
// 32bit.
// The code is using reciprocal division. If SUPERSLABS
// get larger then we should review this code. The modulus code
// has fewer restrictions than division, as it only requires the
// square of the offset to be representable.
static_assert(
SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits");
static constexpr size_t MASK = (bits::one_at_bit(bits::BITS - 1) - 1) ^
(bits::one_at_bit(bits::BITS - 1 - SUPERSLAB_BITS) - 1);
return ((offset * sizeclass_metadata.mod_mult[sc]) & MASK) == 0;
}
else
// Use 32-bit division as considerably faster than 64-bit, and
// everything fits into 32bits here.
return static_cast<uint32_t>(offset % sizeclass_to_size(sc)) == 0;
}
} // namespace snmalloc

7
src/test/func/memory/memory.cc
View File

@@ -243,8 +243,11 @@ void test_external_pointer()
void* p2 = pointer_offset(p1, offset);
void* p3 = alloc->external_pointer(p2);
void* p4 = alloc->external_pointer<End>(p2);
UNUSED(p3);
UNUSED(p4);
if (p1 != p3)
{
std::cout << "size: " << size << " offset: " << offset << " p1: " << p1
<< " p3: " << p3 << std::endl;
}
SNMALLOC_CHECK(p1 == p3);
SNMALLOC_CHECK((size_t)p4 == (size_t)p1 + size - 1);
}

16
src/test/func/release-rounding/rounding.cc
View File

@@ -18,18 +18,28 @@ int main(int argc, char** argv)
for (size_t size_class = 0; size_class < NUM_SIZECLASSES; size_class++)
{
size_t rsize = sizeclass_to_size((uint8_t)size_class);
for (size_t offset = 0; offset < SUPERSLAB_SIZE; offset++)
size_t max_offset =
size_class < NUM_SMALL_CLASSES ? SLAB_SIZE : SUPERSLAB_SIZE;
for (size_t offset = 0; offset < max_offset; offset++)
{
size_t rounded = (offset / rsize) * rsize;
bool mod_0 = (offset % rsize) == 0;
size_t opt_rounded = round_by_sizeclass(rsize, offset);
size_t opt_rounded = round_by_sizeclass(size_class, offset);
if (rounded != opt_rounded)
{
std::cout << "rsize " << rsize << " offset " << offset << " opt "
<< opt_rounded << std::endl;
abort();
}
bool opt_mod_0 = is_multiple_of_sizeclass(rsize, offset);
bool opt_mod_0 = is_multiple_of_sizeclass(size_class, offset);
if (opt_mod_0 != mod_0)
{
std::cout << "rsize " << rsize << " offset " << offset << " opt_mod "
<< opt_mod_0 << std::endl;
abort();
}
}
}
return 0;