Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Factor sizeclass meta-data for cache locality

Browse Source 
This commit splits the sizeclass meta-data to generate better cache
locality for various lookups for checking for size and start of
sizeclasses.

Also, contains some tidying including removing sizeclasses covering
large range. This is left over from an alternative design for large
classes that is no longer in use.
This commit is contained in:
Matthew Parkinson
2021-08-25 14:49:06 +01:00
committed by Matthew Parkinson
parent 0f70494d55
commit 44416ed70e
4 changed files with 115 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/mem/localalloc.h
View File

@@ -189,8 +189,8 @@ namespace snmalloc
// set up meta data so sizeclass is correct, and hence alloc size, and
// external pointer.
#ifdef SNMALLOC_TRACING
std::cout << "size " << size << " sizeclass " << size_to_sizeclass(size)
<< std::endl;
std::cout << "size " << size << " pow2 size "
<< bits::next_pow2_bits(size) << std::endl;
#endif
// Note that meta data is not currently used for large allocs.

213
src/mem/sizeclasstable.h
View File

@@ -30,6 +30,7 @@ namespace snmalloc
static inline size_t large_sizeclass_to_size(uint8_t large_class)
{
// TODO. Remove
UNUSED(large_class);
abort();
// return bits::one_at_bit(large_class + SUPERSLAB_BITS);
@@ -53,98 +54,77 @@ namespace snmalloc
return ((alignment - 1) | (size - 1)) + 1;
}
constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s)
/**
* This structure contains the fields required for fast paths for sizeclasses.
*/
struct sizeclass_data_fast
{
// We subtract and shift to reduce the size of the table, i.e. we don't have
// to store a value for every size.
return (s - 1) >> MIN_ALLOC_BITS;
}
size_t size;
// We store the mask as it is used more on the fast path, and the size of
// the slab.
size_t slab_mask;
// Table of constants for reciprocal division for each sizeclass.
size_t div_mult;
// Table of constants for reciprocal modulus for each sizeclass.
size_t mod_mult;
};
constexpr static size_t NUM_SIZECLASSES_EXTENDED =
size_to_sizeclass_const(bits::one_at_bit(bits::ADDRESS_BITS - 1));
constexpr static size_t sizeclass_lookup_size =
sizeclass_lookup_index(MAX_SIZECLASS_SIZE);
/**
* This structure contains the remaining fields required for slow paths for
* sizeclasses.
*/
struct sizeclass_data_slow
{
uint16_t capacity;
uint16_t waking;
};
struct SizeClassTable
{
sizeclass_compress_t sizeclass_lookup[sizeclass_lookup_size] = {{}};
ModArray<NUM_SIZECLASSES_EXTENDED, size_t> size;
ModArray<NUM_SIZECLASSES, sizeclass_data_fast> fast;
ModArray<NUM_SIZECLASSES, sizeclass_data_slow> slow;
ModArray<NUM_SIZECLASSES, uint16_t> capacity;
ModArray<NUM_SIZECLASSES, uint16_t> waking;
// We store the mask as it is used more on the fast path, and the size of
// the slab.
ModArray<NUM_SIZECLASSES, size_t> slab_mask;
// Table of constants for reciprocal division for each sizeclass.
ModArray<NUM_SIZECLASSES, size_t> div_mult;
// Table of constants for reciprocal modulus for each sizeclass.
ModArray<NUM_SIZECLASSES, size_t> mod_mult;
constexpr SizeClassTable()
: size(), capacity(), waking(), slab_mask(), div_mult(), mod_mult()
constexpr SizeClassTable() : fast(), slow()
{
for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES;
sizeclass++)
{
size_t rsize =
bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(sizeclass);
size[sizeclass] = rsize;
fast[sizeclass].size = rsize;
size_t slab_bits = bits::max(
bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS);
slab_mask[sizeclass] = bits::one_at_bit(slab_bits) - 1;
fast[sizeclass].slab_mask = bits::one_at_bit(slab_bits) - 1;
capacity[sizeclass] =
static_cast<uint16_t>((slab_mask[sizeclass] + 1) / rsize);
slow[sizeclass].capacity =
static_cast<uint16_t>((fast[sizeclass].slab_mask + 1) / rsize);
waking[sizeclass] =
slow[sizeclass].waking =
#ifdef SNMALLOC_CHECK_CLIENT
static_cast<uint16_t>(capacity[sizeclass] / 4);
static_cast<uint16_t>(slow[sizeclass].capacity / 4);
#else
static_cast<uint16_t>(bits::min((capacity[sizeclass] / 4), 32));
static_cast<uint16_t>(bits::min((slow[sizeclass].capacity / 4), 32));
#endif
}
for (sizeclass_compress_t sizeclass = NUM_SIZECLASSES;
sizeclass < NUM_SIZECLASSES_EXTENDED;
sizeclass++)
{
size[sizeclass] = bits::prev_pow2_const(
bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(sizeclass));
}
for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES;
sizeclass++)
{
div_mult[sizeclass] = // TODO is MAX_SIZECLASS_BITS right?
fast[sizeclass].div_mult = // TODO is MAX_SIZECLASS_BITS right?
(bits::one_at_bit(bits::BITS - 24) /
(size[sizeclass] / MIN_ALLOC_SIZE));
if (!bits::is_pow2(size[sizeclass]))
div_mult[sizeclass]++;
(fast[sizeclass].size / MIN_ALLOC_SIZE));
if (!bits::is_pow2(fast[sizeclass].size))
fast[sizeclass].div_mult++;
mod_mult[sizeclass] =
(bits::one_at_bit(bits::BITS - 1) / size[sizeclass]);
if (!bits::is_pow2(size[sizeclass]))
mod_mult[sizeclass]++;
fast[sizeclass].mod_mult =
(bits::one_at_bit(bits::BITS - 1) / fast[sizeclass].size);
if (!bits::is_pow2(fast[sizeclass].size))
fast[sizeclass].mod_mult++;
// Shift multiplier, so that the result of division completely
// overflows, and thus the top SUPERSLAB_BITS will be zero if the mod is
// zero.
mod_mult[sizeclass] *= 2;
}
size_t curr = 1;
for (sizeclass_compress_t sizeclass = 0; sizeclass <= NUM_SIZECLASSES;
sizeclass++)
{
for (; curr <= size[sizeclass]; curr += 1 << MIN_ALLOC_BITS)
{
auto i = sizeclass_lookup_index(curr);
if (i == sizeclass_lookup_size)
break;
sizeclass_lookup[i] = sizeclass;
}
fast[sizeclass].mod_mult *= 2;
}
}
};
@@ -153,12 +133,12 @@ namespace snmalloc
constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass)
{
return sizeclass_metadata.size[sizeclass];
return sizeclass_metadata.fast[sizeclass].size;
}
inline static size_t sizeclass_to_slab_size(sizeclass_t sizeclass)
{
return sizeclass_metadata.slab_mask[sizeclass] + 1;
return sizeclass_metadata.fast[sizeclass].slab_mask + 1;
}
/**
@@ -170,12 +150,7 @@ namespace snmalloc
*/
inline uint16_t threshold_for_waking_slab(sizeclass_t sizeclass)
{
// #ifdef SNMALLOC_CHECK_CLIENT
return sizeclass_metadata.waking[sizeclass];
// #else
// UNUSED(sizeclass);
// return 1;
// #endif
return sizeclass_metadata.slow[sizeclass].waking;
}
inline static size_t sizeclass_to_slab_sizeclass(sizeclass_t sizeclass)
@@ -193,26 +168,7 @@ namespace snmalloc
inline constexpr static uint16_t
sizeclass_to_slab_object_count(sizeclass_t sizeclass)
{
return sizeclass_metadata.capacity[sizeclass];
}
static inline sizeclass_t size_to_sizeclass(size_t size)
{
auto index = sizeclass_lookup_index(size);
if (index < sizeclass_lookup_size)
{
return sizeclass_metadata.sizeclass_lookup[index];
}
// Don't use sizeclasses that are not a multiple of the alignment.
// For example, 24 byte allocations can be
// problematic for some data due to alignment issues.
// TODO hack to power of 2 for large sizes
size = bits::next_pow2(size);
return static_cast<sizeclass_t>(
bits::to_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(size));
return sizeclass_metadata.slow[sizeclass].capacity;
}
inline static size_t round_by_sizeclass(sizeclass_t sc, size_t offset)
@@ -236,7 +192,8 @@ namespace snmalloc
// SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits");
// TODO 24 hack
return (((offset >> MIN_ALLOC_BITS) * sizeclass_metadata.div_mult[sc]) >>
return (((offset >> MIN_ALLOC_BITS) *
sizeclass_metadata.fast[sc].div_mult) >>
(bits::BITS - 24)) *
rsize;
}
@@ -265,7 +222,7 @@ namespace snmalloc
static constexpr size_t MASK =
~(bits::one_at_bit(bits::BITS - 1 - 24) - 1);
return ((offset * sizeclass_metadata.mod_mult[sc]) & MASK) == 0;
return ((offset * sizeclass_metadata.fast[sc].mod_mult) & MASK) == 0;
}
else
// Use 32-bit division as considerably faster than 64-bit, and
@@ -273,6 +230,68 @@ namespace snmalloc
return static_cast<uint32_t>(offset % sizeclass_to_size(sc)) == 0;
}
inline static size_t large_size_to_chunk_size(size_t size)
{
return bits::next_pow2(size);
}
inline static size_t large_size_to_chunk_sizeclass(size_t size)
{
return bits::next_pow2_bits(size) - MIN_CHUNK_BITS;
}
constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s)
{
// We subtract and shift to reduce the size of the table, i.e. we don't have
// to store a value for every size.
return (s - 1) >> MIN_ALLOC_BITS;
}
static inline sizeclass_t size_to_sizeclass(size_t size)
{
constexpr static size_t sizeclass_lookup_size =
sizeclass_lookup_index(MAX_SIZECLASS_SIZE);
/**
* This struct is used to statically initialise a table for looking up
* the correct sizeclass.
*/
struct SizeClassLookup
{
sizeclass_compress_t table[sizeclass_lookup_size] = {{}};
constexpr SizeClassLookup()
{
size_t curr = 1;
for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES;
sizeclass++)
{
for (; curr <= sizeclass_metadata.fast[sizeclass].size;
curr += 1 << MIN_ALLOC_BITS)
{
auto i = sizeclass_lookup_index(curr);
if (i == sizeclass_lookup_size)
break;
table[i] = sizeclass;
}
}
}
};
static constexpr SizeClassLookup sizeclass_lookup = SizeClassLookup();
auto index = sizeclass_lookup_index(size);
if (index < sizeclass_lookup_size)
{
return sizeclass_lookup.table[index];
}
// Check this is not called on large sizes.
SNMALLOC_ASSERT(size == 0);
// Map size == 0 to the first sizeclass.
return 0;
}
inline SNMALLOC_FAST_PATH static size_t round_size(size_t size)
{
if (size > sizeclass_to_size(NUM_SIZECLASSES - 1))
@@ -295,14 +314,4 @@ namespace snmalloc
return 1;
return bits::one_at_bit(bits::ctz(rsize));
}
inline static size_t large_size_to_chunk_size(size_t size)
{
return bits::next_pow2(size);
}
inline static size_t large_size_to_chunk_sizeclass(size_t size)
{
return bits::next_pow2_bits(size) - MIN_CHUNK_BITS;
}
} // namespace snmalloc

2
src/test/func/memory/memory.cc
View File

@@ -415,7 +415,7 @@ void test_static_sized_allocs()
{
// For each small, medium, and large class, do each kind dealloc. This is
// mostly to ensure that all of these forms compile.
for (size_t sc = 0; sc < NUM_SIZECLASSES_EXTENDED; sc++)
for (size_t sc = 0; sc < NUM_SIZECLASSES; sc++)
{
// test_static_sized_alloc<sc, 0>();
// test_static_sized_alloc<sc, 1>();

3
src/test/func/sizeclass/sizeclass.cc
View File

@@ -78,9 +78,8 @@ int main(int, char**)
std::cout << "sizeclass |-> [size_low, size_high] " << std::endl;
size_t slab_size = 0;
for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES + 20; sz++)
for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++)
{
// Separate printing for small and medium sizeclasses
if (
sz < snmalloc::NUM_SIZECLASSES &&
slab_size != snmalloc::sizeclass_to_slab_size(sz))