#pragma once #include "../pal/pal.h" #include "allocconfig.h" namespace snmalloc { // Both usings should compile // We use size_t as it generates better code. using sizeclass_t = size_t; // using sizeclass_t = uint8_t; using sizeclass_compress_t = uint8_t; constexpr static uintptr_t SIZECLASS_MASK = 0xFF; constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short); constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); constexpr static size_t sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass); static sizeclass_t size_to_sizeclass(size_t size); constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) { // Don't use sizeclasses that are not a multiple of the alignment. // For example, 24 byte allocations can be // problematic for some data due to alignment issues. auto sc = static_cast( bits::to_exp_mant_const(size)); SNMALLOC_ASSERT(sc == static_cast(sc)); return sc; } constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class) { return bits::one_at_bit(large_class + SUPERSLAB_BITS); } // Small classes range from [MIN, SLAB], i.e. inclusive. static constexpr size_t NUM_SMALL_CLASSES = size_to_sizeclass_const(bits::one_at_bit(SLAB_BITS)) + 1; static constexpr size_t NUM_SIZECLASSES = size_to_sizeclass_const(SUPERSLAB_SIZE); // Medium classes range from (SLAB, SUPERSLAB), i.e. non-inclusive. static constexpr size_t NUM_MEDIUM_CLASSES = NUM_SIZECLASSES - NUM_SMALL_CLASSES; // Large classes range from [SUPERSLAB, ADDRESS_SPACE). static constexpr size_t NUM_LARGE_CLASSES = bits::ADDRESS_BITS - SUPERSLAB_BITS; inline static size_t round_by_sizeclass(size_t rsize, size_t offset) { // check_same(); // Must be called with a rounded size. SNMALLOC_ASSERT(sizeclass_to_size(size_to_sizeclass(rsize)) == rsize); // Only works up to certain offsets, exhaustively tested upto // SUPERSLAB_SIZE. SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); size_t align = bits::ctz(rsize); size_t divider = rsize >> align; // Maximum of 24 bits for 16MiB super/medium slab if (INTERMEDIATE_BITS == 0 || divider == 1) { SNMALLOC_ASSERT(divider == 1); return offset & ~(rsize - 1); } if constexpr (bits::is64() && INTERMEDIATE_BITS <= 2) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. // The code is using reciprocal division, with a shift of 26 bits, this // is considerably more bits than we need in the result. If SUPERSLABS // get larger then we should review this code. static_assert(SUPERSLAB_BITS <= 24, "The following code assumes 24 bits"); static constexpr size_t shift = 26; size_t back_shift = shift + align; static constexpr size_t mul_shift = 1ULL << shift; static constexpr uint32_t constants[8] = {0, mul_shift, 0, (mul_shift / 3) + 1, 0, (mul_shift / 5) + 1, 0, (mul_shift / 7) + 1}; return ((constants[divider] * offset) >> back_shift) * rsize; } else // Use 32-bit division as considerably faster than 64-bit, and // everything fits into 32bits here. return static_cast(offset / rsize) * rsize; } inline static bool is_multiple_of_sizeclass(size_t rsize, size_t offset) { // Must be called with a rounded size. SNMALLOC_ASSERT(sizeclass_to_size(size_to_sizeclass(rsize)) == rsize); // Only works up to certain offsets, exhaustively tested upto // SUPERSLAB_SIZE. SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); size_t align = bits::ctz(rsize); size_t divider = rsize >> align; // Maximum of 24 bits for 16MiB super/medium slab if (INTERMEDIATE_BITS == 0 || divider == 1) { SNMALLOC_ASSERT(divider == 1); return (offset & (rsize - 1)) == 0; } if constexpr (bits::is64() && INTERMEDIATE_BITS <= 2) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. // The code is using reciprocal division, with a shift of 26 bits, this // is considerably more bits than we need in the result. If SUPERSLABS // get larger then we should review this code. static_assert(SUPERSLAB_BITS <= 24, "The following code assumes 24 bits"); static constexpr size_t shift = 31; static constexpr size_t mul_shift = 1ULL << shift; static constexpr uint32_t constants[8] = {0, mul_shift, 0, (mul_shift / 3) + 1, 0, (mul_shift / 5) + 1, 0, (mul_shift / 7) + 1}; // There is a long chain of zeros after the backshift // However, not all zero so just check a range. // This is exhaustively tested for the current use case return (((constants[divider] * offset)) & (((1ULL << (align + 3)) - 1) << (shift - 3))) == 0; } else // Use 32-bit division as considerably faster than 64-bit, and // everything fits into 32bits here. return static_cast(offset % rsize) == 0; } #ifdef CACHE_FRIENDLY_OFFSET SNMALLOC_FAST_PATH static void* remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) { size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass); return p = (void*)((uintptr_t)p & mask); } SNMALLOC_FAST_PATH static uintptr_t remove_cache_friendly_offset(uintptr_t relative, sizeclass_t sizeclass) { size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass); return relative & mask; } #else SNMALLOC_FAST_PATH static void* remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) { UNUSED(sizeclass); return p; } SNMALLOC_FAST_PATH static uintptr_t remove_cache_friendly_offset(uintptr_t relative, sizeclass_t sizeclass) { UNUSED(sizeclass); return relative; } #endif SNMALLOC_FAST_PATH static size_t aligned_size(size_t alignment, size_t size) { // Client responsible for checking alignment is not zero SNMALLOC_ASSERT(alignment != 0); // Client responsible for checking alignment is a power of two SNMALLOC_ASSERT(bits::next_pow2(alignment) == alignment); return ((alignment - 1) | (size - 1)) + 1; } SNMALLOC_FAST_PATH static size_t round_size(size_t size) { if (size > sizeclass_to_size(NUM_SIZECLASSES - 1)) { return bits::next_pow2(size); } if (size == 0) { size = 1; } return sizeclass_to_size(size_to_sizeclass(size)); } /// Returns the alignment that this size naturally has, that is /// all allocations of size `size` will be aligned to the returned value. SNMALLOC_FAST_PATH static size_t natural_alignment(size_t size) { auto rsize = round_size(size); return bits::one_at_bit(bits::ctz(rsize)); } } // namespace snmalloc