Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #21 from plietar/smart-next

Browse Source 
Place the next pointer at a different place on every object.
This commit is contained in:
Matthew Parkinson
2019-02-19 09:04:23 +00:00
committed by GitHub
parent c47eed1922 ffe496dd18
commit aac2b9212e
6 changed files with 143 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
CMakeLists.txt
View File

@@ -5,6 +5,8 @@ option(USE_SNMALLOC_STATS "Track allocation stats" OFF)
option(USE_MEASURE "Measure performance with histograms" OFF)
option(USE_SBRK "Use sbrk instead of mmap" OFF)
set(CACHE_FRIENDLY_OFFSET OFF CACHE STRING "Base offset to place linked-list nodes.")
macro(subdirlist result curdir)
file(GLOB children LIST_DIRECTORIES true RELATIVE ${curdir} ${curdir}/*)
set(dirlist "")
@@ -60,6 +62,10 @@ macro(add_shim name)
target_include_directories(${name} PRIVATE src)
target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))")
set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden)
if(CACHE_FRIENDLY_OFFSET)
target_compile_definitions(${name} PRIVATE -DCACHE_FRIENDLY_OFFSET=${CACHE_FRIENDLY_OFFSET})
endif()
endmacro()
if(NOT MSVC)
@@ -73,20 +79,29 @@ enable_testing()
set(TESTDIR ${CMAKE_CURRENT_SOURCE_DIR}/src/test)
subdirlist(TEST_CATEGORIES ${TESTDIR})
foreach(SUPER_SLAB_SIZE 1;16)
foreach(TEST_CATEGORY ${TEST_CATEGORIES})
subdirlist(TESTS ${TESTDIR}/${TEST_CATEGORY})
foreach(TEST ${TESTS})
unset(SRC)
aux_source_directory(${TESTDIR}/${TEST_CATEGORY}/${TEST} SRC)
set(TESTNAME "${TEST_CATEGORY}-${TEST}-${SUPER_SLAB_SIZE}")
add_executable(${TESTNAME} ${SRC} src/override/new.cc)
if (${SUPER_SLAB_SIZE} EQUAL 1)
target_compile_definitions(${TESTNAME} PRIVATE IS_ADDRESS_SPACE_CONSTRAINED)
endif()
target_include_directories(${TESTNAME} PRIVATE src)
linklibs(${TESTNAME})
add_test(${TESTNAME} ${TESTNAME})
foreach(TEST_CACHE_FRIENDLY_OFFSET OFF;ON)
foreach(SUPER_SLAB_SIZE 1;16)
foreach(TEST_CATEGORY ${TEST_CATEGORIES})
subdirlist(TESTS ${TESTDIR}/${TEST_CATEGORY})
foreach(TEST ${TESTS})
unset(SRC)
aux_source_directory(${TESTDIR}/${TEST_CATEGORY}/${TEST} SRC)
set(TESTNAME "${TEST_CATEGORY}-${TEST}-${SUPER_SLAB_SIZE}")
if (TEST_CACHE_FRIENDLY_OFFSET)
set(TESTNAME "${TESTNAME}-cache-friendly")
endif()
add_executable(${TESTNAME} ${SRC} src/override/new.cc)
if (${SUPER_SLAB_SIZE} EQUAL 1)
target_compile_definitions(${TESTNAME} PRIVATE IS_ADDRESS_SPACE_CONSTRAINED)
endif()
if(TEST_CACHE_FRIENDLY_OFFSET)
target_compile_definitions(${TESTNAME} PRIVATE CACHE_FRIENDLY_OFFSET=64)
endif()
target_include_directories(${TESTNAME} PRIVATE src)
linklibs(${TESTNAME})
add_test(${TESTNAME} ${TESTNAME})
endforeach()
endforeach()
endforeach()
endforeach()

44
src/mem/alloc.h
View File

@@ -628,6 +628,26 @@ namespace snmalloc
std::conditional_t<IsQueueInline, RemoteAllocator, RemoteAllocator*>
remote_alloc;
#ifdef CACHE_FRIENDLY_OFFSET
size_t remote_offset = 0;
void* apply_cache_friendly_offset(void* p, uint8_t sizeclass)
{
size_t mask = sizeclass_to_cache_friendly_mask(sizeclass);
size_t offset = remote_offset & mask;
remote_offset += CACHE_FRIENDLY_OFFSET;
return (void*)((uintptr_t)p + offset);
}
#else
void* apply_cache_friendly_offset(void* p, uint8_t sizeclass)
{
UNUSED(sizeclass);
return p;
}
#endif
auto* public_state()
{
if constexpr (IsQueueInline)
@@ -726,7 +746,7 @@ namespace snmalloc
Metaslab& meta = super->get_meta(slab);
if (p->target_id() == id())
{
small_dealloc(super, p, meta.sizeclass);
small_dealloc_offseted(super, p, meta.sizeclass);
}
else
{
@@ -739,7 +759,9 @@ namespace snmalloc
Mediumslab* slab = Mediumslab::get(p);
if (p->target_id() == id())
{
medium_dealloc(slab, p, slab->get_sizeclass());
uint8_t sizeclass = slab->get_sizeclass();
void* start = remove_cache_friendly_offset(p, sizeclass);
medium_dealloc(slab, start, sizeclass);
}
else
{
@@ -909,6 +931,20 @@ namespace snmalloc
void small_dealloc(Superslab* super, void* p, uint8_t sizeclass)
{
#ifndef SNMALLOC_SAFE_CLIENT
Slab* slab = Slab::get(p);
if (!slab->is_start_of_object(super, p))
{
error("Not deallocating start of an object");
}
#endif
void* offseted = apply_cache_friendly_offset(p, sizeclass);
small_dealloc_offseted(super, offseted, sizeclass);
}
void small_dealloc_offseted(Superslab* super, void* p, uint8_t sizeclass)
{
MEASURE_TIME(small_dealloc, 4, 16);
stats().sizeclass_dealloc(sizeclass);
@@ -1111,8 +1147,10 @@ namespace snmalloc
{
MEASURE_TIME(remote_dealloc, 4, 16);
void* offseted = apply_cache_friendly_offset(p, sizeclass);
stats().remote_free(sizeclass);
remote.dealloc(target->id(), p, sizeclass);
remote.dealloc(target->id(), offseted, sizeclass);
if (remote.size < REMOTE_CACHE)
return;

14
src/mem/metaslab.h
View File

@@ -102,7 +102,12 @@ namespace snmalloc
{
size_t size = sizeclass_to_size(sizeclass);
size_t offset = get_slab_offset(sizeclass, is_short);
return ((((head & ~(size_t)1) - (offset & ~(size_t)1)) % size) == 0);
size_t head_start =
remove_cache_friendly_offset(head & ~(size_t)1, sizeclass);
size_t slab_start = offset & ~(size_t)1;
return ((head_start - slab_start) % size) == 0;
}
void debug_slab_invariant(bool is_short, Slab* slab)
@@ -129,7 +134,9 @@ namespace snmalloc
while ((curr & 1) != 1)
{
// Check we are looking at a correctly aligned block
assert((curr - offset) % size == 0);
uint16_t start = remove_cache_friendly_offset(curr, sizeclass);
assert((start - offset) % size == 0);
// Account for free elements in free list
accounted_for += size;
assert(SLAB_SIZE >= accounted_for);
@@ -143,7 +150,8 @@ namespace snmalloc
}
// Check we terminated traversal on a correctly aligned block
assert(((curr & ~1) - offset) % size == 0);
uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass);
assert((start - offset) % size == 0);
if (curr != link)
{

29
src/mem/sizeclass.h
View File

@@ -6,6 +6,8 @@ namespace snmalloc
{
constexpr static uint16_t get_slab_offset(uint8_t sc, bool is_short);
constexpr static size_t sizeclass_to_size(uint8_t sizeclass);
constexpr static size_t sizeclass_to_cache_friendly_mask(uint8_t sizeclass);
constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(uint8_t sc);
constexpr static uint16_t medium_slab_free(uint8_t sizeclass);
static inline uint8_t size_to_sizeclass(size_t size)
@@ -137,4 +139,31 @@ namespace snmalloc
// everything fits into 32bits here.
return (uint32_t)(offset % rsize) == 0;
}
#ifdef CACHE_FRIENDLY_OFFSET
inline static void* remove_cache_friendly_offset(void* p, uint8_t sizeclass)
{
size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass);
return p = (void*)((uintptr_t)p & mask);
}
inline static uint16_t
remove_cache_friendly_offset(uint16_t relative, uint8_t sizeclass)
{
size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass);
return relative & mask;
}
#else
inline static void* remove_cache_friendly_offset(void* p, uint8_t sizeclass)
{
UNUSED(sizeclass);
return p;
}
inline static uint16_t
remove_cache_friendly_offset(uint16_t relative, uint8_t sizeclass)
{
UNUSED(sizeclass);
return relative;
}
#endif
};

21
src/mem/sizeclasstable.h
View File

@@ -8,6 +8,8 @@ namespace snmalloc
struct SizeClassTable
{
ModArray<NUM_SIZECLASSES, size_t> size;
ModArray<NUM_SIZECLASSES, size_t> cache_friendly_mask;
ModArray<NUM_SIZECLASSES, size_t> inverse_cache_friendly_mask;
ModArray<NUM_SMALL_CLASSES, uint16_t> bump_ptr_start;
ModArray<NUM_SMALL_CLASSES, uint16_t> short_bump_ptr_start;
ModArray<NUM_SMALL_CLASSES, uint16_t> count_per_slab;
@@ -15,6 +17,8 @@ namespace snmalloc
constexpr SizeClassTable()
: size(),
cache_friendly_mask(),
inverse_cache_friendly_mask(),
bump_ptr_start(),
short_bump_ptr_start(),
count_per_slab(),
@@ -24,6 +28,11 @@ namespace snmalloc
{
size[sizeclass] =
bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(sizeclass);
size_t alignment =
std::min((size_t)1 << bits::ctz_const(size[sizeclass]), OS_PAGE_SIZE);
cache_friendly_mask[sizeclass] = (alignment - 1);
inverse_cache_friendly_mask[sizeclass] = ~(alignment - 1);
}
size_t header_size = sizeof(Superslab);
@@ -60,6 +69,18 @@ namespace snmalloc
return sizeclass_metadata.size[sizeclass];
}
constexpr static inline size_t
sizeclass_to_cache_friendly_mask(uint8_t sizeclass)
{
return sizeclass_metadata.cache_friendly_mask[sizeclass];
}
constexpr static inline size_t
sizeclass_to_inverse_cache_friendly_mask(uint8_t sizeclass)
{
return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass];
}
constexpr static inline size_t sizeclass_to_count(uint8_t sizeclass)
{
return sizeclass_metadata.count_per_slab[sizeclass];

23
src/mem/slab.h
View File

@@ -48,11 +48,13 @@ namespace snmalloc
if ((head & 1) == 0)
{
p = (void*)((size_t)this + head);
void* node = (void*)((size_t)this + head);
// Read the next slot from the memory that's about to be allocated.
uint16_t next = *(uint16_t*)p;
uint16_t next = *(uint16_t*)node;
meta.head = next;
p = remove_cache_friendly_offset(node, meta.sizeclass);
}
else
{
@@ -82,6 +84,14 @@ namespace snmalloc
return p;
}
bool is_start_of_object(Superslab* super, void* p)
{
Metaslab& meta = super->get_meta(this);
return is_multiple_of_sizeclass(
sizeclass_to_size(meta.sizeclass),
(uintptr_t)this + SLAB_SIZE - (uintptr_t)p);
}
// Returns true, if it alters get_status.
template<typename MemoryProvider>
inline typename Superslab::Action dealloc(
@@ -93,15 +103,6 @@ namespace snmalloc
meta.debug_slab_invariant(is_short(), this);
meta.sub_use();
#ifndef SNMALLOC_SAFE_CLIENT
if (!is_multiple_of_sizeclass(
sizeclass_to_size(meta.sizeclass),
(uintptr_t)this + SLAB_SIZE - (uintptr_t)p))
{
error("Not deallocating start of an object");
}
#endif
if (was_full)
{
// We are not on the sizeclass list.