Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Major refactor of snmalloc (#343)

Browse Source 
# Pagemap
 
The Pagemap now stores all the meta-data for the object allocation. The meta-data in the pagemap is effectively a triple of the sizeclass, the remote allocator, and a pointer to a 64 byte block of meta-data for this chunk of memory. By storing the pointer to a block, it allows the pagemap to handle multiple slab sizes without branching on the fast path. There is one entry in the pagemap per 16KiB of address space, but by using the same entry in the pagemap for 4 adjacent entries, then we can treat a 64KiB range can be treated as a single slab of allocations.

This change also means there is almost no capability amplification required by the implementation on CHERI for finding meta-data. The only amplification is required, when we change the way a chunk is used to a size of object allocation.


# Backend

There is a second major aspect of the refactor that there is now a narrow API that abstracts the Pagemap, PAL and address space management. This should better enable the compartmentalisation and makes it easier to produce alternative backends for various research directions. This is a template parameter that can be used to specialised by the front-end in different ways.

# Thread local state

The thread local state has been refactored into two components, one (called 'localalloc') that is stored directly in the TLS and is constant initialised, and one that is allocated in the address space (called 'coreallloc') which is lazily created and pooled.

# Difference

This removes Superslabs/Medium slabs as there meta-data is now part of the pagemap.
This commit is contained in:
Matthew Parkinson
2021-07-12 15:53:36 +01:00
committed by GitHub
parent 18d7cc99b6
commit f0e2ab702a
83 changed files with 4404 additions and 5769 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
src/test/func/memory/memory.cc
View File

@@ -19,13 +19,17 @@
# define KiB (1024ull)
# define MiB (KiB * KiB)
# define GiB (KiB * MiB)
#else
using rlim64_t = size_t;
#endif
using namespace snmalloc;
#ifdef TEST_LIMITED
void test_limited(rlim64_t as_limit, size_t& count)
{
UNUSED(as_limit);
UNUSED(count);
#if false && defined(TEST_LIMITED)
auto pid = fork();
if (!pid)
{
@@ -54,10 +58,10 @@ void test_limited(rlim64_t as_limit, size_t& count)
upper_bound = std::min(
upper_bound, static_cast<unsigned long long>(info.freeram >> 3u));
std::cout << "trying to alloc " << upper_bound / KiB << " KiB" << std::endl;
auto alloc = ThreadAlloc::get();
auto& alloc = ThreadAlloc::get();
std::cout << "allocator initialised" << std::endl;
auto chunk = alloc->alloc(upper_bound);
alloc->dealloc(chunk);
auto chunk = alloc.alloc(upper_bound);
alloc.dealloc(chunk);
std::cout << "success" << std::endl;
std::exit(0);
}
@@ -71,12 +75,12 @@ void test_limited(rlim64_t as_limit, size_t& count)
count++;
}
}
}
#endif
}
void test_alloc_dealloc_64k()
{
auto alloc = ThreadAlloc::get();
auto& alloc = ThreadAlloc::get();
constexpr size_t count = 1 << 12;
constexpr size_t outer_count = 12;
@@ -89,26 +93,26 @@ void test_alloc_dealloc_64k()
// This will fill the short slab, and then start a new slab.
for (size_t i = 0; i < count; i++)
{
garbage[i] = alloc->alloc(16);
garbage[i] = alloc.alloc(16);
}
// Allocate one object on the second slab
keep_alive[j] = alloc->alloc(16);
keep_alive[j] = alloc.alloc(16);
for (size_t i = 0; i < count; i++)
{
alloc->dealloc(garbage[i]);
alloc.dealloc(garbage[i]);
}
}
for (size_t j = 0; j < outer_count; j++)
{
alloc->dealloc(keep_alive[j]);
alloc.dealloc(keep_alive[j]);
}
}
void test_random_allocation()
{
auto alloc = ThreadAlloc::get();
auto& alloc = ThreadAlloc::get();
std::unordered_set<void*> allocated;
constexpr size_t count = 10000;
@@ -130,14 +134,14 @@ void test_random_allocation()
auto& cell = objects[index % count];
if (cell != nullptr)
{
alloc->dealloc(cell);
alloc.dealloc(cell);
allocated.erase(cell);
cell = nullptr;
alloc_count--;
}
if (!just_dealloc)
{
cell = alloc->alloc(16);
cell = alloc.alloc(16);
auto pair = allocated.insert(cell);
// Check not already allocated
SNMALLOC_CHECK(pair.second);
@@ -155,20 +159,20 @@ void test_random_allocation()
// Deallocate all the remaining objects
for (size_t i = 0; i < count; i++)
if (objects[i] != nullptr)
alloc->dealloc(objects[i]);
alloc.dealloc(objects[i]);
}
void test_calloc()
{
auto alloc = ThreadAlloc::get();
auto& alloc = ThreadAlloc::get();
for (size_t size = 16; size <= (1 << 24); size <<= 1)
{
void* p = alloc->alloc(size);
void* p = alloc.alloc(size);
memset(p, 0xFF, size);
alloc->dealloc(p, size);
alloc.dealloc(p, size);
p = alloc->alloc<YesZero>(size);
p = alloc.alloc<YesZero>(size);
for (size_t i = 0; i < size; i++)
{
@@ -176,92 +180,97 @@ void test_calloc()
abort();
}
alloc->dealloc(p, size);
alloc.dealloc(p, size);
}
current_alloc_pool()->debug_check_empty();
snmalloc::debug_check_empty(Globals::get_handle());
}
void test_double_alloc()
{
auto* a1 = current_alloc_pool()->acquire();
auto* a2 = current_alloc_pool()->acquire();
const size_t n = (1 << 16) / 32;
for (size_t k = 0; k < 4; k++)
{
std::unordered_set<void*> set1;
std::unordered_set<void*> set2;
auto a1 = snmalloc::get_scoped_allocator();
auto a2 = snmalloc::get_scoped_allocator();
for (size_t i = 0; i < (n * 2); i++)
{
void* p = a1->alloc(20);
SNMALLOC_CHECK(set1.find(p) == set1.end());
set1.insert(p);
}
const size_t n = (1 << 16) / 32;
for (size_t i = 0; i < (n * 2); i++)
for (size_t k = 0; k < 4; k++)
{
void* p = a2->alloc(20);
SNMALLOC_CHECK(set2.find(p) == set2.end());
set2.insert(p);
}
std::unordered_set<void*> set1;
std::unordered_set<void*> set2;
while (!set1.empty())
{
auto it = set1.begin();
a2->dealloc(*it, 20);
set1.erase(it);
}
for (size_t i = 0; i < (n * 2); i++)
{
void* p = a1->alloc(20);
SNMALLOC_CHECK(set1.find(p) == set1.end());
set1.insert(p);
}
while (!set2.empty())
{
auto it = set2.begin();
a1->dealloc(*it, 20);
set2.erase(it);
for (size_t i = 0; i < (n * 2); i++)
{
void* p = a2->alloc(20);
SNMALLOC_CHECK(set2.find(p) == set2.end());
set2.insert(p);
}
while (!set1.empty())
{
auto it = set1.begin();
a2->dealloc(*it, 20);
set1.erase(it);
}
while (!set2.empty())
{
auto it = set2.begin();
a1->dealloc(*it, 20);
set2.erase(it);
}
}
}
current_alloc_pool()->release(a1);
current_alloc_pool()->release(a2);
current_alloc_pool()->debug_check_empty();
snmalloc::debug_check_empty(Globals::get_handle());
}
void test_external_pointer()
{
// Malloc does not have an external pointer querying mechanism.
auto alloc = ThreadAlloc::get();
auto& alloc = ThreadAlloc::get();
for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++)
{
size_t size = sizeclass_to_size(sc);
void* p1 = alloc->alloc(size);
void* p1 = alloc.alloc(size);
for (size_t offset = 0; offset < size; offset += 17)
{
void* p2 = pointer_offset(p1, offset);
void* p3 = alloc->external_pointer(p2);
void* p4 = alloc->external_pointer<End>(p2);
void* p3 = alloc.external_pointer(p2);
void* p4 = alloc.external_pointer<End>(p2);
if (p1 != p3)
{
std::cout << "size: " << size << " offset: " << offset << " p1: " << p1
<< " p3: " << p3 << std::endl;
}
SNMALLOC_CHECK(p1 == p3);
if ((size_t)p4 != (size_t)p1 + size - 1)
{
std::cout << "size: " << size << " end(p4): " << p4 << " p1: " << p1
<< " p1+size-1: " << (void*)((size_t)p1 + size - 1)
<< std::endl;
}
SNMALLOC_CHECK((size_t)p4 == (size_t)p1 + size - 1);
}
alloc->dealloc(p1, size);
alloc.dealloc(p1, size);
}
current_alloc_pool()->debug_check_empty();
snmalloc::debug_check_empty(Globals::get_handle());
};
void check_offset(void* base, void* interior)
{
auto alloc = ThreadAlloc::get();
void* calced_base = alloc->external_pointer((void*)interior);
auto& alloc = ThreadAlloc::get();
void* calced_base = alloc.external_pointer((void*)interior);
if (calced_base != (void*)base)
abort();
}
@@ -281,7 +290,7 @@ void test_external_pointer_large()
{
xoroshiro::p128r64 r;
auto alloc = ThreadAlloc::get();
auto& alloc = ThreadAlloc::get();
constexpr size_t count_log = snmalloc::bits::is64() ? 5 : 3;
constexpr size_t count = 1 << count_log;
@@ -292,14 +301,14 @@ void test_external_pointer_large()
for (size_t i = 0; i < count; i++)
{
size_t b = SUPERSLAB_BITS + 3;
size_t b = MAX_SIZECLASS_BITS + 3;
size_t rand = r.next() & ((1 << b) - 1);
size_t size = (1 << 24) + rand;
total_size += size;
// store object
objects[i] = (size_t*)alloc->alloc(size);
objects[i] = (size_t*)alloc.alloc(size);
// Store allocators size for this object
*objects[i] = alloc->alloc_size(objects[i]);
*objects[i] = alloc.alloc_size(objects[i]);
check_external_pointer_large(objects[i]);
if (i > 0)
@@ -317,87 +326,87 @@ void test_external_pointer_large()
// Deallocate everything
for (size_t i = 0; i < count; i++)
{
alloc->dealloc(objects[i]);
alloc.dealloc(objects[i]);
}
}
void test_external_pointer_dealloc_bug()
{
auto alloc = ThreadAlloc::get();
constexpr size_t count = (SUPERSLAB_SIZE / SLAB_SIZE) * 2;
auto& alloc = ThreadAlloc::get();
constexpr size_t count = MIN_CHUNK_SIZE;
void* allocs[count];
for (size_t i = 0; i < count; i++)
{
allocs[i] = alloc->alloc(SLAB_SIZE / 2);
allocs[i] = alloc.alloc(MIN_CHUNK_BITS / 2);
}
for (size_t i = 1; i < count; i++)
{
alloc->dealloc(allocs[i]);
alloc.dealloc(allocs[i]);
}
for (size_t i = 0; i < count; i++)
{
alloc->external_pointer(allocs[i]);
alloc.external_pointer(allocs[i]);
}
alloc->dealloc(allocs[0]);
alloc.dealloc(allocs[0]);
}
void test_alloc_16M()
{
auto alloc = ThreadAlloc::get();
auto& alloc = ThreadAlloc::get();
// sizes >= 16M use large_alloc
const size_t size = 16'000'000;
void* p1 = alloc->alloc(size);
SNMALLOC_CHECK(alloc->alloc_size(alloc->external_pointer(p1)) >= size);
alloc->dealloc(p1);
void* p1 = alloc.alloc(size);
SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size);
alloc.dealloc(p1);
}
void test_calloc_16M()
{
auto alloc = ThreadAlloc::get();
auto& alloc = ThreadAlloc::get();
// sizes >= 16M use large_alloc
const size_t size = 16'000'000;
void* p1 = alloc->alloc<YesZero>(size);
SNMALLOC_CHECK(alloc->alloc_size(alloc->external_pointer(p1)) >= size);
alloc->dealloc(p1);
void* p1 = alloc.alloc<YesZero>(size);
SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size);
alloc.dealloc(p1);
}
void test_calloc_large_bug()
{
auto alloc = ThreadAlloc::get();
auto& alloc = ThreadAlloc::get();
// Perform large calloc, to check for correct zeroing from PAL.
// Some PALS have special paths for PAGE aligned zeroing of large
// allocations. This is a large allocation that is intentionally
// not a multiple of page size.
const size_t size = (SUPERSLAB_SIZE << 3) - 7;
const size_t size = (MAX_SIZECLASS_SIZE << 3) - 7;
void* p1 = alloc->alloc<YesZero>(size);
SNMALLOC_CHECK(alloc->alloc_size(alloc->external_pointer(p1)) >= size);
alloc->dealloc(p1);
void* p1 = alloc.alloc<YesZero>(size);
SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size);
alloc.dealloc(p1);
}
template<size_t asz, int dealloc>
void test_static_sized_alloc()
{
auto alloc = ThreadAlloc::get();
auto p = alloc->alloc<asz>();
auto& alloc = ThreadAlloc::get();
auto p = alloc.alloc<asz>();
static_assert((dealloc >= 0) && (dealloc <= 2), "bad dealloc flavor");
switch (dealloc)
{
case 0:
alloc->dealloc(p);
alloc.dealloc(p);
break;
case 1:
alloc->dealloc(p, asz);
alloc.dealloc(p, asz);
break;
case 2:
alloc->dealloc<asz>(p);
alloc.dealloc<asz>(p);
break;
}
}
@@ -406,18 +415,19 @@ void test_static_sized_allocs()
{
// For each small, medium, and large class, do each kind dealloc. This is
// mostly to ensure that all of these forms compile.
for (size_t sc = 0; sc < NUM_SIZECLASSES_EXTENDED; sc++)
{
// test_static_sized_alloc<sc, 0>();
// test_static_sized_alloc<sc, 1>();
// test_static_sized_alloc<sc, 2>();
}
// test_static_sized_alloc<sizeclass_to_size(NUM_SMALL_CLASSES + 1), 0>();
// test_static_sized_alloc<sizeclass_to_size(NUM_SMALL_CLASSES + 1), 1>();
// test_static_sized_alloc<sizeclass_to_size(NUM_SMALL_CLASSES + 1), 2>();
test_static_sized_alloc<sizeclass_to_size(1), 0>();
test_static_sized_alloc<sizeclass_to_size(1), 1>();
test_static_sized_alloc<sizeclass_to_size(1), 2>();
test_static_sized_alloc<sizeclass_to_size(NUM_SMALL_CLASSES + 1), 0>();
test_static_sized_alloc<sizeclass_to_size(NUM_SMALL_CLASSES + 1), 1>();
test_static_sized_alloc<sizeclass_to_size(NUM_SMALL_CLASSES + 1), 2>();
test_static_sized_alloc<large_sizeclass_to_size(0), 0>();
test_static_sized_alloc<large_sizeclass_to_size(0), 1>();
test_static_sized_alloc<large_sizeclass_to_size(0), 2>();
// test_static_sized_alloc<large_sizeclass_to_size(0), 0>();
// test_static_sized_alloc<large_sizeclass_to_size(0), 1>();
// test_static_sized_alloc<large_sizeclass_to_size(0), 2>();
}
int main(int argc, char** argv)