Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #124 from microsoft/alignment

Browse Source 
Make Large allocations naturally aligned
This commit is contained in:
Matthew Parkinson
2020-02-05 14:40:13 +00:00
committed by GitHub
parent f2c22ceed6 de64a8c0c2
commit 1d72024a9f
14 changed files with 133 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
src/mem/largealloc.h
View File

@@ -63,16 +63,12 @@ namespace snmalloc
void new_block()
{
size_t size = SUPERSLAB_SIZE;
void* r = reserve<false>(&size, SUPERSLAB_SIZE);
if (size < SUPERSLAB_SIZE)
error("out of memory");
// Reserve the smallest large_class which is SUPERSLAB_SIZE
void* r = reserve<false>(0);
PAL::template notify_using<NoZero>(r, OS_PAGE_SIZE);
bump = r;
remaining = size;
remaining = SUPERSLAB_SIZE;
}
/**
@@ -126,6 +122,24 @@ namespace snmalloc
lazy_decommit_guard.clear();
}
void push_space(address_t start, size_t large_class)
{
void* p = pointer_cast<void>(start);
if (large_class > 0)
PAL::template notify_using<YesZero>(p, OS_PAGE_SIZE);
else
{
if (decommit_strategy == DecommitSuperLazy)
{
PAL::template notify_using<YesZero>(p, OS_PAGE_SIZE);
p = new (p) Decommittedslab();
}
else
PAL::template notify_using<YesZero>(p, SUPERSLAB_SIZE);
}
large_stack[large_class].push(reinterpret_cast<Largeslab*>(p));
}
public:
/**
* Stack of large allocations that have been returned for reuse.
@@ -202,35 +216,68 @@ namespace snmalloc
}
template<bool committed>
void* reserve(size_t* size, size_t align) noexcept
void* reserve(size_t large_class) noexcept
{
size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
size_t align = size;
if constexpr (pal_supports<AlignedAllocation, PAL>)
{
return PAL::template reserve<committed>(size, align);
}
else
{
size_t request = *size;
// Add align, so we can guarantee to provide at least size.
request += align;
// Alignment must be a power of 2.
assert(align == bits::next_pow2(align));
// Reserve 4 times the amount, and put aligned leftovers into the
// large_stack
size_t request = bits::max(size * 4, SUPERSLAB_SIZE * 8);
void* p = PAL::template reserve<false>(request);
void* p = PAL::template reserve<committed>(&request);
address_t p0 = address_cast(p);
address_t start = bits::align_up(p0, align);
address_t p1 = p0 + request;
address_t end = start + size;
*size = request;
auto p0 = address_cast(p);
auto start = bits::align_up(p0, align);
if (start > p0)
for (; end < bits::align_down(p1, align); end += size)
{
uintptr_t end = bits::align_down(p0 + request, align);
*size = end - start;
PAL::notify_not_using(p, start - p0);
PAL::notify_not_using(pointer_cast<void>(end), (p0 + request) - end);
p = pointer_cast<void>(start);
push_space(end, large_class);
}
return p;
// Put offcuts before alignment into the large stack
address_t offcut_end = start;
address_t offcut_start;
for (size_t i = large_class; i > 0;)
{
i--;
size_t offcut_align = bits::one_at_bit(SUPERSLAB_BITS) << i;
offcut_start = bits::align_up(p0, offcut_align);
if (offcut_start != offcut_end)
{
push_space(offcut_start, i);
offcut_end = offcut_start;
}
}
// Put offcuts after returned block into the large stack
offcut_start = end;
for (size_t i = large_class; i > 0;)
{
i--;
auto offcut_align = bits::one_at_bit(SUPERSLAB_BITS) << i;
offcut_end = bits::align_down(p1, offcut_align);
if (offcut_start != offcut_end)
{
push_space(offcut_start, i);
offcut_start = offcut_end;
}
}
// printf("Alloc %zx (size = %zx)\n", start, size);
void* result = pointer_cast<void>(start);
if (committed)
PAL::template notify_using<NoZero>(result, size);
return result;
}
}
@@ -281,9 +328,6 @@ namespace snmalloc
template<class MemoryProvider>
class LargeAlloc
{
void* reserved_start = nullptr;
void* reserved_end = nullptr;
public:
// This will be a zero-size structure if stats are not enabled.
Stats stats;
@@ -292,38 +336,6 @@ namespace snmalloc
LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {}
template<AllowReserve allow_reserve>
bool reserve_memory(size_t need, size_t add)
{
assert(reserved_start <= reserved_end);
/*
* Spell this comparison in terms of pointer subtraction like this,
* rather than "reserved_start + need < reserved_end" because the
* sum might not be representable on CHERI.
*/
if (pointer_diff(reserved_start, reserved_end) < need)
{
if constexpr (allow_reserve == YesReserve)
{
stats.segment_create();
reserved_start =
memory_provider.template reserve<false>(&add, SUPERSLAB_SIZE);
reserved_end = pointer_offset(reserved_start, add);
reserved_start = pointer_align_up<SUPERSLAB_SIZE>(reserved_start);
if (add < need)
return false;
}
else
{
return false;
}
}
return true;
}
template<ZeroMem zero_mem = NoZero, AllowReserve allow_reserve = YesReserve>
void* alloc(size_t large_class, size_t size)
{
@@ -337,23 +349,8 @@ namespace snmalloc
if (p == nullptr)
{
assert(reserved_start <= reserved_end);
size_t add;
if ((rsize + SUPERSLAB_SIZE) < RESERVE_SIZE)
add = RESERVE_SIZE;
else
add = rsize + SUPERSLAB_SIZE;
if (!reserve_memory<allow_reserve>(rsize, add))
return nullptr;
p = reserved_start;
reserved_start = pointer_offset(p, rsize);
stats.superslab_fresh();
// All memory is zeroed since it comes from reserved space.
memory_provider.template notify_using<NoZero>(p, size);
p = memory_provider.template reserve<false>(large_class);
memory_provider.template notify_using<zero_mem>(p, size);
}
else
{
@@ -389,6 +386,7 @@ namespace snmalloc
}
}
assert(p == pointer_align_up(p, rsize));
return p;
}

2
src/mem/sizeclass.h
View File

@@ -179,8 +179,6 @@ namespace snmalloc
{
// Client responsible for checking alignment is not zero
assert(alignment != 0);
// Client responsible for checking alignment is not above SUPERSLAB_SIZE
assert(alignment <= SUPERSLAB_SIZE);
// Client responsible for checking alignment is a power of two
assert(bits::next_pow2(alignment) == alignment);

4
src/override/malloc.cc
View File

@@ -107,9 +107,7 @@ extern "C"
SNMALLOC_EXPORT void*
SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size)
{
if (
(alignment == 0) || (alignment == size_t(-1)) ||
(alignment > SUPERSLAB_SIZE))
if ((alignment == 0) || (alignment == size_t(-1)))
{
errno = EINVAL;
return nullptr;

4
src/pal/pal_apple.h
View File

@@ -56,11 +56,11 @@ namespace snmalloc
* See comment below.
*/
template<bool committed>
void* reserve(const size_t* size)
void* reserve(size_t size)
{
void* p = mmap(
nullptr,
*size,
size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
pal_anon_id,

4
src/pal/pal_bsd_aligned.h
View File

@@ -27,7 +27,7 @@ namespace snmalloc
* Reserve memory at a specific alignment.
*/
template<bool committed>
void* reserve(const size_t* size, size_t align) noexcept
void* reserve(size_t size, size_t align) noexcept
{
// Alignment must be a power of 2.
assert(align == bits::next_pow2(align));
@@ -38,7 +38,7 @@ namespace snmalloc
void* p = mmap(
nullptr,
*size,
size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED(log2align),
-1,

9
src/pal/pal_freebsd_kernel.h
View File

@@ -61,13 +61,12 @@ namespace snmalloc
}
template<bool committed>
void* reserve(size_t* size, size_t align)
void* reserve(size_t size, size_t align)
{
size_t request = *size;
vm_offset_t addr;
if (vmem_xalloc(
kernel_arena,
request,
size,
align,
0,
0,
@@ -81,10 +80,10 @@ namespace snmalloc
if (committed)
{
if (
kmem_back(kernel_object, addr, request, M_ZERO | M_WAITOK) !=
kmem_back(kernel_object, addr, size, M_ZERO | M_WAITOK) !=
KERN_SUCCESS)
{
vmem_xfree(kernel_arena, addr, request);
vmem_xfree(kernel_arena, addr, size);
return nullptr;
}
}

11
src/pal/pal_open_enclave.h
View File

@@ -18,7 +18,7 @@ namespace snmalloc
* Bitmap of PalFeatures flags indicating the optional features that this
* PAL supports.
*/
static constexpr uint64_t pal_features = AlignedAllocation;
static constexpr uint64_t pal_features = 0;
static void error(const char* const str)
{
UNUSED(str);
@@ -26,7 +26,7 @@ namespace snmalloc
}
template<bool committed>
void* reserve(size_t* size, size_t align) noexcept
void* reserve(size_t size) noexcept
{
if (oe_base == 0)
{
@@ -36,21 +36,18 @@ namespace snmalloc
}
void* old_base = oe_base;
void* old_base2 = old_base;
void* next_base;
auto end = __oe_get_heap_end();
do
{
old_base2 = old_base;
auto new_base = pointer_align_up(old_base, align);
next_base = pointer_offset(new_base, *size);
auto new_base = old_base;
next_base = pointer_offset(new_base, size);
if (next_base > end)
error("Out of memory");
} while (oe_base.compare_exchange_strong(old_base, next_base));
*size = pointer_diff(old_base2, next_base);
return old_base;
}

20
src/pal/pal_posix.h
View File

@@ -54,8 +54,12 @@ namespace snmalloc
void notify_not_using(void* p, size_t size) noexcept
{
assert(is_aligned_block<OS_PAGE_SIZE>(p, size));
#ifdef USE_POSIX_COMMIT_CHECKS
mprotect(p, size, PROT_NONE);
#else
UNUSED(p);
UNUSED(size);
#endif
}
/**
@@ -72,11 +76,13 @@ namespace snmalloc
if constexpr (zero_mem == YesZero)
static_cast<OS*>(this)->template zero<true>(p, size);
else
{
UNUSED(p);
UNUSED(size);
}
#ifdef USE_POSIX_COMMIT_CHECKS
mprotect(p, size, PROT_READ | PROT_WRITE);
#else
UNUSED(p);
UNUSED(size);
#endif
}
/**
@@ -121,11 +127,11 @@ namespace snmalloc
* greater than a page.
*/
template<bool committed>
void* reserve(const size_t* size) noexcept
void* reserve(size_t size) noexcept
{
void* p = mmap(
nullptr,
*size,
size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
-1,

8
src/pal/pal_windows.h
View File

@@ -183,7 +183,7 @@ namespace snmalloc
}
# elif defined(PLATFORM_HAS_VIRTUALALLOC2)
template<bool committed>
void* reserve(size_t* size, size_t align) noexcept
void* reserve(size_t size, size_t align) noexcept
{
DWORD flags = MEM_RESERVE;
@@ -209,7 +209,7 @@ namespace snmalloc
param.Pointer = &addressReqs;
void* ret = VirtualAlloc2FromApp(
nullptr, nullptr, *size, flags, PAGE_READWRITE, &param, 1);
nullptr, nullptr, size, flags, PAGE_READWRITE, &param, 1);
if (ret == nullptr)
{
error("Failed to allocate memory\n");
@@ -218,14 +218,14 @@ namespace snmalloc
}
# else
template<bool committed>
void* reserve(size_t* size) noexcept
void* reserve(size_t size) noexcept
{
DWORD flags = MEM_RESERVE;
if (committed)
flags |= MEM_COMMIT;
void* ret = VirtualAlloc(nullptr, *size, flags, PAGE_READWRITE);
void* ret = VirtualAlloc(nullptr, size, flags, PAGE_READWRITE);
if (ret == nullptr)
{
error("Failed to allocate memory\n");

8
src/test/func/fixed_region/fixed_region.cc
View File

@@ -32,8 +32,12 @@ int main()
{
MemoryProviderStateMixin<DefaultPal> mp;
size_t size = 1ULL << 28;
oe_base = mp.reserve<true>(&size, 0);
// 28 is large enough to produce a nested allocator.
// It is also large enough for the example to run in.
// For 1MiB superslabs, SUPERSLAB_BITS + 4 is not big enough for the example.
size_t large_class = 28 - SUPERSLAB_BITS;
size_t size = 1ULL << (SUPERSLAB_BITS + large_class);
oe_base = mp.reserve<true>(large_class);
oe_end = (uint8_t*)oe_base + size;
std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;

3
src/test/func/malloc/malloc.cc
View File

@@ -108,7 +108,8 @@ int main(int argc, char** argv)
test_posix_memalign((size_t)-1, 0, EINVAL, true);
test_posix_memalign(OS_PAGE_SIZE, sizeof(uintptr_t) / 2, EINVAL, true);
for (size_t align = sizeof(uintptr_t); align <= SUPERSLAB_SIZE; align <<= 1)
for (size_t align = sizeof(uintptr_t); align <= SUPERSLAB_SIZE * 8;
align <<= 1)
{
for (snmalloc::sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++)
{

8
src/test/func/two_alloc_types/main.cc
View File

@@ -45,8 +45,12 @@ int main()
MemoryProviderStateMixin<DefaultPal> mp;
size_t size = 1ULL << 26;
oe_base = mp.reserve<true>(&size, 1);
// 26 is large enough to produce a nested allocator.
// It is also large enough for the example to run in.
// For 1MiB superslabs, SUPERSLAB_BITS + 2 is not big enough for the example.
size_t large_class = 26 - SUPERSLAB_BITS;
size_t size = 1ULL << (SUPERSLAB_BITS + large_class);
oe_base = mp.reserve<true>(large_class);
oe_end = (uint8_t*)oe_base + size;
std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;