Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SP: use CapPtr<>s in address_space, largealloc

Browse Source
This commit is contained in:
Nathaniel Filardo
2021-03-23 14:56:24 +00:00
committed by Nathaniel Wesley Filardo
parent 005f5787ef
commit d9ee19a16c
7 changed files with 78 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/mem/address_space.h
View File

@@ -34,7 +34,7 @@ namespace snmalloc
* bits::BITS is used for simplicity, we do not use below the pointer size,
* and large entries will be unlikely to be supported by the platform.
*/
std::array<std::array<void*, 2>, bits::BITS> ranges = {};
std::array<std::array<CapPtr<void, CBArena>, 2>, bits::BITS> ranges = {};
/**
* This is infrequently used code, a spin lock simplifies the code
@@ -45,7 +45,7 @@ namespace snmalloc
/**
* Checks a block satisfies its invariant.
*/
inline void check_block(void* base, size_t align_bits)
inline void check_block(CapPtr<void, CBArena> base, size_t align_bits)
{
SNMALLOC_ASSERT(
base == pointer_align_up(base, bits::one_at_bit(align_bits)));
@@ -58,7 +58,7 @@ namespace snmalloc
/**
* Adds a block to `ranges`.
*/
void add_block(size_t align_bits, void* base)
void add_block(size_t align_bits, CapPtr<void, CBArena> base)
{
check_block(base, align_bits);
SNMALLOC_ASSERT(align_bits < 64);
@@ -73,7 +73,8 @@ namespace snmalloc
{
// Add to linked list.
commit_block(base, sizeof(void*));
*reinterpret_cast<void**>(base) = ranges[align_bits][1];
*(base.template as_static<CapPtr<void, CBArena>>().unsafe_capptr) =
ranges[align_bits][1];
check_block(ranges[align_bits][1], align_bits);
}
@@ -86,9 +87,9 @@ namespace snmalloc
* Find a block of the correct size. May split larger blocks
* to satisfy this request.
*/
void* remove_block(size_t align_bits)
CapPtr<void, CBArena> remove_block(size_t align_bits)
{
auto first = ranges[align_bits][0];
CapPtr<void, CBArena> first = ranges[align_bits][0];
if (first == nullptr)
{
if (align_bits == (bits::BITS - 1))
@@ -98,7 +99,7 @@ namespace snmalloc
}
// Look for larger block and split up recursively
void* bigger = remove_block(align_bits + 1);
CapPtr<void, CBArena> bigger = remove_block(align_bits + 1);
if (bigger != nullptr)
{
auto left_over = pointer_offset(bigger, bits::one_at_bit(align_bits));
@@ -109,14 +110,16 @@ namespace snmalloc
return bigger;
}
auto second = ranges[align_bits][1];
CapPtr<void, CBArena> second = ranges[align_bits][1];
if (second != nullptr)
{
commit_block(second, sizeof(void*));
auto next = *reinterpret_cast<void**>(second);
auto psecond =
second.template as_static<CapPtr<void, CBArena>>().unsafe_capptr;
auto next = *psecond;
ranges[align_bits][1] = next;
// Zero memory. Client assumes memory contains only zeros.
*reinterpret_cast<void**>(second) = nullptr;
*psecond = nullptr;
check_block(second, align_bits);
check_block(next, align_bits);
return second;
@@ -131,7 +134,7 @@ namespace snmalloc
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(void* base, size_t length)
void add_range(CapPtr<void, CBArena> base, size_t length)
{
// Find the minimum set of maximally aligned blocks in this range.
// Each block's alignment and size are equal.
@@ -153,14 +156,14 @@ namespace snmalloc
/**
* Commit a block of memory
*/
void commit_block(void* base, size_t size)
void commit_block(CapPtr<void, CBArena> base, size_t size)
{
// Rounding required for sub-page allocations.
auto page_start = pointer_align_down<OS_PAGE_SIZE, char>(base);
auto page_end =
pointer_align_up<OS_PAGE_SIZE, char>(pointer_offset(base, size));
PAL::template notify_using<NoZero>(
page_start, static_cast<size_t>(page_end - page_start));
size_t using_size = pointer_diff(page_start, page_end);
PAL::template notify_using<NoZero>(page_start.unsafe_capptr, using_size);
}
public:
@@ -172,7 +175,7 @@ namespace snmalloc
* Only request 2^n sizes, and not less than a pointer.
*/
template<bool committed>
void* reserve(size_t size)
CapPtr<void, CBArena> reserve(size_t size)
{
SNMALLOC_ASSERT(bits::is_pow2(size));
SNMALLOC_ASSERT(size >= sizeof(void*));
@@ -180,22 +183,24 @@ namespace snmalloc
if constexpr (pal_supports<AlignedAllocation, PAL>)
{
if (size >= PAL::minimum_alloc_size)
return PAL::template reserve_aligned<committed>(size);
return CapPtr<void, CBArena>(
PAL::template reserve_aligned<committed>(size));
}
void* res;
CapPtr<void, CBArena> res;
{
FlagLock lock(spin_lock);
res = remove_block(bits::next_pow2_bits(size));
if (res == nullptr)
{
// Allocation failed ask OS for more memory
void* block = nullptr;
CapPtr<void, CBArena> block = nullptr;
size_t block_size = 0;
if constexpr (pal_supports<AlignedAllocation, PAL>)
{
block_size = PAL::minimum_alloc_size;
block = PAL::template reserve_aligned<false>(block_size);
block = CapPtr<void, CBArena>(
PAL::template reserve_aligned<false>(block_size));
}
else if constexpr (!pal_supports<NoAllocation, PAL>)
{
@@ -204,7 +209,7 @@ namespace snmalloc
// the PAL, and this could lead to suprious OOM. This is
// particularly bad if the PAL gives all the memory on first call.
auto block_and_size = PAL::reserve_at_least(size * 2);
block = block_and_size.first;
block = CapPtr<void, CBArena>(block_and_size.first);
block_size = block_and_size.second;
// Ensure block is pointer aligned.
@@ -244,7 +249,7 @@ namespace snmalloc
* used, by smaller objects.
*/
template<bool committed>
void* reserve_with_left_over(size_t size)
CapPtr<void, CBArena> reserve_with_left_over(size_t size)
{
SNMALLOC_ASSERT(size >= sizeof(void*));
@@ -279,7 +284,7 @@ namespace snmalloc
* Constructor that pre-initialises the address-space manager with a region
* of memory.
*/
AddressSpaceManager(void* base, size_t length)
AddressSpaceManager(CapPtr<void, CBArena> base, size_t length)
{
add_range(base, length);
}

17
src/mem/alloc.h
View File

@@ -955,9 +955,10 @@ namespace snmalloc
if (super != nullptr)
return super;
super =
reinterpret_cast<Superslab*>(large_allocator.template alloc<NoZero>(
0, SUPERSLAB_SIZE, SUPERSLAB_SIZE));
super = large_allocator
.template alloc<NoZero>(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE)
.template as_reinterpret<Superslab>()
.unsafe_capptr;
if (super == nullptr)
return super;
@@ -1371,9 +1372,9 @@ namespace snmalloc
sizeclass, rsize, size);
});
}
slab = CapPtr<void, CBArena>(large_allocator.template alloc<NoZero>(
0, SUPERSLAB_SIZE, SUPERSLAB_SIZE))
.template as_static<Mediumslab>();
slab = large_allocator
.template alloc<NoZero>(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE)
.template as_reinterpret<Mediumslab>();
if (slab == nullptr)
return nullptr;
@@ -1503,7 +1504,7 @@ namespace snmalloc
if (large_class == 0)
size = rsize;
void* p =
auto p =
large_allocator.template alloc<zero_mem>(large_class, rsize, size);
if (likely(p != nullptr))
{
@@ -1512,7 +1513,7 @@ namespace snmalloc
stats().alloc_request(size);
stats().large_alloc(large_class);
}
return p;
return p.unsafe_capptr;
}
void large_dealloc_unchecked(

2
src/mem/chunkmap.h
View File

@@ -149,7 +149,7 @@ namespace snmalloc
* Update the pagemap to reflect a large allocation, of `size` bytes from
* address `p`.
*/
static void set_large_size(void* p, size_t size)
static void set_large_size(CapPtr<Largeslab, CBArena> p, size_t size)
{
size_t size_bits = bits::next_pow2_bits(size);
set(address_cast(p), static_cast<uint8_t>(size_bits));

59
src/mem/largealloc.h
View File

@@ -32,7 +32,7 @@ namespace snmalloc
friend class MPMCStack;
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
friend class MemoryProviderStateMixin;
std::atomic<Largeslab*> next;
AtomicCapPtr<Largeslab, CBArena> next = nullptr;
public:
void init()
@@ -89,7 +89,10 @@ namespace snmalloc
/**
* Stack of large allocations that have been returned for reuse.
*/
ModArray<NUM_LARGE_CLASSES, MPMCStack<Largeslab, RequiresInit>> large_stack;
ModArray<
NUM_LARGE_CLASSES,
MPMCStack<Largeslab, RequiresInit, CapPtrCBArena, AtomicCapPtrCBArena>>
large_stack;
public:
using Pal = PAL;
@@ -99,9 +102,10 @@ namespace snmalloc
* concurrently with other acceses. If there is no large allocation on a
* particular stack then this will return `nullptr`.
*/
SNMALLOC_FAST_PATH void* pop_large_stack(size_t large_class)
SNMALLOC_FAST_PATH CapPtr<Largeslab, CBArena>
pop_large_stack(size_t large_class)
{
void* p = large_stack[large_class].pop();
auto p = large_stack[large_class].pop();
if (p != nullptr)
{
const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
@@ -119,7 +123,7 @@ namespace snmalloc
{
const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
available_large_chunks_in_bytes += rsize;
large_stack[large_class].push(slab.unsafe_capptr);
large_stack[large_class].push(slab);
}
/**
@@ -133,7 +137,7 @@ namespace snmalloc
* memory providers constructed in this way does not have to be able to
* allocate memory, if the initial reservation is sufficient.
*/
MemoryProviderStateMixin(void* start, size_t len)
MemoryProviderStateMixin(CapPtr<void, CBArena> start, size_t len)
: address_space(start, len)
{}
/**
@@ -146,9 +150,11 @@ namespace snmalloc
// Allocate permanent storage for the allocator usung temporary allocator
MemoryProviderStateMixin<PAL>* allocated =
reinterpret_cast<MemoryProviderStateMixin<PAL>*>(
local.template reserve_with_left_over<true>(
sizeof(MemoryProviderStateMixin<PAL>)));
local
.template reserve_with_left_over<true>(
sizeof(MemoryProviderStateMixin<PAL>))
.template as_static<MemoryProviderStateMixin<PAL>>()
.unsafe_capptr;
if (allocated == nullptr)
error("Failed to initialise system!");
@@ -193,22 +199,23 @@ namespace snmalloc
size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
size_t decommit_size = rsize - OS_PAGE_SIZE;
// Grab all of the chunks of this size class.
auto* slab = large_stack[large_class].pop_all();
while (slab)
CapPtr<Largeslab, CBArena> slab = large_stack[large_class].pop_all();
while (slab != nullptr)
{
// Decommit all except for the first page and then put it back on
// the stack.
if (slab->get_kind() != Decommitted)
{
PAL::notify_not_using(
pointer_offset(slab, OS_PAGE_SIZE), decommit_size);
pointer_offset(slab.unsafe_capptr, OS_PAGE_SIZE), decommit_size);
}
// Once we've removed these from the stack, there will be no
// concurrent accesses and removal should have established a
// happens-before relationship, so it's safe to use relaxed loads
// here.
auto next = slab->next.load(std::memory_order_relaxed);
large_stack[large_class].push(new (slab) Decommittedslab());
large_stack[large_class].push(CapPtr<Largeslab, CBArena>(
new (slab.unsafe_capptr) Decommittedslab()));
slab = next;
}
}
@@ -247,21 +254,22 @@ namespace snmalloc
// Cache line align
size_t size = bits::align_up(sizeof(T), 64);
size = bits::max(size, alignment);
void* p = address_space.template reserve_with_left_over<true>(size);
auto p = address_space.template reserve_with_left_over<true>(size);
if (p == nullptr)
return nullptr;
peak_memory_used_bytes += size;
return new (p) T(std::forward<Args...>(args)...);
return new (p.unsafe_capptr) T(std::forward<Args...>(args)...);
}
template<bool committed>
void* reserve(size_t large_class) noexcept
CapPtr<Largeslab, CBArena> reserve(size_t large_class) noexcept
{
size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
peak_memory_used_bytes += size;
return address_space.template reserve<committed>(size);
return address_space.template reserve<committed>(size)
.template as_static<Largeslab>();
}
/**
@@ -290,19 +298,22 @@ namespace snmalloc
LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {}
template<ZeroMem zero_mem = NoZero>
void* alloc(size_t large_class, size_t rsize, size_t size)
CapPtr<Largeslab, CBArena>
alloc(size_t large_class, size_t rsize, size_t size)
{
SNMALLOC_ASSERT(
(bits::one_at_bit(SUPERSLAB_BITS) << large_class) == rsize);
void* p = memory_provider.pop_large_stack(large_class);
CapPtr<Largeslab, CBArena> p =
memory_provider.pop_large_stack(large_class);
if (p == nullptr)
{
p = memory_provider.template reserve<false>(large_class);
if (p == nullptr)
return nullptr;
MemoryProvider::Pal::template notify_using<zero_mem>(p, rsize);
MemoryProvider::Pal::template notify_using<zero_mem>(
p.unsafe_capptr, rsize);
}
else
{
@@ -311,7 +322,8 @@ namespace snmalloc
// Cross-reference alloc.h's large_dealloc decommitment condition.
bool decommitted =
((decommit_strategy == DecommitSuperLazy) &&
(static_cast<Baseslab*>(p)->get_kind() == Decommitted)) ||
(p.template as_static<Baseslab>().unsafe_capptr->get_kind() ==
Decommitted)) ||
(large_class > 0) || (decommit_strategy == DecommitSuper);
if (decommitted)
@@ -325,7 +337,8 @@ namespace snmalloc
// Passing zero_mem ensures the PAL provides zeroed pages if
// required.
MemoryProvider::Pal::template notify_using<zero_mem>(
pointer_offset(p, OS_PAGE_SIZE), rsize - OS_PAGE_SIZE);
pointer_offset(p.unsafe_capptr, OS_PAGE_SIZE),
rsize - OS_PAGE_SIZE);
}
else
{
@@ -338,7 +351,7 @@ namespace snmalloc
}
}
SNMALLOC_ASSERT(p == pointer_align_up(p, rsize));
SNMALLOC_ASSERT(p.as_void() == pointer_align_up(p.as_void(), rsize));
return p;
}

2
src/test/func/fixed_region/fixed_region.cc
View File

@@ -40,7 +40,7 @@ int main()
// For 1MiB superslabs, SUPERSLAB_BITS + 4 is not big enough for the example.
size_t large_class = 28 - SUPERSLAB_BITS;
size_t size = bits::one_at_bit(SUPERSLAB_BITS + large_class);
void* oe_base = mp.reserve<true>(large_class);
void* oe_base = mp.reserve<true>(large_class).unsafe_capptr;
void* oe_end = (uint8_t*)oe_base + size;
PALOpenEnclave::setup_initial_range(oe_base, oe_end);
std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;

6
src/test/func/sandbox/sandbox.cc
View File

@@ -75,7 +75,7 @@ namespace
*
* This method must be implemented for `LargeAlloc` to work.
*/
void* pop_large_stack(size_t large_class)
CapPtr<Largeslab, CBArena> pop_large_stack(size_t large_class)
{
return real_state->pop_large_stack(large_class);
};
@@ -98,7 +98,7 @@ namespace
* This method must be implemented for `LargeAlloc` to work.
*/
template<bool committed>
void* reserve(size_t large_class) noexcept
CapPtr<Largeslab, CBArena> reserve(size_t large_class) noexcept
{
return real_state->template reserve<committed>(large_class);
}
@@ -159,7 +159,7 @@ namespace
top(pointer_offset(start, sb_size)),
shared_state(new (start) SharedState()),
state(
pointer_offset(start, sizeof(SharedState)),
pointer_offset(CapPtr<void, CBArena>(start), sizeof(SharedState)),
sb_size - sizeof(SharedState)),
alloc(state, SNMALLOC_DEFAULT_CHUNKMAP(), &shared_state->queue)
{

2
src/test/func/two_alloc_types/main.cc
View File

@@ -48,7 +48,7 @@ int main()
// For 1MiB superslabs, SUPERSLAB_BITS + 2 is not big enough for the example.
size_t large_class = 26 - SUPERSLAB_BITS;
size_t size = bits::one_at_bit(SUPERSLAB_BITS + large_class);
void* oe_base = mp.reserve<true>(large_class);
void* oe_base = mp.reserve<true>(large_class).unsafe_capptr;
void* oe_end = (uint8_t*)oe_base + size;
oe_allocator_init(oe_base, oe_end);
std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;