From d9ee19a16cf93a2de31c0c46f13a4d709154e128 Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Tue, 23 Mar 2021 14:56:24 +0000 Subject: [PATCH] SP: use CapPtr<>s in address_space, largealloc --- src/mem/address_space.h | 49 ++++++++++-------- src/mem/alloc.h | 17 ++++--- src/mem/chunkmap.h | 2 +- src/mem/largealloc.h | 59 +++++++++++++--------- src/test/func/fixed_region/fixed_region.cc | 2 +- src/test/func/sandbox/sandbox.cc | 6 +-- src/test/func/two_alloc_types/main.cc | 2 +- 7 files changed, 78 insertions(+), 59 deletions(-) diff --git a/src/mem/address_space.h b/src/mem/address_space.h index b490e22..1ef4e40 100644 --- a/src/mem/address_space.h +++ b/src/mem/address_space.h @@ -34,7 +34,7 @@ namespace snmalloc * bits::BITS is used for simplicity, we do not use below the pointer size, * and large entries will be unlikely to be supported by the platform. */ - std::array, bits::BITS> ranges = {}; + std::array, 2>, bits::BITS> ranges = {}; /** * This is infrequently used code, a spin lock simplifies the code @@ -45,7 +45,7 @@ namespace snmalloc /** * Checks a block satisfies its invariant. */ - inline void check_block(void* base, size_t align_bits) + inline void check_block(CapPtr base, size_t align_bits) { SNMALLOC_ASSERT( base == pointer_align_up(base, bits::one_at_bit(align_bits))); @@ -58,7 +58,7 @@ namespace snmalloc /** * Adds a block to `ranges`. */ - void add_block(size_t align_bits, void* base) + void add_block(size_t align_bits, CapPtr base) { check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); @@ -73,7 +73,8 @@ namespace snmalloc { // Add to linked list. commit_block(base, sizeof(void*)); - *reinterpret_cast(base) = ranges[align_bits][1]; + *(base.template as_static>().unsafe_capptr) = + ranges[align_bits][1]; check_block(ranges[align_bits][1], align_bits); } @@ -86,9 +87,9 @@ namespace snmalloc * Find a block of the correct size. May split larger blocks * to satisfy this request. */ - void* remove_block(size_t align_bits) + CapPtr remove_block(size_t align_bits) { - auto first = ranges[align_bits][0]; + CapPtr first = ranges[align_bits][0]; if (first == nullptr) { if (align_bits == (bits::BITS - 1)) @@ -98,7 +99,7 @@ namespace snmalloc } // Look for larger block and split up recursively - void* bigger = remove_block(align_bits + 1); + CapPtr bigger = remove_block(align_bits + 1); if (bigger != nullptr) { auto left_over = pointer_offset(bigger, bits::one_at_bit(align_bits)); @@ -109,14 +110,16 @@ namespace snmalloc return bigger; } - auto second = ranges[align_bits][1]; + CapPtr second = ranges[align_bits][1]; if (second != nullptr) { commit_block(second, sizeof(void*)); - auto next = *reinterpret_cast(second); + auto psecond = + second.template as_static>().unsafe_capptr; + auto next = *psecond; ranges[align_bits][1] = next; // Zero memory. Client assumes memory contains only zeros. - *reinterpret_cast(second) = nullptr; + *psecond = nullptr; check_block(second, align_bits); check_block(next, align_bits); return second; @@ -131,7 +134,7 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - void add_range(void* base, size_t length) + void add_range(CapPtr base, size_t length) { // Find the minimum set of maximally aligned blocks in this range. // Each block's alignment and size are equal. @@ -153,14 +156,14 @@ namespace snmalloc /** * Commit a block of memory */ - void commit_block(void* base, size_t size) + void commit_block(CapPtr base, size_t size) { // Rounding required for sub-page allocations. auto page_start = pointer_align_down(base); auto page_end = pointer_align_up(pointer_offset(base, size)); - PAL::template notify_using( - page_start, static_cast(page_end - page_start)); + size_t using_size = pointer_diff(page_start, page_end); + PAL::template notify_using(page_start.unsafe_capptr, using_size); } public: @@ -172,7 +175,7 @@ namespace snmalloc * Only request 2^n sizes, and not less than a pointer. */ template - void* reserve(size_t size) + CapPtr reserve(size_t size) { SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -180,22 +183,24 @@ namespace snmalloc if constexpr (pal_supports) { if (size >= PAL::minimum_alloc_size) - return PAL::template reserve_aligned(size); + return CapPtr( + PAL::template reserve_aligned(size)); } - void* res; + CapPtr res; { FlagLock lock(spin_lock); res = remove_block(bits::next_pow2_bits(size)); if (res == nullptr) { // Allocation failed ask OS for more memory - void* block = nullptr; + CapPtr block = nullptr; size_t block_size = 0; if constexpr (pal_supports) { block_size = PAL::minimum_alloc_size; - block = PAL::template reserve_aligned(block_size); + block = CapPtr( + PAL::template reserve_aligned(block_size)); } else if constexpr (!pal_supports) { @@ -204,7 +209,7 @@ namespace snmalloc // the PAL, and this could lead to suprious OOM. This is // particularly bad if the PAL gives all the memory on first call. auto block_and_size = PAL::reserve_at_least(size * 2); - block = block_and_size.first; + block = CapPtr(block_and_size.first); block_size = block_and_size.second; // Ensure block is pointer aligned. @@ -244,7 +249,7 @@ namespace snmalloc * used, by smaller objects. */ template - void* reserve_with_left_over(size_t size) + CapPtr reserve_with_left_over(size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -279,7 +284,7 @@ namespace snmalloc * Constructor that pre-initialises the address-space manager with a region * of memory. */ - AddressSpaceManager(void* base, size_t length) + AddressSpaceManager(CapPtr base, size_t length) { add_range(base, length); } diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 50aed78..8eeb19a 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -955,9 +955,10 @@ namespace snmalloc if (super != nullptr) return super; - super = - reinterpret_cast(large_allocator.template alloc( - 0, SUPERSLAB_SIZE, SUPERSLAB_SIZE)); + super = large_allocator + .template alloc(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE) + .template as_reinterpret() + .unsafe_capptr; if (super == nullptr) return super; @@ -1371,9 +1372,9 @@ namespace snmalloc sizeclass, rsize, size); }); } - slab = CapPtr(large_allocator.template alloc( - 0, SUPERSLAB_SIZE, SUPERSLAB_SIZE)) - .template as_static(); + slab = large_allocator + .template alloc(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE) + .template as_reinterpret(); if (slab == nullptr) return nullptr; @@ -1503,7 +1504,7 @@ namespace snmalloc if (large_class == 0) size = rsize; - void* p = + auto p = large_allocator.template alloc(large_class, rsize, size); if (likely(p != nullptr)) { @@ -1512,7 +1513,7 @@ namespace snmalloc stats().alloc_request(size); stats().large_alloc(large_class); } - return p; + return p.unsafe_capptr; } void large_dealloc_unchecked( diff --git a/src/mem/chunkmap.h b/src/mem/chunkmap.h index c8e462b..4c037ff 100644 --- a/src/mem/chunkmap.h +++ b/src/mem/chunkmap.h @@ -149,7 +149,7 @@ namespace snmalloc * Update the pagemap to reflect a large allocation, of `size` bytes from * address `p`. */ - static void set_large_size(void* p, size_t size) + static void set_large_size(CapPtr p, size_t size) { size_t size_bits = bits::next_pow2_bits(size); set(address_cast(p), static_cast(size_bits)); diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 4bec104..edf44ed 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -32,7 +32,7 @@ namespace snmalloc friend class MPMCStack; template friend class MemoryProviderStateMixin; - std::atomic next; + AtomicCapPtr next = nullptr; public: void init() @@ -89,7 +89,10 @@ namespace snmalloc /** * Stack of large allocations that have been returned for reuse. */ - ModArray> large_stack; + ModArray< + NUM_LARGE_CLASSES, + MPMCStack> + large_stack; public: using Pal = PAL; @@ -99,9 +102,10 @@ namespace snmalloc * concurrently with other acceses. If there is no large allocation on a * particular stack then this will return `nullptr`. */ - SNMALLOC_FAST_PATH void* pop_large_stack(size_t large_class) + SNMALLOC_FAST_PATH CapPtr + pop_large_stack(size_t large_class) { - void* p = large_stack[large_class].pop(); + auto p = large_stack[large_class].pop(); if (p != nullptr) { const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; @@ -119,7 +123,7 @@ namespace snmalloc { const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; available_large_chunks_in_bytes += rsize; - large_stack[large_class].push(slab.unsafe_capptr); + large_stack[large_class].push(slab); } /** @@ -133,7 +137,7 @@ namespace snmalloc * memory providers constructed in this way does not have to be able to * allocate memory, if the initial reservation is sufficient. */ - MemoryProviderStateMixin(void* start, size_t len) + MemoryProviderStateMixin(CapPtr start, size_t len) : address_space(start, len) {} /** @@ -146,9 +150,11 @@ namespace snmalloc // Allocate permanent storage for the allocator usung temporary allocator MemoryProviderStateMixin* allocated = - reinterpret_cast*>( - local.template reserve_with_left_over( - sizeof(MemoryProviderStateMixin))); + local + .template reserve_with_left_over( + sizeof(MemoryProviderStateMixin)) + .template as_static>() + .unsafe_capptr; if (allocated == nullptr) error("Failed to initialise system!"); @@ -193,22 +199,23 @@ namespace snmalloc size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; size_t decommit_size = rsize - OS_PAGE_SIZE; // Grab all of the chunks of this size class. - auto* slab = large_stack[large_class].pop_all(); - while (slab) + CapPtr slab = large_stack[large_class].pop_all(); + while (slab != nullptr) { // Decommit all except for the first page and then put it back on // the stack. if (slab->get_kind() != Decommitted) { PAL::notify_not_using( - pointer_offset(slab, OS_PAGE_SIZE), decommit_size); + pointer_offset(slab.unsafe_capptr, OS_PAGE_SIZE), decommit_size); } // Once we've removed these from the stack, there will be no // concurrent accesses and removal should have established a // happens-before relationship, so it's safe to use relaxed loads // here. auto next = slab->next.load(std::memory_order_relaxed); - large_stack[large_class].push(new (slab) Decommittedslab()); + large_stack[large_class].push(CapPtr( + new (slab.unsafe_capptr) Decommittedslab())); slab = next; } } @@ -247,21 +254,22 @@ namespace snmalloc // Cache line align size_t size = bits::align_up(sizeof(T), 64); size = bits::max(size, alignment); - void* p = address_space.template reserve_with_left_over(size); + auto p = address_space.template reserve_with_left_over(size); if (p == nullptr) return nullptr; peak_memory_used_bytes += size; - return new (p) T(std::forward(args)...); + return new (p.unsafe_capptr) T(std::forward(args)...); } template - void* reserve(size_t large_class) noexcept + CapPtr reserve(size_t large_class) noexcept { size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class; peak_memory_used_bytes += size; - return address_space.template reserve(size); + return address_space.template reserve(size) + .template as_static(); } /** @@ -290,19 +298,22 @@ namespace snmalloc LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {} template - void* alloc(size_t large_class, size_t rsize, size_t size) + CapPtr + alloc(size_t large_class, size_t rsize, size_t size) { SNMALLOC_ASSERT( (bits::one_at_bit(SUPERSLAB_BITS) << large_class) == rsize); - void* p = memory_provider.pop_large_stack(large_class); + CapPtr p = + memory_provider.pop_large_stack(large_class); if (p == nullptr) { p = memory_provider.template reserve(large_class); if (p == nullptr) return nullptr; - MemoryProvider::Pal::template notify_using(p, rsize); + MemoryProvider::Pal::template notify_using( + p.unsafe_capptr, rsize); } else { @@ -311,7 +322,8 @@ namespace snmalloc // Cross-reference alloc.h's large_dealloc decommitment condition. bool decommitted = ((decommit_strategy == DecommitSuperLazy) && - (static_cast(p)->get_kind() == Decommitted)) || + (p.template as_static().unsafe_capptr->get_kind() == + Decommitted)) || (large_class > 0) || (decommit_strategy == DecommitSuper); if (decommitted) @@ -325,7 +337,8 @@ namespace snmalloc // Passing zero_mem ensures the PAL provides zeroed pages if // required. MemoryProvider::Pal::template notify_using( - pointer_offset(p, OS_PAGE_SIZE), rsize - OS_PAGE_SIZE); + pointer_offset(p.unsafe_capptr, OS_PAGE_SIZE), + rsize - OS_PAGE_SIZE); } else { @@ -338,7 +351,7 @@ namespace snmalloc } } - SNMALLOC_ASSERT(p == pointer_align_up(p, rsize)); + SNMALLOC_ASSERT(p.as_void() == pointer_align_up(p.as_void(), rsize)); return p; } diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 09d68ee..06d138a 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -40,7 +40,7 @@ int main() // For 1MiB superslabs, SUPERSLAB_BITS + 4 is not big enough for the example. size_t large_class = 28 - SUPERSLAB_BITS; size_t size = bits::one_at_bit(SUPERSLAB_BITS + large_class); - void* oe_base = mp.reserve(large_class); + void* oe_base = mp.reserve(large_class).unsafe_capptr; void* oe_end = (uint8_t*)oe_base + size; PALOpenEnclave::setup_initial_range(oe_base, oe_end); std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl; diff --git a/src/test/func/sandbox/sandbox.cc b/src/test/func/sandbox/sandbox.cc index 99da8f9..051e288 100644 --- a/src/test/func/sandbox/sandbox.cc +++ b/src/test/func/sandbox/sandbox.cc @@ -75,7 +75,7 @@ namespace * * This method must be implemented for `LargeAlloc` to work. */ - void* pop_large_stack(size_t large_class) + CapPtr pop_large_stack(size_t large_class) { return real_state->pop_large_stack(large_class); }; @@ -98,7 +98,7 @@ namespace * This method must be implemented for `LargeAlloc` to work. */ template - void* reserve(size_t large_class) noexcept + CapPtr reserve(size_t large_class) noexcept { return real_state->template reserve(large_class); } @@ -159,7 +159,7 @@ namespace top(pointer_offset(start, sb_size)), shared_state(new (start) SharedState()), state( - pointer_offset(start, sizeof(SharedState)), + pointer_offset(CapPtr(start), sizeof(SharedState)), sb_size - sizeof(SharedState)), alloc(state, SNMALLOC_DEFAULT_CHUNKMAP(), &shared_state->queue) { diff --git a/src/test/func/two_alloc_types/main.cc b/src/test/func/two_alloc_types/main.cc index 82a0328..946fcd2 100644 --- a/src/test/func/two_alloc_types/main.cc +++ b/src/test/func/two_alloc_types/main.cc @@ -48,7 +48,7 @@ int main() // For 1MiB superslabs, SUPERSLAB_BITS + 2 is not big enough for the example. size_t large_class = 26 - SUPERSLAB_BITS; size_t size = bits::one_at_bit(SUPERSLAB_BITS + large_class); - void* oe_base = mp.reserve(large_class); + void* oe_base = mp.reserve(large_class).unsafe_capptr; void* oe_end = (uint8_t*)oe_base + size; oe_allocator_init(oe_base, oe_end); std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;