From 6a5359b1777cb8d7b3dbd6a3d54ab31ad9d23b59 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Fri, 22 Feb 2019 11:45:19 +0000 Subject: [PATCH] Pull aligned allocation out of the PAL. The PAL can now advertise that it supports aligned allocation. If it does not, then the memory provider will do the alignment for it. This change still leaves the PAL responsible for systematic testing, but it should now be much easier to lift that out. --- src/mem/largealloc.h | 36 +++++++++- src/pal/pal.h | 12 +++- src/pal/pal_free_bsd_kernel.h | 2 +- src/pal/pal_freebsd.h | 2 +- src/pal/pal_linux.h | 21 +----- src/pal/pal_open_enclave.h | 2 +- src/pal/pal_windows.h | 76 ++++++++++++++-------- src/test/func/fixed_region/fixed_region.cc | 6 +- src/test/func/two_alloc_types/main.cc | 4 +- 9 files changed, 105 insertions(+), 56 deletions(-) diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 3bc2359..e06ec5c 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -54,7 +54,7 @@ namespace snmalloc std::pair reserve_block() noexcept { size_t size = SUPERSLAB_SIZE; - void* r = ((PAL*)this)->template reserve(&size, SUPERSLAB_SIZE); + void* r = reserve(&size, SUPERSLAB_SIZE); if (size < SUPERSLAB_SIZE) error("out of memory"); @@ -197,6 +197,40 @@ namespace snmalloc } } + template + void* reserve(size_t* size, size_t align) noexcept + { + if constexpr (pal_supports()) + { + return PAL::template reserve(size, align); + } + else + { + size_t request = *size; + // Add align, so we can guarantee to provide at least size. + request += align; + // Alignment must be a power of 2. + assert(align == bits::next_pow2(align)); + + void* p = PAL::template reserve(&request); + + *size = request; + uintptr_t p0 = (uintptr_t)p; + uintptr_t start = bits::align_up(p0, align); + + if (start > (uintptr_t)p0) + { + uintptr_t end = bits::align_down(p0 + request, align); + *size = end - start; + PAL::notify_not_using(p, start - p0); + PAL::notify_not_using( + reinterpret_cast(end), (p0 + request) - end); + p = reinterpret_cast(start); + } + return p; + } + } + ALWAYSINLINE void lazy_decommit_if_needed() { #ifdef TEST_LAZY_DECOMMIT diff --git a/src/pal/pal.h b/src/pal/pal.h index 3f58f16..e917025 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -17,7 +17,17 @@ namespace snmalloc * `expensive_low_memory_check()` method that returns a `bool` indicating * whether low memory conditions are still in effect. */ - LowMemoryNotification = (1 << 0) + LowMemoryNotification = (1 << 0), + /** + * This PAL natively supports allocation with a guaranteed alignment. If + * this is not supported, then we will over-allocate and round the + * allocation. + * + * A PAL that does supports this must expose a `request()` method that takes + * a size and alignment. A PAL that does *not* support it must expose a + * `request()` method that takes only a size. + */ + AlignedAllocation = (1 << 1) }; } diff --git a/src/pal/pal_free_bsd_kernel.h b/src/pal/pal_free_bsd_kernel.h index 9a3b991..3af8d14 100644 --- a/src/pal/pal_free_bsd_kernel.h +++ b/src/pal/pal_free_bsd_kernel.h @@ -28,7 +28,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = 0; + static constexpr uint64_t pal_features = AlignedAllocation; void error(const char* const str) { panic("snmalloc error: %s", str); diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index f5c843c..122b8b9 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -18,7 +18,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = 0; + static constexpr uint64_t pal_features = AlignedAllocation; static void error(const char* const str) { puts(str); diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index ad86ec7..f1e2d01 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -61,17 +61,11 @@ namespace snmalloc } template - void* reserve(size_t* size, size_t align) noexcept + void* reserve(size_t* size) noexcept { - size_t request = *size; - // Add align, so we can guarantee to provide at least size. - request += align; - // Alignment must be a power of 2. - assert(align == bits::next_pow2(align)); - void* p = mmap( NULL, - request, + *size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, @@ -79,18 +73,7 @@ namespace snmalloc if (p == MAP_FAILED) error("Out of memory"); - *size = request; - uintptr_t p0 = (uintptr_t)p; - uintptr_t start = bits::align_up(p0, align); - if (start > (uintptr_t)p0) - { - uintptr_t end = bits::align_down(p0 + request, align); - *size = end - start; - munmap(p, start - p0); - munmap((void*)end, (p0 + request) - end); - p = (void*)start; - } return p; } }; diff --git a/src/pal/pal_open_enclave.h b/src/pal/pal_open_enclave.h index 3f8d9c6..7e1b559 100644 --- a/src/pal/pal_open_enclave.h +++ b/src/pal/pal_open_enclave.h @@ -18,7 +18,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = 0; + static constexpr uint64_t pal_features = AlignedAllocation; static void error(const char* const str) { UNUSED(str); diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 420f421..0fafea4 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -68,7 +68,11 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. This PAL supports low-memory notifications. */ - static constexpr uint64_t pal_features = LowMemoryNotification; + static constexpr uint64_t pal_features = LowMemoryNotification +# if defined(PLATFORM_HAS_VIRTUALALLOC2) || defined(USE_SYSTEMATIC_TESTING) + | AlignedAllocation +# endif + ; /** * Counter values for the number of times that a low-pressure notification * has been delivered. Callers should compare this with a previous value @@ -139,8 +143,6 @@ namespace snmalloc static size_t bump_ptr = (size_t)0x4000'0000'0000; return bump_ptr; } -# endif - template void* reserve(size_t* size, size_t align) noexcept { @@ -149,26 +151,6 @@ namespace snmalloc if (committed) flags |= MEM_COMMIT; -# ifdef PLATFORM_HAS_VIRTUALALLOC2 - // If we're on Windows 10 or newer, we can use the VirtualAlloc2 - // function. The FromApp variant is useable by UWP applications and - // cannot allocate executable memory. - MEM_ADDRESS_REQUIREMENTS addressReqs = {0}; - MEM_EXTENDED_PARAMETER param = {0}; - addressReqs.Alignment = align; - param.Type = MemExtendedParameterAddressRequirements; - param.Pointer = &addressReqs; - return VirtualAlloc2FromApp( - nullptr, nullptr, *size, flags, PAGE_READWRITE, ¶m, 1); -# else - // Add align, so we can guarantee to provide at least size. - size_t request = *size + align; - - // Alignment must be a power of 2. - assert(align == bits::next_pow2(align)); - - void* p; -# ifdef USE_SYSTEMATIC_TESTING size_t retries = 1000; do { @@ -178,9 +160,6 @@ namespace snmalloc systematic_bump_ptr() += request; retries--; } while (p == nullptr && retries > 0); -# else - p = VirtualAlloc(nullptr, request, flags, PAGE_READWRITE); -# endif uintptr_t aligned_p = bits::align_up((size_t)p, align); @@ -193,8 +172,51 @@ namespace snmalloc } *size = request; return p; -# endif } +# elif defined(PLATFORM_HAS_VIRTUALALLOC2) + template + void* reserve(size_t* size, size_t align) noexcept + { + DWORD flags = MEM_RESERVE; + + if (committed) + flags |= MEM_COMMIT; + + // Windows doesn't let you request memory less than 64KB aligned. Most + // operating systems will simply give you something more aligned than you + // ask for, but Windows complains about invalid parameters. + const size_t min_align = 64 * 1024; + if (align < min_align) + align = min_align; + + // If we're on Windows 10 or newer, we can use the VirtualAlloc2 + // function. The FromApp variant is useable by UWP applications and + // cannot allocate executable memory. + MEM_ADDRESS_REQUIREMENTS addressReqs = {0}; + MEM_EXTENDED_PARAMETER param = {0}; + addressReqs.Alignment = align; + param.Type = MemExtendedParameterAddressRequirements; + param.Pointer = &addressReqs; + void* ret = VirtualAlloc2FromApp( + nullptr, nullptr, *size, flags, PAGE_READWRITE, ¶m, 1); + if (ret == nullptr) + { + error("Failed to allocate memory\n"); + } + return ret; + } +# else + template + void* reserve(size_t* size) noexcept + { + DWORD flags = MEM_RESERVE; + + if (committed) + flags |= MEM_COMMIT; + + return VirtualAlloc(nullptr, *size, flags, PAGE_READWRITE); + } +# endif }; HEADER_GLOBAL std::atomic PALWindows::pressure_epoch; HEADER_GLOBAL std::atomic PALWindows::registered_for_notifications; diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index b00a684..fe1ae39 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -30,10 +30,10 @@ extern "C" void oe_abort() using namespace snmalloc; int main() { - DefaultPal pal; + MemoryProviderStateMixin mp; size_t size = 1ULL << 28; - oe_base = pal.reserve(&size, 0); + oe_base = mp.reserve(&size, 0); oe_end = (uint8_t*)oe_base + size; std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl; @@ -49,4 +49,4 @@ int main() if (oe_end < r1) abort(); } -} \ No newline at end of file +} diff --git a/src/test/func/two_alloc_types/main.cc b/src/test/func/two_alloc_types/main.cc index f34ee6d..bed9e87 100644 --- a/src/test/func/two_alloc_types/main.cc +++ b/src/test/func/two_alloc_types/main.cc @@ -35,10 +35,10 @@ extern "C" void enclave_free(void*); using namespace snmalloc; int main() { - DefaultPal pal; + MemoryProviderStateMixin mp; size_t size = 1ULL << 28; - oe_base = pal.reserve(&size, 0); + oe_base = mp.reserve(&size, 1); oe_end = (uint8_t*)oe_base + size; std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;