diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 3bc2359..e06ec5c 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -54,7 +54,7 @@ namespace snmalloc std::pair reserve_block() noexcept { size_t size = SUPERSLAB_SIZE; - void* r = ((PAL*)this)->template reserve(&size, SUPERSLAB_SIZE); + void* r = reserve(&size, SUPERSLAB_SIZE); if (size < SUPERSLAB_SIZE) error("out of memory"); @@ -197,6 +197,40 @@ namespace snmalloc } } + template + void* reserve(size_t* size, size_t align) noexcept + { + if constexpr (pal_supports()) + { + return PAL::template reserve(size, align); + } + else + { + size_t request = *size; + // Add align, so we can guarantee to provide at least size. + request += align; + // Alignment must be a power of 2. + assert(align == bits::next_pow2(align)); + + void* p = PAL::template reserve(&request); + + *size = request; + uintptr_t p0 = (uintptr_t)p; + uintptr_t start = bits::align_up(p0, align); + + if (start > (uintptr_t)p0) + { + uintptr_t end = bits::align_down(p0 + request, align); + *size = end - start; + PAL::notify_not_using(p, start - p0); + PAL::notify_not_using( + reinterpret_cast(end), (p0 + request) - end); + p = reinterpret_cast(start); + } + return p; + } + } + ALWAYSINLINE void lazy_decommit_if_needed() { #ifdef TEST_LAZY_DECOMMIT diff --git a/src/pal/pal.h b/src/pal/pal.h index 3f58f16..e917025 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -17,7 +17,17 @@ namespace snmalloc * `expensive_low_memory_check()` method that returns a `bool` indicating * whether low memory conditions are still in effect. */ - LowMemoryNotification = (1 << 0) + LowMemoryNotification = (1 << 0), + /** + * This PAL natively supports allocation with a guaranteed alignment. If + * this is not supported, then we will over-allocate and round the + * allocation. + * + * A PAL that does supports this must expose a `request()` method that takes + * a size and alignment. A PAL that does *not* support it must expose a + * `request()` method that takes only a size. + */ + AlignedAllocation = (1 << 1) }; } diff --git a/src/pal/pal_free_bsd_kernel.h b/src/pal/pal_free_bsd_kernel.h index 9a3b991..3af8d14 100644 --- a/src/pal/pal_free_bsd_kernel.h +++ b/src/pal/pal_free_bsd_kernel.h @@ -28,7 +28,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = 0; + static constexpr uint64_t pal_features = AlignedAllocation; void error(const char* const str) { panic("snmalloc error: %s", str); diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index f5c843c..122b8b9 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -18,7 +18,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = 0; + static constexpr uint64_t pal_features = AlignedAllocation; static void error(const char* const str) { puts(str); diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index ad86ec7..f1e2d01 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -61,17 +61,11 @@ namespace snmalloc } template - void* reserve(size_t* size, size_t align) noexcept + void* reserve(size_t* size) noexcept { - size_t request = *size; - // Add align, so we can guarantee to provide at least size. - request += align; - // Alignment must be a power of 2. - assert(align == bits::next_pow2(align)); - void* p = mmap( NULL, - request, + *size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, @@ -79,18 +73,7 @@ namespace snmalloc if (p == MAP_FAILED) error("Out of memory"); - *size = request; - uintptr_t p0 = (uintptr_t)p; - uintptr_t start = bits::align_up(p0, align); - if (start > (uintptr_t)p0) - { - uintptr_t end = bits::align_down(p0 + request, align); - *size = end - start; - munmap(p, start - p0); - munmap((void*)end, (p0 + request) - end); - p = (void*)start; - } return p; } }; diff --git a/src/pal/pal_open_enclave.h b/src/pal/pal_open_enclave.h index 3f8d9c6..7e1b559 100644 --- a/src/pal/pal_open_enclave.h +++ b/src/pal/pal_open_enclave.h @@ -18,7 +18,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = 0; + static constexpr uint64_t pal_features = AlignedAllocation; static void error(const char* const str) { UNUSED(str); diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 420f421..0fafea4 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -68,7 +68,11 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. This PAL supports low-memory notifications. */ - static constexpr uint64_t pal_features = LowMemoryNotification; + static constexpr uint64_t pal_features = LowMemoryNotification +# if defined(PLATFORM_HAS_VIRTUALALLOC2) || defined(USE_SYSTEMATIC_TESTING) + | AlignedAllocation +# endif + ; /** * Counter values for the number of times that a low-pressure notification * has been delivered. Callers should compare this with a previous value @@ -139,8 +143,6 @@ namespace snmalloc static size_t bump_ptr = (size_t)0x4000'0000'0000; return bump_ptr; } -# endif - template void* reserve(size_t* size, size_t align) noexcept { @@ -149,26 +151,6 @@ namespace snmalloc if (committed) flags |= MEM_COMMIT; -# ifdef PLATFORM_HAS_VIRTUALALLOC2 - // If we're on Windows 10 or newer, we can use the VirtualAlloc2 - // function. The FromApp variant is useable by UWP applications and - // cannot allocate executable memory. - MEM_ADDRESS_REQUIREMENTS addressReqs = {0}; - MEM_EXTENDED_PARAMETER param = {0}; - addressReqs.Alignment = align; - param.Type = MemExtendedParameterAddressRequirements; - param.Pointer = &addressReqs; - return VirtualAlloc2FromApp( - nullptr, nullptr, *size, flags, PAGE_READWRITE, ¶m, 1); -# else - // Add align, so we can guarantee to provide at least size. - size_t request = *size + align; - - // Alignment must be a power of 2. - assert(align == bits::next_pow2(align)); - - void* p; -# ifdef USE_SYSTEMATIC_TESTING size_t retries = 1000; do { @@ -178,9 +160,6 @@ namespace snmalloc systematic_bump_ptr() += request; retries--; } while (p == nullptr && retries > 0); -# else - p = VirtualAlloc(nullptr, request, flags, PAGE_READWRITE); -# endif uintptr_t aligned_p = bits::align_up((size_t)p, align); @@ -193,8 +172,51 @@ namespace snmalloc } *size = request; return p; -# endif } +# elif defined(PLATFORM_HAS_VIRTUALALLOC2) + template + void* reserve(size_t* size, size_t align) noexcept + { + DWORD flags = MEM_RESERVE; + + if (committed) + flags |= MEM_COMMIT; + + // Windows doesn't let you request memory less than 64KB aligned. Most + // operating systems will simply give you something more aligned than you + // ask for, but Windows complains about invalid parameters. + const size_t min_align = 64 * 1024; + if (align < min_align) + align = min_align; + + // If we're on Windows 10 or newer, we can use the VirtualAlloc2 + // function. The FromApp variant is useable by UWP applications and + // cannot allocate executable memory. + MEM_ADDRESS_REQUIREMENTS addressReqs = {0}; + MEM_EXTENDED_PARAMETER param = {0}; + addressReqs.Alignment = align; + param.Type = MemExtendedParameterAddressRequirements; + param.Pointer = &addressReqs; + void* ret = VirtualAlloc2FromApp( + nullptr, nullptr, *size, flags, PAGE_READWRITE, ¶m, 1); + if (ret == nullptr) + { + error("Failed to allocate memory\n"); + } + return ret; + } +# else + template + void* reserve(size_t* size) noexcept + { + DWORD flags = MEM_RESERVE; + + if (committed) + flags |= MEM_COMMIT; + + return VirtualAlloc(nullptr, *size, flags, PAGE_READWRITE); + } +# endif }; HEADER_GLOBAL std::atomic PALWindows::pressure_epoch; HEADER_GLOBAL std::atomic PALWindows::registered_for_notifications; diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index b00a684..fe1ae39 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -30,10 +30,10 @@ extern "C" void oe_abort() using namespace snmalloc; int main() { - DefaultPal pal; + MemoryProviderStateMixin mp; size_t size = 1ULL << 28; - oe_base = pal.reserve(&size, 0); + oe_base = mp.reserve(&size, 0); oe_end = (uint8_t*)oe_base + size; std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl; @@ -49,4 +49,4 @@ int main() if (oe_end < r1) abort(); } -} \ No newline at end of file +} diff --git a/src/test/func/two_alloc_types/main.cc b/src/test/func/two_alloc_types/main.cc index f34ee6d..bed9e87 100644 --- a/src/test/func/two_alloc_types/main.cc +++ b/src/test/func/two_alloc_types/main.cc @@ -35,10 +35,10 @@ extern "C" void enclave_free(void*); using namespace snmalloc; int main() { - DefaultPal pal; + MemoryProviderStateMixin mp; size_t size = 1ULL << 28; - oe_base = pal.reserve(&size, 0); + oe_base = mp.reserve(&size, 1); oe_end = (uint8_t*)oe_base + size; std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;