From 3d3b048776fd576c725a06bd8c4432c50abca81b Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Tue, 12 May 2020 17:35:52 +0000 Subject: [PATCH 1/4] mediumslab: stop assuming page-alignedness of objects With large pages (e.g. the 64K that Debian defaults to for ppc64), this is a bit much to ask. It's only not true for the bottom few medium size classes, tho', as all sizes above 256K are multiples of 64K with the current two mantissa bits size schedule. --- src/mem/alloc.h | 6 ------ src/mem/mediumslab.h | 7 +++---- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index b238bc2..b568cbc 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -733,12 +733,6 @@ namespace snmalloc size_t size1 = sizeclass_to_size(sc1); size_t size2 = sizeclass_to_size(sc2); - // All medium size classes are page aligned. - if (i > NUM_SMALL_CLASSES) - { - SNMALLOC_ASSERT(is_aligned_block(nullptr, size1)); - } - SNMALLOC_ASSERT(sc1 == i); SNMALLOC_ASSERT(sc1 == sc2); SNMALLOC_ASSERT(size1 == size); diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index d7632d5..59a10f2 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -85,11 +85,10 @@ namespace snmalloc void* p = pointer_offset(this, (static_cast(index) << 8)); free--; - SNMALLOC_ASSERT(is_aligned_block(p, OS_PAGE_SIZE)); - size = bits::align_up(size, OS_PAGE_SIZE); - if constexpr (zero_mem == YesZero) - memory_provider.template zero(p, size); + memory_provider.zero(p, size); + else + UNUSED(size); return p; } From b2ee1902dcf76fa78216a266a2921b0ca5eaaf4f Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Wed, 13 May 2020 13:58:12 +0000 Subject: [PATCH 2/4] AAL: add AalName and smallest_page_size --- src/aal/aal.h | 7 +++++++ src/aal/aal_arm.h | 4 ++++ src/aal/aal_x86.h | 4 ++++ src/aal/aal_x86_sgx.h | 4 ++++ 4 files changed, 19 insertions(+) diff --git a/src/aal/aal.h b/src/aal/aal.h index b632fbd..bf22af8 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -44,6 +44,13 @@ namespace snmalloc StrictProvenance = (1 << 2), }; + enum AalName : int + { + ARM, + X86, + X86_SGX, + }; + /** * Architecture Abstraction Layer. Includes default implementations of some * functions using compiler builtins. Falls back to the definitions in the diff --git a/src/aal/aal_arm.h b/src/aal/aal_arm.h index baa690f..bd5f08b 100644 --- a/src/aal/aal_arm.h +++ b/src/aal/aal_arm.h @@ -27,6 +27,10 @@ namespace snmalloc static constexpr uint64_t aal_features = IntegerPointers | NoCpuCycleCounters; + static constexpr enum AalName aal_name = ARM; + + static constexpr size_t smallest_page_size = 0x1000; + /** * On pipelined processors, notify the core that we are in a spin loop and * that speculative execution past this point may not be a performance gain. diff --git a/src/aal/aal_x86.h b/src/aal/aal_x86.h index 2473b5f..cc20e77 100644 --- a/src/aal/aal_x86.h +++ b/src/aal/aal_x86.h @@ -60,6 +60,10 @@ namespace snmalloc */ static constexpr uint64_t aal_features = IntegerPointers; + static constexpr enum AalName aal_name = X86; + + static constexpr size_t smallest_page_size = 0x1000; + /** * On pipelined processors, notify the core that we are in a spin loop and * that speculative execution past this point may not be a performance gain. diff --git a/src/aal/aal_x86_sgx.h b/src/aal/aal_x86_sgx.h index 3d9a467..87808a4 100644 --- a/src/aal/aal_x86_sgx.h +++ b/src/aal/aal_x86_sgx.h @@ -26,6 +26,10 @@ namespace snmalloc */ static constexpr uint64_t aal_features = IntegerPointers; + static constexpr enum AalName aal_name = X86_SGX; + + static constexpr size_t smallest_page_size = 0x1000; + /** * On pipelined processors, notify the core that we are in a spin loop and * that speculative execution past this point may not be a performance gain. From 44e9abe8888ba82b596d74f82a4861c20a70c47e Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Tue, 12 May 2020 03:21:31 +0000 Subject: [PATCH 3/4] Move OS_PAGE_SIZE to PAL --- src/mem/allocconfig.h | 11 ----------- src/mem/sizeclass.h | 2 +- src/pal/pal.h | 21 +++++++++++++++++++++ src/pal/pal_apple.h | 4 ++-- src/pal/pal_bsd.h | 2 +- src/pal/pal_freebsd_kernel.h | 1 - src/pal/pal_linux.h | 9 +++++---- src/pal/pal_open_enclave.h | 3 +++ src/pal/pal_plain.h | 1 - src/pal/pal_posix.h | 11 ++++++----- src/pal/pal_windows.h | 11 ++++++----- 11 files changed, 45 insertions(+), 31 deletions(-) diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 2e9bb6d..9704b94 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -92,18 +92,7 @@ namespace snmalloc // Used to isolate values on cache lines to prevent false sharing. static constexpr size_t CACHELINE_SIZE = 64; - // Used to keep Superslab metadata committed. - static constexpr size_t OS_PAGE_SIZE = 0x1000; static constexpr size_t PAGE_ALIGNED_SIZE = OS_PAGE_SIZE << INTERMEDIATE_BITS; - // Some system headers (e.g. Linux' sys/user.h, FreeBSD's machine/param.h) - // define `PAGE_SIZE` as a macro. We don't use `PAGE_SIZE` as our variable - // name, to avoid conflicts, but if we do see a macro definition then check - // that our value matches the platform's expected value. -#ifdef PAGE_SIZE - static_assert( - PAGE_SIZE == OS_PAGE_SIZE, - "Page size from system header does not match snmalloc config page size."); -#endif // Minimum allocation size is space for two pointers. static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*)); diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 5b200d1..e5bb6e3 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -1,6 +1,6 @@ #pragma once -#include "../pal/pal_consts.h" +#include "../pal/pal.h" #include "allocconfig.h" namespace snmalloc diff --git a/src/pal/pal.h b/src/pal/pal.h index ed5ea11..d8a106c 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -59,4 +59,25 @@ namespace snmalloc */ template constexpr static bool pal_supports = (PAL::pal_features & F) == F; + + // Used to keep Superslab metadata committed. + static constexpr size_t OS_PAGE_SIZE = Pal::page_size; + + static_assert( + bits::next_pow2_const(OS_PAGE_SIZE) == OS_PAGE_SIZE, + "OS_PAGE_SIZE must be a power of two"); + static_assert( + OS_PAGE_SIZE % Aal::smallest_page_size == 0, + "The smallest architectural page size must divide OS_PAGE_SIZE"); + + // Some system headers (e.g. Linux' sys/user.h, FreeBSD's machine/param.h) + // define `PAGE_SIZE` as a macro. We don't use `PAGE_SIZE` as our variable + // name, to avoid conflicts, but if we do see a macro definition then check + // that our value matches the platform's expected value. +#ifdef PAGE_SIZE + static_assert( + PAGE_SIZE == OS_PAGE_SIZE, + "Page size from system header does not match snmalloc config page size."); +#endif + } // namespace snmalloc diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index b27dd43..0f10ee4 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -32,9 +32,9 @@ namespace snmalloc template void zero(void* p, size_t size) { - if (page_aligned || is_aligned_block(p, size)) + if (page_aligned || is_aligned_block(p, size)) { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); void* r = mmap( p, size, diff --git a/src/pal/pal_bsd.h b/src/pal/pal_bsd.h index 2641a03..c446dcb 100644 --- a/src/pal/pal_bsd.h +++ b/src/pal/pal_bsd.h @@ -33,7 +33,7 @@ namespace snmalloc */ void notify_not_using(void* p, size_t size) noexcept { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); madvise(p, size, MADV_FREE); } }; diff --git a/src/pal/pal_freebsd_kernel.h b/src/pal/pal_freebsd_kernel.h index 6327942..84f4841 100644 --- a/src/pal/pal_freebsd_kernel.h +++ b/src/pal/pal_freebsd_kernel.h @@ -1,7 +1,6 @@ #pragma once #include "../ds/bits.h" -#include "../mem/allocconfig.h" #if defined(FreeBSD_KERNEL) extern "C" diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index d4c20a7..3ed63a3 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -2,7 +2,6 @@ #if defined(__linux__) # include "../ds/bits.h" -# include "../mem/allocconfig.h" # include "pal_posix.h" # include @@ -26,6 +25,8 @@ namespace snmalloc */ static constexpr uint64_t pal_features = PALPOSIX::pal_features; + static constexpr size_t page_size = 0x1000; + /** * OS specific function for zeroing memory. * @@ -41,12 +42,12 @@ namespace snmalloc // MADV_DONTNEED. switch back to memset only for QEMU. # ifndef SNMALLOC_QEMU_WORKAROUND if ( - (page_aligned || is_aligned_block(p, size)) && - (size > SLAB_SIZE)) + (page_aligned || is_aligned_block(p, size)) && + (size > 16 * page_size)) { // Only use this on large allocations as memset faster, and doesn't // introduce IPI so faster for small allocations. - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); madvise(p, size, MADV_DONTNEED); } else diff --git a/src/pal/pal_open_enclave.h b/src/pal/pal_open_enclave.h index 25983a4..77101db 100644 --- a/src/pal/pal_open_enclave.h +++ b/src/pal/pal_open_enclave.h @@ -19,6 +19,9 @@ namespace snmalloc * PAL supports. */ static constexpr uint64_t pal_features = 0; + + static constexpr size_t page_size = 0x1000; + [[noreturn]] static void error(const char* const str) { UNUSED(str); diff --git a/src/pal/pal_plain.h b/src/pal/pal_plain.h index b55edb4..a7cd199 100644 --- a/src/pal/pal_plain.h +++ b/src/pal/pal_plain.h @@ -1,7 +1,6 @@ #pragma once #include "../ds/bits.h" -#include "../mem/allocconfig.h" namespace snmalloc { diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index e1bef12..12a85a9 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -1,7 +1,6 @@ #pragma once #include "../ds/address.h" -#include "../mem/allocconfig.h" #if defined(BACKTRACE_HEADER) # include BACKTRACE_HEADER #endif @@ -39,6 +38,8 @@ namespace snmalloc */ static constexpr uint64_t pal_features = LazyCommit; + static constexpr size_t page_size = 0x1000; + static void print_stack_trace() { #ifdef BACKTRACE_HEADER @@ -72,7 +73,7 @@ namespace snmalloc */ void notify_not_using(void* p, size_t size) noexcept { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); #ifdef USE_POSIX_COMMIT_CHECKS mprotect(p, size, PROT_NONE); #else @@ -92,7 +93,7 @@ namespace snmalloc void notify_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT( - is_aligned_block(p, size) || (zero_mem == NoZero)); + is_aligned_block(p, size) || (zero_mem == NoZero)); #ifdef USE_POSIX_COMMIT_CHECKS mprotect(p, size, PROT_READ | PROT_WRITE); @@ -119,9 +120,9 @@ namespace snmalloc template void zero(void* p, size_t size) noexcept { - if (page_aligned || is_aligned_block(p, size)) + if (page_aligned || is_aligned_block(p, size)) { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); void* r = mmap( p, size, diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index e325d3a..f8a5325 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -2,7 +2,6 @@ #include "../ds/address.h" #include "../ds/bits.h" -#include "../mem/allocconfig.h" #ifdef _WIN32 # ifndef _MSC_VER @@ -83,6 +82,8 @@ namespace snmalloc # endif ; + static constexpr size_t page_size = 0x1000; + /** * Check whether the low memory state is still in effect. This is an * expensive operation and should not be on any fast paths. @@ -115,7 +116,7 @@ namespace snmalloc /// Notify platform that we will not be using these pages void notify_not_using(void* p, size_t size) noexcept { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); BOOL ok = VirtualFree(p, size, MEM_DECOMMIT); @@ -128,7 +129,7 @@ namespace snmalloc void notify_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT( - is_aligned_block(p, size) || (zero_mem == NoZero)); + is_aligned_block(p, size) || (zero_mem == NoZero)); void* r = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE); @@ -140,9 +141,9 @@ namespace snmalloc template void zero(void* p, size_t size) noexcept { - if (page_aligned || is_aligned_block(p, size)) + if (page_aligned || is_aligned_block(p, size)) { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); notify_not_using(p, size); notify_using(p, size); } From 37766588deca5f2f515430fd0b099ecbd517b24a Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Tue, 12 May 2020 17:46:33 +0000 Subject: [PATCH 4/4] Linux PowerPC port --- src/aal/aal.h | 7 +++++++ src/aal/aal_powerpc.h | 37 +++++++++++++++++++++++++++++++++++++ src/pal/pal_linux.h | 3 ++- 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 src/aal/aal_powerpc.h diff --git a/src/aal/aal.h b/src/aal/aal.h index bf22af8..ce5e6c1 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -19,6 +19,10 @@ # define PLATFORM_IS_ARM #endif +#if defined(__powerpc__) || defined(__powerpc64__) +# define PLATFORM_IS_POWERPC +#endif + namespace snmalloc { /** @@ -47,6 +51,7 @@ namespace snmalloc enum AalName : int { ARM, + PowerPC, X86, X86_SGX, }; @@ -140,6 +145,8 @@ namespace snmalloc # include "aal_x86_sgx.h" #elif defined(PLATFORM_IS_ARM) # include "aal_arm.h" +#elif defined(PLATFORM_IS_POWERPC) +# include "aal_powerpc.h" #endif namespace snmalloc diff --git a/src/aal/aal_powerpc.h b/src/aal/aal_powerpc.h new file mode 100644 index 0000000..60b9f12 --- /dev/null +++ b/src/aal/aal_powerpc.h @@ -0,0 +1,37 @@ +#pragma once + +#if defined(__powerpc64__) +# define SNMALLOC_VA_BITS_64 +#else +# define SNMALLOC_VA_BITS_32 +#endif + +namespace snmalloc +{ + /** + * ARM-specific architecture abstraction layer. + */ + class AAL_PowerPC + { + public: + /** + * Bitmap of AalFeature flags + */ + static constexpr uint64_t aal_features = IntegerPointers; + + static constexpr enum AalName aal_name = PowerPC; + + static constexpr size_t smallest_page_size = 0x1000; + + /** + * On pipelined processors, notify the core that we are in a spin loop and + * that speculative execution past this point may not be a performance gain. + */ + static inline void pause() + { + __asm__ volatile("or 27,27,27"); // "yield" + } + }; + + using AAL_Arch = AAL_PowerPC; +} // namespace snmalloc diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index 3ed63a3..a645c16 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -25,7 +25,8 @@ namespace snmalloc */ static constexpr uint64_t pal_features = PALPOSIX::pal_features; - static constexpr size_t page_size = 0x1000; + static constexpr size_t page_size = + Aal::aal_name == PowerPC ? 0x10000 : 0x1000; /** * OS specific function for zeroing memory.