diff --git a/src/aal/aal.h b/src/aal/aal.h index b632fbd..ce5e6c1 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -19,6 +19,10 @@ # define PLATFORM_IS_ARM #endif +#if defined(__powerpc__) || defined(__powerpc64__) +# define PLATFORM_IS_POWERPC +#endif + namespace snmalloc { /** @@ -44,6 +48,14 @@ namespace snmalloc StrictProvenance = (1 << 2), }; + enum AalName : int + { + ARM, + PowerPC, + X86, + X86_SGX, + }; + /** * Architecture Abstraction Layer. Includes default implementations of some * functions using compiler builtins. Falls back to the definitions in the @@ -133,6 +145,8 @@ namespace snmalloc # include "aal_x86_sgx.h" #elif defined(PLATFORM_IS_ARM) # include "aal_arm.h" +#elif defined(PLATFORM_IS_POWERPC) +# include "aal_powerpc.h" #endif namespace snmalloc diff --git a/src/aal/aal_arm.h b/src/aal/aal_arm.h index baa690f..bd5f08b 100644 --- a/src/aal/aal_arm.h +++ b/src/aal/aal_arm.h @@ -27,6 +27,10 @@ namespace snmalloc static constexpr uint64_t aal_features = IntegerPointers | NoCpuCycleCounters; + static constexpr enum AalName aal_name = ARM; + + static constexpr size_t smallest_page_size = 0x1000; + /** * On pipelined processors, notify the core that we are in a spin loop and * that speculative execution past this point may not be a performance gain. diff --git a/src/aal/aal_powerpc.h b/src/aal/aal_powerpc.h new file mode 100644 index 0000000..60b9f12 --- /dev/null +++ b/src/aal/aal_powerpc.h @@ -0,0 +1,37 @@ +#pragma once + +#if defined(__powerpc64__) +# define SNMALLOC_VA_BITS_64 +#else +# define SNMALLOC_VA_BITS_32 +#endif + +namespace snmalloc +{ + /** + * ARM-specific architecture abstraction layer. + */ + class AAL_PowerPC + { + public: + /** + * Bitmap of AalFeature flags + */ + static constexpr uint64_t aal_features = IntegerPointers; + + static constexpr enum AalName aal_name = PowerPC; + + static constexpr size_t smallest_page_size = 0x1000; + + /** + * On pipelined processors, notify the core that we are in a spin loop and + * that speculative execution past this point may not be a performance gain. + */ + static inline void pause() + { + __asm__ volatile("or 27,27,27"); // "yield" + } + }; + + using AAL_Arch = AAL_PowerPC; +} // namespace snmalloc diff --git a/src/aal/aal_x86.h b/src/aal/aal_x86.h index 2473b5f..cc20e77 100644 --- a/src/aal/aal_x86.h +++ b/src/aal/aal_x86.h @@ -60,6 +60,10 @@ namespace snmalloc */ static constexpr uint64_t aal_features = IntegerPointers; + static constexpr enum AalName aal_name = X86; + + static constexpr size_t smallest_page_size = 0x1000; + /** * On pipelined processors, notify the core that we are in a spin loop and * that speculative execution past this point may not be a performance gain. diff --git a/src/aal/aal_x86_sgx.h b/src/aal/aal_x86_sgx.h index 3d9a467..87808a4 100644 --- a/src/aal/aal_x86_sgx.h +++ b/src/aal/aal_x86_sgx.h @@ -26,6 +26,10 @@ namespace snmalloc */ static constexpr uint64_t aal_features = IntegerPointers; + static constexpr enum AalName aal_name = X86_SGX; + + static constexpr size_t smallest_page_size = 0x1000; + /** * On pipelined processors, notify the core that we are in a spin loop and * that speculative execution past this point may not be a performance gain. diff --git a/src/mem/alloc.h b/src/mem/alloc.h index b238bc2..b568cbc 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -733,12 +733,6 @@ namespace snmalloc size_t size1 = sizeclass_to_size(sc1); size_t size2 = sizeclass_to_size(sc2); - // All medium size classes are page aligned. - if (i > NUM_SMALL_CLASSES) - { - SNMALLOC_ASSERT(is_aligned_block(nullptr, size1)); - } - SNMALLOC_ASSERT(sc1 == i); SNMALLOC_ASSERT(sc1 == sc2); SNMALLOC_ASSERT(size1 == size); diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 2e9bb6d..9704b94 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -92,18 +92,7 @@ namespace snmalloc // Used to isolate values on cache lines to prevent false sharing. static constexpr size_t CACHELINE_SIZE = 64; - // Used to keep Superslab metadata committed. - static constexpr size_t OS_PAGE_SIZE = 0x1000; static constexpr size_t PAGE_ALIGNED_SIZE = OS_PAGE_SIZE << INTERMEDIATE_BITS; - // Some system headers (e.g. Linux' sys/user.h, FreeBSD's machine/param.h) - // define `PAGE_SIZE` as a macro. We don't use `PAGE_SIZE` as our variable - // name, to avoid conflicts, but if we do see a macro definition then check - // that our value matches the platform's expected value. -#ifdef PAGE_SIZE - static_assert( - PAGE_SIZE == OS_PAGE_SIZE, - "Page size from system header does not match snmalloc config page size."); -#endif // Minimum allocation size is space for two pointers. static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*)); diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index d7632d5..59a10f2 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -85,11 +85,10 @@ namespace snmalloc void* p = pointer_offset(this, (static_cast(index) << 8)); free--; - SNMALLOC_ASSERT(is_aligned_block(p, OS_PAGE_SIZE)); - size = bits::align_up(size, OS_PAGE_SIZE); - if constexpr (zero_mem == YesZero) - memory_provider.template zero(p, size); + memory_provider.zero(p, size); + else + UNUSED(size); return p; } diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 5b200d1..e5bb6e3 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -1,6 +1,6 @@ #pragma once -#include "../pal/pal_consts.h" +#include "../pal/pal.h" #include "allocconfig.h" namespace snmalloc diff --git a/src/pal/pal.h b/src/pal/pal.h index ed5ea11..d8a106c 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -59,4 +59,25 @@ namespace snmalloc */ template constexpr static bool pal_supports = (PAL::pal_features & F) == F; + + // Used to keep Superslab metadata committed. + static constexpr size_t OS_PAGE_SIZE = Pal::page_size; + + static_assert( + bits::next_pow2_const(OS_PAGE_SIZE) == OS_PAGE_SIZE, + "OS_PAGE_SIZE must be a power of two"); + static_assert( + OS_PAGE_SIZE % Aal::smallest_page_size == 0, + "The smallest architectural page size must divide OS_PAGE_SIZE"); + + // Some system headers (e.g. Linux' sys/user.h, FreeBSD's machine/param.h) + // define `PAGE_SIZE` as a macro. We don't use `PAGE_SIZE` as our variable + // name, to avoid conflicts, but if we do see a macro definition then check + // that our value matches the platform's expected value. +#ifdef PAGE_SIZE + static_assert( + PAGE_SIZE == OS_PAGE_SIZE, + "Page size from system header does not match snmalloc config page size."); +#endif + } // namespace snmalloc diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index b27dd43..0f10ee4 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -32,9 +32,9 @@ namespace snmalloc template void zero(void* p, size_t size) { - if (page_aligned || is_aligned_block(p, size)) + if (page_aligned || is_aligned_block(p, size)) { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); void* r = mmap( p, size, diff --git a/src/pal/pal_bsd.h b/src/pal/pal_bsd.h index 2641a03..c446dcb 100644 --- a/src/pal/pal_bsd.h +++ b/src/pal/pal_bsd.h @@ -33,7 +33,7 @@ namespace snmalloc */ void notify_not_using(void* p, size_t size) noexcept { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); madvise(p, size, MADV_FREE); } }; diff --git a/src/pal/pal_freebsd_kernel.h b/src/pal/pal_freebsd_kernel.h index 6327942..84f4841 100644 --- a/src/pal/pal_freebsd_kernel.h +++ b/src/pal/pal_freebsd_kernel.h @@ -1,7 +1,6 @@ #pragma once #include "../ds/bits.h" -#include "../mem/allocconfig.h" #if defined(FreeBSD_KERNEL) extern "C" diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index d4c20a7..a645c16 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -2,7 +2,6 @@ #if defined(__linux__) # include "../ds/bits.h" -# include "../mem/allocconfig.h" # include "pal_posix.h" # include @@ -26,6 +25,9 @@ namespace snmalloc */ static constexpr uint64_t pal_features = PALPOSIX::pal_features; + static constexpr size_t page_size = + Aal::aal_name == PowerPC ? 0x10000 : 0x1000; + /** * OS specific function for zeroing memory. * @@ -41,12 +43,12 @@ namespace snmalloc // MADV_DONTNEED. switch back to memset only for QEMU. # ifndef SNMALLOC_QEMU_WORKAROUND if ( - (page_aligned || is_aligned_block(p, size)) && - (size > SLAB_SIZE)) + (page_aligned || is_aligned_block(p, size)) && + (size > 16 * page_size)) { // Only use this on large allocations as memset faster, and doesn't // introduce IPI so faster for small allocations. - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); madvise(p, size, MADV_DONTNEED); } else diff --git a/src/pal/pal_open_enclave.h b/src/pal/pal_open_enclave.h index 25983a4..77101db 100644 --- a/src/pal/pal_open_enclave.h +++ b/src/pal/pal_open_enclave.h @@ -19,6 +19,9 @@ namespace snmalloc * PAL supports. */ static constexpr uint64_t pal_features = 0; + + static constexpr size_t page_size = 0x1000; + [[noreturn]] static void error(const char* const str) { UNUSED(str); diff --git a/src/pal/pal_plain.h b/src/pal/pal_plain.h index b55edb4..a7cd199 100644 --- a/src/pal/pal_plain.h +++ b/src/pal/pal_plain.h @@ -1,7 +1,6 @@ #pragma once #include "../ds/bits.h" -#include "../mem/allocconfig.h" namespace snmalloc { diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index e1bef12..12a85a9 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -1,7 +1,6 @@ #pragma once #include "../ds/address.h" -#include "../mem/allocconfig.h" #if defined(BACKTRACE_HEADER) # include BACKTRACE_HEADER #endif @@ -39,6 +38,8 @@ namespace snmalloc */ static constexpr uint64_t pal_features = LazyCommit; + static constexpr size_t page_size = 0x1000; + static void print_stack_trace() { #ifdef BACKTRACE_HEADER @@ -72,7 +73,7 @@ namespace snmalloc */ void notify_not_using(void* p, size_t size) noexcept { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); #ifdef USE_POSIX_COMMIT_CHECKS mprotect(p, size, PROT_NONE); #else @@ -92,7 +93,7 @@ namespace snmalloc void notify_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT( - is_aligned_block(p, size) || (zero_mem == NoZero)); + is_aligned_block(p, size) || (zero_mem == NoZero)); #ifdef USE_POSIX_COMMIT_CHECKS mprotect(p, size, PROT_READ | PROT_WRITE); @@ -119,9 +120,9 @@ namespace snmalloc template void zero(void* p, size_t size) noexcept { - if (page_aligned || is_aligned_block(p, size)) + if (page_aligned || is_aligned_block(p, size)) { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); void* r = mmap( p, size, diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index e325d3a..f8a5325 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -2,7 +2,6 @@ #include "../ds/address.h" #include "../ds/bits.h" -#include "../mem/allocconfig.h" #ifdef _WIN32 # ifndef _MSC_VER @@ -83,6 +82,8 @@ namespace snmalloc # endif ; + static constexpr size_t page_size = 0x1000; + /** * Check whether the low memory state is still in effect. This is an * expensive operation and should not be on any fast paths. @@ -115,7 +116,7 @@ namespace snmalloc /// Notify platform that we will not be using these pages void notify_not_using(void* p, size_t size) noexcept { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); BOOL ok = VirtualFree(p, size, MEM_DECOMMIT); @@ -128,7 +129,7 @@ namespace snmalloc void notify_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT( - is_aligned_block(p, size) || (zero_mem == NoZero)); + is_aligned_block(p, size) || (zero_mem == NoZero)); void* r = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE); @@ -140,9 +141,9 @@ namespace snmalloc template void zero(void* p, size_t size) noexcept { - if (page_aligned || is_aligned_block(p, size)) + if (page_aligned || is_aligned_block(p, size)) { - SNMALLOC_ASSERT(is_aligned_block(p, size)); + SNMALLOC_ASSERT(is_aligned_block(p, size)); notify_not_using(p, size); notify_using(p, size); }