diff --git a/src/mem/alloc.h b/src/mem/alloc.h index bd76171..c721948 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -46,8 +46,9 @@ namespace snmalloc // Use flat map is under a single node. # define SNMALLOC_MAX_FLATPAGEMAP_SIZE PAGEMAP_NODE_SIZE #endif - static constexpr bool USE_FLATPAGEMAP = SNMALLOC_MAX_FLATPAGEMAP_SIZE >= - sizeof(FlatPagemap); + static constexpr bool USE_FLATPAGEMAP = pal_supports() || + (SNMALLOC_MAX_FLATPAGEMAP_SIZE >= + sizeof(FlatPagemap)); using SuperslabPagemap = std::conditional_t< USE_FLATPAGEMAP, @@ -1205,9 +1206,7 @@ namespace snmalloc else if constexpr (decommit_strategy == DecommitSuperLazy) { static_assert( - std::remove_reference_t:: - template pal_supports(), + pal_supports(), "A lazy decommit strategy cannot be implemented on platforms " "without low memory notifications"); } @@ -1366,15 +1365,12 @@ namespace snmalloc large_allocator.dealloc(slab, large_class); } -#if defined(__GNUC__) && !defined(__clang__) && !defined(__OPTIMIZE__) - // Don't force this to be always inlined in debug builds with GCC, because - // it will fail and then raise an error. - inline -#else - SNMALLOC_FAST_PATH -#endif - void - remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) + // Note that this is on the slow path as it lead to better code. + // As it is tail, not inlining means that it is jumped to, so has no perf + // impact on the producer consumer scenarios, and doesn't require register + // spills in the fast path for local deallocation. + SNMALLOC_SLOW_PATH + void remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) { MEASURE_TIME(remote_dealloc, 4, 16); assert(target->id() != id()); diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 6d5ee5c..f36c1b5 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -175,15 +175,6 @@ namespace snmalloc return new (p) T(std::forward(args)...); } - /** - * Query whether the PAL supports a specific feature. - */ - template - constexpr static bool pal_supports() - { - return (PAL::pal_features & F) == F; - } - /** * Returns the number of low memory notifications that have been received * (over the lifetime of this process). If the underlying system does not @@ -192,7 +183,7 @@ namespace snmalloc SNMALLOC_FAST_PATH uint64_t low_memory_epoch() { - if constexpr (pal_supports()) + if constexpr (pal_supports()) { return PAL::low_memory_epoch(); } @@ -205,7 +196,7 @@ namespace snmalloc template void* reserve(size_t* size, size_t align) noexcept { - if constexpr (pal_supports()) + if constexpr (pal_supports()) { return PAL::template reserve(size, align); } diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 4ac6708..28bfafc 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -334,7 +334,7 @@ namespace snmalloc * so. If we have not allocated a per-thread allocator yet, then this * function will allocate one. */ - ALWAYSINLINE void* lazy_replacement(void* existing) + SNMALLOC_FAST_PATH void* lazy_replacement(void* existing) { if (existing != &GlobalPlaceHolder) { diff --git a/src/pal/pal.h b/src/pal/pal.h index 7eb5a8f..799f859 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -50,4 +50,13 @@ namespace snmalloc { Pal::error(str); } + + /** + * Query whether the PAL supports a specific feature. + */ + template + constexpr static bool pal_supports() + { + return (PAL::pal_features & F) == F; + } } // namespace snmalloc diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index 9e456d5..97f72bb 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -22,7 +22,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = 0; + static constexpr uint64_t pal_features = LazyCommit; static void error(const char* const str) { puts(str); diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index e5e165d..b80c543 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -25,7 +25,13 @@ namespace snmalloc * a size and alignment. A PAL that does *not* support it must expose a * `request()` method that takes only a size. */ - AlignedAllocation = (1 << 1) + AlignedAllocation = (1 << 1), + /** + * This PAL natively supports lazy commit of pages. This means have large + * allocations and not touching them does not increase memory usage. This is + * exposed in the Pal. + */ + LazyCommit = (1 << 2), }; /** * Flag indicating whether requested memory should be zeroed. diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index 8487f19..d0e6bc6 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -17,7 +17,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = AlignedAllocation; + static constexpr uint64_t pal_features = AlignedAllocation | LazyCommit; static void error(const char* const str) { puts(str); diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index b2af742..dcb75dd 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -18,7 +18,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = 0; + static constexpr uint64_t pal_features = LazyCommit; static void error(const char* const str) { puts(str); diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 70d8149..0105a47 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -18,8 +18,14 @@ namespace test { size_t rand = (size_t)r.next(); size_t offset = bits::clz(rand); - if (offset > 30) - offset = 30; + if constexpr (bits::is64()) + { + if (offset > 30) + offset = 30; + } + else if (offset > 20) + offset = 20; + size_t size = (rand & 15) << offset; if (size < 16) size = 16; @@ -44,11 +50,15 @@ namespace test void test_external_pointer(xoroshiro::p128r64& r) { auto& alloc = ThreadAlloc::get(); - +#ifdef NDEBUG + static constexpr size_t iterations = 10000000; +#else + static constexpr size_t iterations = 100000; +#endif setup(r, alloc); DO_TIME("External pointer queries ", { - for (size_t i = 0; i < 10000000; i++) + for (size_t i = 0; i < iterations; i++) { size_t rand = (size_t)r.next(); size_t oid = rand & (((size_t)1 << count_log) - 1);