From e240dd279a0c40247abd1324604eb58be6cf620c Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 9 Jul 2019 10:41:58 +0100 Subject: [PATCH 1/8] Use FlatPageMap on OS with lazy commit If the operating system will allocate private pages on demand for the pagemap then use the FlatPageMap by default as it generates better code for deallocation. --- src/mem/alloc.h | 8 ++++++-- src/pal/pal_apple.h | 2 +- src/pal/pal_consts.h | 8 +++++++- src/pal/pal_freebsd.h | 2 +- src/pal/pal_linux.h | 2 +- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index bd76171..c08ac2d 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -46,8 +46,10 @@ namespace snmalloc // Use flat map is under a single node. # define SNMALLOC_MAX_FLATPAGEMAP_SIZE PAGEMAP_NODE_SIZE #endif - static constexpr bool USE_FLATPAGEMAP = SNMALLOC_MAX_FLATPAGEMAP_SIZE >= - sizeof(FlatPagemap); + static constexpr bool USE_FLATPAGEMAP = + (Pal::pal_features & PalFeatures::LazyCommit) || + (SNMALLOC_MAX_FLATPAGEMAP_SIZE >= + sizeof(FlatPagemap)); using SuperslabPagemap = std::conditional_t< USE_FLATPAGEMAP, @@ -1152,6 +1154,8 @@ namespace snmalloc SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow( Superslab* super, void* p, sizeclass_t sizeclass) { + handle_message_queue(); + bool was_full = super->is_full(); SlabList* sl = &small_classes[sizeclass]; Slab* slab = Slab::get(p); diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index 9e456d5..97f72bb 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -22,7 +22,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = 0; + static constexpr uint64_t pal_features = LazyCommit; static void error(const char* const str) { puts(str); diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index e5e165d..40e50bb 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -25,7 +25,13 @@ namespace snmalloc * a size and alignment. A PAL that does *not* support it must expose a * `request()` method that takes only a size. */ - AlignedAllocation = (1 << 1) + AlignedAllocation = (1 << 1), + /** + * This PAL natively supports lazy commit of pages. This means have large + * allocations and not touching them does not up memory usage. This is + * exposed in the P + */ + LazyCommit = (1 << 2), }; /** * Flag indicating whether requested memory should be zeroed. diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index 8487f19..d0e6bc6 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -17,7 +17,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = AlignedAllocation; + static constexpr uint64_t pal_features = AlignedAllocation | LazyCommit; static void error(const char* const str) { puts(str); diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index b2af742..dcb75dd 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -18,7 +18,7 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. */ - static constexpr uint64_t pal_features = 0; + static constexpr uint64_t pal_features = LazyCommit; static void error(const char* const str) { puts(str); From 9fd238d5faac5d90fd76e6017083a41fb2a92a04 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 9 Jul 2019 10:52:52 +0100 Subject: [PATCH 2/8] Move remote dealloc to slow path It is only ever called in a tail position, so slow path means it is just a jump, but improves the local deallocation fast path's codegen considerably. --- src/mem/alloc.h | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index c08ac2d..fa691ba 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1370,15 +1370,8 @@ namespace snmalloc large_allocator.dealloc(slab, large_class); } -#if defined(__GNUC__) && !defined(__clang__) && !defined(__OPTIMIZE__) - // Don't force this to be always inlined in debug builds with GCC, because - // it will fail and then raise an error. - inline -#else - SNMALLOC_FAST_PATH -#endif - void - remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) + SNMALLOC_SLOW_PATH + void remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) { MEASURE_TIME(remote_dealloc, 4, 16); assert(target->id() != id()); From c6178322b07b89f74c2c7a88c4d711bda9ff616e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 9 Jul 2019 10:53:07 +0100 Subject: [PATCH 3/8] Minor --- src/mem/threadalloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 4ac6708..28bfafc 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -334,7 +334,7 @@ namespace snmalloc * so. If we have not allocated a per-thread allocator yet, then this * function will allocate one. */ - ALWAYSINLINE void* lazy_replacement(void* existing) + SNMALLOC_FAST_PATH void* lazy_replacement(void* existing) { if (existing != &GlobalPlaceHolder) { From 23b3e35d6e125f2cbc8ef9db0297de08c9ba2324 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 9 Jul 2019 13:14:17 +0100 Subject: [PATCH 4/8] Add comment about inlining choices. --- src/mem/alloc.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index fa691ba..66f4091 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1370,6 +1370,10 @@ namespace snmalloc large_allocator.dealloc(slab, large_class); } + // Note that this is on the slow path as it lead to better code. + // As it is tail, not inlining means that it is jumped to, so has no perf + // impact on the producer consumer scenarios, and doesn't require register + // spills in the fast path for local deallocation. SNMALLOC_SLOW_PATH void remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) { From 45f47499c549d76d6e1ab10ef1a4a7849770c18b Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 9 Jul 2019 13:14:48 +0100 Subject: [PATCH 5/8] Improved pal_supports --- src/mem/alloc.h | 7 ++----- src/mem/largealloc.h | 13 ++----------- src/pal/pal.h | 9 +++++++++ src/pal/pal_consts.h | 4 ++-- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 66f4091..023b2c1 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -46,8 +46,7 @@ namespace snmalloc // Use flat map is under a single node. # define SNMALLOC_MAX_FLATPAGEMAP_SIZE PAGEMAP_NODE_SIZE #endif - static constexpr bool USE_FLATPAGEMAP = - (Pal::pal_features & PalFeatures::LazyCommit) || + static constexpr bool USE_FLATPAGEMAP = pal_supports() || (SNMALLOC_MAX_FLATPAGEMAP_SIZE >= sizeof(FlatPagemap)); @@ -1209,9 +1208,7 @@ namespace snmalloc else if constexpr (decommit_strategy == DecommitSuperLazy) { static_assert( - std::remove_reference_t:: - template pal_supports(), + pal_supports(), "A lazy decommit strategy cannot be implemented on platforms " "without low memory notifications"); } diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 6d5ee5c..f36c1b5 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -175,15 +175,6 @@ namespace snmalloc return new (p) T(std::forward(args)...); } - /** - * Query whether the PAL supports a specific feature. - */ - template - constexpr static bool pal_supports() - { - return (PAL::pal_features & F) == F; - } - /** * Returns the number of low memory notifications that have been received * (over the lifetime of this process). If the underlying system does not @@ -192,7 +183,7 @@ namespace snmalloc SNMALLOC_FAST_PATH uint64_t low_memory_epoch() { - if constexpr (pal_supports()) + if constexpr (pal_supports()) { return PAL::low_memory_epoch(); } @@ -205,7 +196,7 @@ namespace snmalloc template void* reserve(size_t* size, size_t align) noexcept { - if constexpr (pal_supports()) + if constexpr (pal_supports()) { return PAL::template reserve(size, align); } diff --git a/src/pal/pal.h b/src/pal/pal.h index 7eb5a8f..799f859 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -50,4 +50,13 @@ namespace snmalloc { Pal::error(str); } + + /** + * Query whether the PAL supports a specific feature. + */ + template + constexpr static bool pal_supports() + { + return (PAL::pal_features & F) == F; + } } // namespace snmalloc diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 40e50bb..b80c543 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -28,8 +28,8 @@ namespace snmalloc AlignedAllocation = (1 << 1), /** * This PAL natively supports lazy commit of pages. This means have large - * allocations and not touching them does not up memory usage. This is - * exposed in the P + * allocations and not touching them does not increase memory usage. This is + * exposed in the Pal. */ LazyCommit = (1 << 2), }; From 33ff935aeed75ae28eaedc9f8218337a7dba94a5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 9 Jul 2019 14:57:23 +0100 Subject: [PATCH 6/8] Remove infinite loop from adding message queue check Adding a handle_message_queue here lead to an infinite loop. --- src/mem/alloc.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 023b2c1..c721948 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1153,8 +1153,6 @@ namespace snmalloc SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow( Superslab* super, void* p, sizeclass_t sizeclass) { - handle_message_queue(); - bool was_full = super->is_full(); SlabList* sl = &small_classes[sizeclass]; Slab* slab = Slab::get(p); From c2785ec66137c260965b0b654a1e8b5ead44e6fc Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 9 Jul 2019 15:19:46 +0100 Subject: [PATCH 7/8] Reduce test size. --- src/test/perf/external_pointer/externalpointer.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 70d8149..06a6a9e 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -44,11 +44,15 @@ namespace test void test_external_pointer(xoroshiro::p128r64& r) { auto& alloc = ThreadAlloc::get(); - +#ifdef NDEBUG + static constexpr size_t iterations = 10000000; +#else + static constexpr size_t iterations = 100000; +#endif setup(r, alloc); DO_TIME("External pointer queries ", { - for (size_t i = 0; i < 10000000; i++) + for (size_t i = 0; i < iterations; i++) { size_t rand = (size_t)r.next(); size_t oid = rand & (((size_t)1 << count_log) - 1); From c1c9237b8d2793cc88922c1efdb7918f94f63d9d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 10 Jul 2019 16:01:17 +0100 Subject: [PATCH 8/8] Handle 32bit to not allocate way to mcuch. --- src/test/perf/external_pointer/externalpointer.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 06a6a9e..0105a47 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -18,8 +18,14 @@ namespace test { size_t rand = (size_t)r.next(); size_t offset = bits::clz(rand); - if (offset > 30) - offset = 30; + if constexpr (bits::is64()) + { + if (offset > 30) + offset = 30; + } + else if (offset > 20) + offset = 20; + size_t size = (rand & 15) << offset; if (size < 16) size = 16;