From 813367286eace2edc8811fe704a290b12c8b8152 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 21 Feb 2020 11:34:39 +0000 Subject: [PATCH] Make Lazy Decomit asynchronous On platforms that support low-memory notifications register callbacks that perform lazy decommit. This allows idle processes to return memory to the OS. Without incurring the cost of constantly committing and decommitting memory. Code review and CI changes * Fixed test to use a template to make constexpr magic work * Factored out basic notification mechanism so can be reused on other platforms. --- src/mem/largealloc.h | 82 ++++++-------------------- src/pal/pal_consts.h | 63 +++++++++++++++++++- src/pal/pal_windows.h | 36 +++++------ src/test/perf/low_memory/low-memory.cc | 78 +++++++++++++++++++----- 4 files changed, 161 insertions(+), 98 deletions(-) diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 5c1437b..d0c2762 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -55,7 +55,7 @@ namespace snmalloc // global state of the allocator. This is currently stored in the memory // provider, so we add this in. template - class MemoryProviderStateMixin : public PAL + class MemoryProviderStateMixin : public PalNotificationObject, public PAL { /** * Flag to protect the bump allocator @@ -72,11 +72,6 @@ namespace snmalloc **/ size_t remaining = 0; - /** - * The last time we saw a low memory notification. - */ - std::atomic last_low_memory_epoch = 0; - /** * Simple flag for checking if another instance of lazy-decommit is * running @@ -105,6 +100,13 @@ namespace snmalloc // memcpy is safe as this is entirely single threaded. memcpy(allocated, &local, sizeof(MemoryProviderStateMixin)); + // Register this allocator for low-memory call-backs + if constexpr (pal_supports) + { + allocated->PalNotificationObject::pal_notify = &(allocated->process); + PAL::register_for_low_memory_callback(allocated); + } + return allocated; } @@ -183,6 +185,16 @@ namespace snmalloc large_stack[large_class].push(reinterpret_cast(p)); } + /*** + * Method for callback object to perform lazy decommit. + **/ + static void process(PalNotificationObject* p) + { + // Unsafe downcast here. Don't want vtable and RTTI. + auto self = reinterpret_cast*>(p); + self->lazy_decommit(); + } + public: /** * Primitive allocator for structure that are required before @@ -235,24 +247,6 @@ namespace snmalloc return new (p) T(std::forward(args)...); } - /** - * Returns the number of low memory notifications that have been received - * (over the lifetime of this process). If the underlying system does not - * support low memory notifications, this will return 0. - */ - SNMALLOC_FAST_PATH - uint64_t low_memory_epoch() - { - if constexpr (pal_supports) - { - return PAL::low_memory_epoch(); - } - else - { - return 0; - } - } - template void* reserve(size_t large_class) noexcept { @@ -316,41 +310,6 @@ namespace snmalloc return result; } } - - SNMALLOC_FAST_PATH void lazy_decommit_if_needed() - { -#ifdef TEST_LAZY_DECOMMIT - static_assert( - TEST_LAZY_DECOMMIT > 0, - "TEST_LAZY_DECOMMIT must be a positive integer value."); - static std::atomic counter; - auto c = counter++; - if (c % TEST_LAZY_DECOMMIT == 0) - { - lazy_decommit(); - } -#else - if constexpr (decommit_strategy == DecommitSuperLazy) - { - auto new_epoch = low_memory_epoch(); - auto old_epoch = last_low_memory_epoch.load(std::memory_order_acquire); - if (new_epoch > old_epoch) - { - // Try to update the epoch to the value that we've seen. If - // another thread has seen a newer epoch than us (or done the same - // update) let them win. - do - { - if (last_low_memory_epoch.compare_exchange_strong( - old_epoch, new_epoch)) - { - lazy_decommit(); - } - } while (old_epoch < new_epoch); - } - } -#endif - } }; using Stats = AllocStats; @@ -381,7 +340,6 @@ namespace snmalloc size = rsize; void* p = memory_provider.large_stack[large_class].pop(); - memory_provider.lazy_decommit_if_needed(); if (p == nullptr) { @@ -392,8 +350,7 @@ namespace snmalloc { stats.superslab_pop(); - // Cross-reference alloc.h's large_dealloc decommitment condition - // and lazy_decommit_if_needed. + // Cross-reference alloc.h's large_dealloc decommitment condition. bool decommitted = ((decommit_strategy == DecommitSuperLazy) && (static_cast(p)->get_kind() == Decommitted)) || @@ -449,7 +406,6 @@ namespace snmalloc stats.superslab_push(); memory_provider.large_stack[large_class].push(static_cast(p)); - memory_provider.lazy_decommit_if_needed(); } }; diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 0261155..0c04793 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -10,8 +10,8 @@ namespace snmalloc { /** * This PAL supports low memory notifications. It must implement a - * `low_memory_epoch` method that returns a `uint64_t` of the number of - * times that a low-memory notification has been raised and an + * `register_for_low_memory_callback` method that allows callbacks to be + * registered when the platform detects low-memory and an * `expensive_low_memory_check()` method that returns a `bool` indicating * whether low memory conditions are still in effect. */ @@ -53,4 +53,63 @@ namespace snmalloc * Default Tag ID for the Apple class */ static const int PALAnonDefaultID = 241; + + /** + * This struct is used to represent callbacks for notification from the + * platform. It contains a next pointer as client is responsible for + * allocation as we cannot assume an allocator at this point. + **/ + struct PalNotificationObject + { + std::atomic pal_next; + + void (*pal_notify)(PalNotificationObject* self); + }; + + /*** + * Wrapper for managing notifications for PAL events + **/ + class PalNotifier + { + /** + * List of callbacks to notify + **/ + std::atomic callbacks = nullptr; + + public: + /** + * Register a callback object to be notified + * + * The object should never be deallocated by the client after calling + * this. + **/ + void register_notification(PalNotificationObject* callback) + { + callback->pal_next = nullptr; + + auto prev = &callbacks; + auto curr = prev->load(); + do + { + while (curr != nullptr) + { + prev = &(curr->pal_next); + curr = prev->load(); + } + } while (!prev->compare_exchange_weak(curr, callback)); + } + + /** + * Calls the pal_notify of all the registered objects. + **/ + void notify_all() + { + PalNotificationObject* curr = callbacks; + while (curr != nullptr) + { + curr->pal_notify(curr); + curr = curr->pal_next; + } + } + }; } // namespace snmalloc diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 42cb699..b96deb2 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -20,23 +20,25 @@ namespace snmalloc { class PALWindows { - /** - * The number of times that the memory pressure notification has fired. - */ - static inline std::atomic pressure_epoch; /** * A flag indicating that we have tried to register for low-memory * notifications. */ static inline std::atomic registered_for_notifications; static inline HANDLE lowMemoryObject; + + /** + * List of callbacks for low-memory notification + **/ + static inline PalNotifier low_memory_callbacks; + /** * Callback, used when the system delivers a low-memory notification. This - * simply increments an atomic counter each time the notification is raised. + * calls all the handlers registered with the PAL. */ static void CALLBACK low_memory(_In_ PVOID, _In_ BOOLEAN) { - pressure_epoch++; + low_memory_callbacks.notify_all(); } public: @@ -76,17 +78,6 @@ namespace snmalloc # endif ; - /** - * Counter values for the number of times that a low-pressure notification - * has been delivered. Callers should compare this with a previous value - * to see if the low memory state has been triggered since they last - * checked. - */ - uint64_t low_memory_epoch() - { - return pressure_epoch.load(std::memory_order_acquire); - } - /** * Check whether the low memory state is still in effect. This is an * expensive operation and should not be on any fast paths. @@ -98,6 +89,17 @@ namespace snmalloc return result; } + /** + * Register callback object for low-memory notifications. + * Client is responsible for allocation, and ensuring the object is live + * for the duration of the program. + **/ + static void + register_for_low_memory_callback(PalNotificationObject* callback) + { + low_memory_callbacks.register_notification(callback); + } + static void error(const char* const str) { puts(str); diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index dd890ae..046a2cc 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -37,29 +38,34 @@ public: tail = tail->next; } - void try_remove() + bool try_remove() { if (head->next == nullptr) - return; + return false; Node* next = head->next; ThreadAlloc::get()->dealloc(head); head = next; + return true; } }; +std::atomic global_epoch = 0; + +void advance(PalNotificationObject* unused) +{ + UNUSED(unused); + global_epoch++; +} + +PalNotificationObject update_epoch = {nullptr, &advance}; + bool has_pressure() { - static thread_local uint64_t epoch; + static thread_local uint64_t epoch = 0; - if constexpr (!pal_supports) - { - return false; - } - - uint64_t current_epoch = default_memory_provider().low_memory_epoch(); - bool result = epoch != current_epoch; - epoch = current_epoch; + bool result = epoch != global_epoch; + epoch = global_epoch; return result; } @@ -79,7 +85,7 @@ void reach_pressure(Queue& allocations) void reduce_pressure(Queue& allocations) { size_t size = 4096; - for (size_t n = 0; n < 1000; n++) + for (size_t n = 0; n < 10000; n++) { allocations.try_remove(); allocations.try_remove(); @@ -87,21 +93,51 @@ void reduce_pressure(Queue& allocations) } } -int main(int, char**) +/** + * Wrapper to handle Pals that don't have the method. + * Template parameter required to handle `if constexpr` always evaluating both + * sides. + **/ +template +void register_for_pal_notifications() { -#ifndef NDEBUG - Queue allocations; + PAL::register_for_low_memory_callback(&update_epoch); +} - if constexpr (!pal_supports) +int main(int argc, char** argv) +{ + opt::Opt opt(argc, argv); + + if constexpr (pal_supports) + { + register_for_pal_notifications(); + } + else { std::cout << "Pal does not support low-memory notification! Test not run" << std::endl; return 0; } +#ifdef NDEBUG # if defined(WIN32) && !defined(SNMALLOC_VA_BITS_64) std::cout << "32-bit windows not supported for this test." << std::endl; # else + + bool interactive = opt.has("--interactive"); + + Queue allocations; + + std::cout + << "Expected use:" << std::endl + << " run first instances with --interactive. Wait for first to print " + << std::endl + << " 'No allocations left. Press any key to terminate'" << std::endl + << "watch working set, and start second instance working set of first " + << "should drop to almost zero," << std::endl + << "and second should climb to physical ram." << std::endl + << std::endl; + setup(); for (size_t i = 0; i < 10; i++) @@ -111,6 +147,16 @@ int main(int, char**) reduce_pressure(allocations); } + + // Deallocate everything + while (allocations.try_remove()) + ; + + if (interactive) + { + std::cout << "No allocations left. Press any key to terminate" << std::endl; + getchar(); + } # endif #else std::cout << "Release test only." << std::endl;