diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 583acc4..ab1507b 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -56,7 +56,7 @@ namespace snmalloc // global state of the allocator. This is currently stored in the memory // provider, so we add this in. template - class MemoryProviderStateMixin : public PAL + class MemoryProviderStateMixin : public PalNotificationObject, public PAL { /** * Flag to protect the bump allocator @@ -73,11 +73,6 @@ namespace snmalloc **/ size_t remaining = 0; - /** - * The last time we saw a low memory notification. - */ - std::atomic last_low_memory_epoch = 0; - /** * Simple flag for checking if another instance of lazy-decommit is * running @@ -106,6 +101,13 @@ namespace snmalloc // memcpy is safe as this is entirely single threaded. memcpy(allocated, &local, sizeof(MemoryProviderStateMixin)); + // Register this allocator for low-memory call-backs + if constexpr (pal_supports) + { + allocated->PalNotificationObject::pal_notify = &(allocated->process); + PAL::register_for_low_memory_callback(allocated); + } + return allocated; } @@ -184,6 +186,16 @@ namespace snmalloc large_stack[large_class].push(reinterpret_cast(p)); } + /*** + * Method for callback object to perform lazy decommit. + **/ + static void process(PalNotificationObject* p) + { + // Unsafe downcast here. Don't want vtable and RTTI. + auto self = reinterpret_cast*>(p); + self->lazy_decommit(); + } + public: /** * Primitive allocator for structure that are required before @@ -236,24 +248,6 @@ namespace snmalloc return new (p) T(std::forward(args)...); } - /** - * Returns the number of low memory notifications that have been received - * (over the lifetime of this process). If the underlying system does not - * support low memory notifications, this will return 0. - */ - SNMALLOC_FAST_PATH - uint64_t low_memory_epoch() - { - if constexpr (pal_supports) - { - return PAL::low_memory_epoch(); - } - else - { - return 0; - } - } - template void* reserve(size_t large_class) noexcept { @@ -317,41 +311,6 @@ namespace snmalloc return result; } } - - SNMALLOC_FAST_PATH void lazy_decommit_if_needed() - { -#ifdef TEST_LAZY_DECOMMIT - static_assert( - TEST_LAZY_DECOMMIT > 0, - "TEST_LAZY_DECOMMIT must be a positive integer value."); - static std::atomic counter; - auto c = counter++; - if (c % TEST_LAZY_DECOMMIT == 0) - { - lazy_decommit(); - } -#else - if constexpr (decommit_strategy == DecommitSuperLazy) - { - auto new_epoch = low_memory_epoch(); - auto old_epoch = last_low_memory_epoch.load(std::memory_order_acquire); - if (new_epoch > old_epoch) - { - // Try to update the epoch to the value that we've seen. If - // another thread has seen a newer epoch than us (or done the same - // update) let them win. - do - { - if (last_low_memory_epoch.compare_exchange_strong( - old_epoch, new_epoch)) - { - lazy_decommit(); - } - } while (old_epoch < new_epoch); - } - } -#endif - } }; using Stats = AllocStats; @@ -382,7 +341,6 @@ namespace snmalloc size = rsize; void* p = memory_provider.large_stack[large_class].pop(); - memory_provider.lazy_decommit_if_needed(); if (p == nullptr) { @@ -393,8 +351,7 @@ namespace snmalloc { stats.superslab_pop(); - // Cross-reference alloc.h's large_dealloc decommitment condition - // and lazy_decommit_if_needed. + // Cross-reference alloc.h's large_dealloc decommitment condition. bool decommitted = ((decommit_strategy == DecommitSuperLazy) && (static_cast(p)->get_kind() == Decommitted)) || @@ -450,7 +407,6 @@ namespace snmalloc stats.superslab_push(); memory_provider.large_stack[large_class].push(static_cast(p)); - memory_provider.lazy_decommit_if_needed(); } }; diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 0261155..0c04793 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -10,8 +10,8 @@ namespace snmalloc { /** * This PAL supports low memory notifications. It must implement a - * `low_memory_epoch` method that returns a `uint64_t` of the number of - * times that a low-memory notification has been raised and an + * `register_for_low_memory_callback` method that allows callbacks to be + * registered when the platform detects low-memory and an * `expensive_low_memory_check()` method that returns a `bool` indicating * whether low memory conditions are still in effect. */ @@ -53,4 +53,63 @@ namespace snmalloc * Default Tag ID for the Apple class */ static const int PALAnonDefaultID = 241; + + /** + * This struct is used to represent callbacks for notification from the + * platform. It contains a next pointer as client is responsible for + * allocation as we cannot assume an allocator at this point. + **/ + struct PalNotificationObject + { + std::atomic pal_next; + + void (*pal_notify)(PalNotificationObject* self); + }; + + /*** + * Wrapper for managing notifications for PAL events + **/ + class PalNotifier + { + /** + * List of callbacks to notify + **/ + std::atomic callbacks = nullptr; + + public: + /** + * Register a callback object to be notified + * + * The object should never be deallocated by the client after calling + * this. + **/ + void register_notification(PalNotificationObject* callback) + { + callback->pal_next = nullptr; + + auto prev = &callbacks; + auto curr = prev->load(); + do + { + while (curr != nullptr) + { + prev = &(curr->pal_next); + curr = prev->load(); + } + } while (!prev->compare_exchange_weak(curr, callback)); + } + + /** + * Calls the pal_notify of all the registered objects. + **/ + void notify_all() + { + PalNotificationObject* curr = callbacks; + while (curr != nullptr) + { + curr->pal_notify(curr); + curr = curr->pal_next; + } + } + }; } // namespace snmalloc diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 42cb699..b96deb2 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -20,23 +20,25 @@ namespace snmalloc { class PALWindows { - /** - * The number of times that the memory pressure notification has fired. - */ - static inline std::atomic pressure_epoch; /** * A flag indicating that we have tried to register for low-memory * notifications. */ static inline std::atomic registered_for_notifications; static inline HANDLE lowMemoryObject; + + /** + * List of callbacks for low-memory notification + **/ + static inline PalNotifier low_memory_callbacks; + /** * Callback, used when the system delivers a low-memory notification. This - * simply increments an atomic counter each time the notification is raised. + * calls all the handlers registered with the PAL. */ static void CALLBACK low_memory(_In_ PVOID, _In_ BOOLEAN) { - pressure_epoch++; + low_memory_callbacks.notify_all(); } public: @@ -76,17 +78,6 @@ namespace snmalloc # endif ; - /** - * Counter values for the number of times that a low-pressure notification - * has been delivered. Callers should compare this with a previous value - * to see if the low memory state has been triggered since they last - * checked. - */ - uint64_t low_memory_epoch() - { - return pressure_epoch.load(std::memory_order_acquire); - } - /** * Check whether the low memory state is still in effect. This is an * expensive operation and should not be on any fast paths. @@ -98,6 +89,17 @@ namespace snmalloc return result; } + /** + * Register callback object for low-memory notifications. + * Client is responsible for allocation, and ensuring the object is live + * for the duration of the program. + **/ + static void + register_for_low_memory_callback(PalNotificationObject* callback) + { + low_memory_callbacks.register_notification(callback); + } + static void error(const char* const str) { puts(str); diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index dd890ae..046a2cc 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -37,29 +38,34 @@ public: tail = tail->next; } - void try_remove() + bool try_remove() { if (head->next == nullptr) - return; + return false; Node* next = head->next; ThreadAlloc::get()->dealloc(head); head = next; + return true; } }; +std::atomic global_epoch = 0; + +void advance(PalNotificationObject* unused) +{ + UNUSED(unused); + global_epoch++; +} + +PalNotificationObject update_epoch = {nullptr, &advance}; + bool has_pressure() { - static thread_local uint64_t epoch; + static thread_local uint64_t epoch = 0; - if constexpr (!pal_supports) - { - return false; - } - - uint64_t current_epoch = default_memory_provider().low_memory_epoch(); - bool result = epoch != current_epoch; - epoch = current_epoch; + bool result = epoch != global_epoch; + epoch = global_epoch; return result; } @@ -79,7 +85,7 @@ void reach_pressure(Queue& allocations) void reduce_pressure(Queue& allocations) { size_t size = 4096; - for (size_t n = 0; n < 1000; n++) + for (size_t n = 0; n < 10000; n++) { allocations.try_remove(); allocations.try_remove(); @@ -87,21 +93,51 @@ void reduce_pressure(Queue& allocations) } } -int main(int, char**) +/** + * Wrapper to handle Pals that don't have the method. + * Template parameter required to handle `if constexpr` always evaluating both + * sides. + **/ +template +void register_for_pal_notifications() { -#ifndef NDEBUG - Queue allocations; + PAL::register_for_low_memory_callback(&update_epoch); +} - if constexpr (!pal_supports) +int main(int argc, char** argv) +{ + opt::Opt opt(argc, argv); + + if constexpr (pal_supports) + { + register_for_pal_notifications(); + } + else { std::cout << "Pal does not support low-memory notification! Test not run" << std::endl; return 0; } +#ifdef NDEBUG # if defined(WIN32) && !defined(SNMALLOC_VA_BITS_64) std::cout << "32-bit windows not supported for this test." << std::endl; # else + + bool interactive = opt.has("--interactive"); + + Queue allocations; + + std::cout + << "Expected use:" << std::endl + << " run first instances with --interactive. Wait for first to print " + << std::endl + << " 'No allocations left. Press any key to terminate'" << std::endl + << "watch working set, and start second instance working set of first " + << "should drop to almost zero," << std::endl + << "and second should climb to physical ram." << std::endl + << std::endl; + setup(); for (size_t i = 0; i < 10; i++) @@ -111,6 +147,16 @@ int main(int, char**) reduce_pressure(allocations); } + + // Deallocate everything + while (allocations.try_remove()) + ; + + if (interactive) + { + std::cout << "No allocations left. Press any key to terminate" << std::endl; + getchar(); + } # endif #else std::cout << "Release test only." << std::endl;