Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make Lazy Decomit asynchronous

Browse Source 
On platforms that support low-memory notifications register callbacks
that perform lazy decommit. This allows idle processes to return memory
to the OS. Without incurring the cost of constantly committing and
decommitting memory.

Code review and CI changes

* Fixed test to use a template to make constexpr magic work
* Factored out basic notification mechanism so can be reused on other
platforms.
This commit is contained in:
Matthew Parkinson
2020-02-21 11:34:39 +00:00
parent 8723ae2443
commit 813367286e
4 changed files with 161 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
src/mem/largealloc.h
View File

@@ -55,7 +55,7 @@ namespace snmalloc
// global state of the allocator. This is currently stored in the memory
// provider, so we add this in.
template<class PAL>
class MemoryProviderStateMixin : public PAL
class MemoryProviderStateMixin : public PalNotificationObject, public PAL
{
/**
* Flag to protect the bump allocator
@@ -72,11 +72,6 @@ namespace snmalloc
**/
size_t remaining = 0;
/**
* The last time we saw a low memory notification.
*/
std::atomic<uint64_t> last_low_memory_epoch = 0;
/**
* Simple flag for checking if another instance of lazy-decommit is
* running
@@ -105,6 +100,13 @@ namespace snmalloc
// memcpy is safe as this is entirely single threaded.
memcpy(allocated, &local, sizeof(MemoryProviderStateMixin<PAL>));
// Register this allocator for low-memory call-backs
if constexpr (pal_supports<LowMemoryNotification, PAL>)
{
allocated->PalNotificationObject::pal_notify = &(allocated->process);
PAL::register_for_low_memory_callback(allocated);
}
return allocated;
}
@@ -183,6 +185,16 @@ namespace snmalloc
large_stack[large_class].push(reinterpret_cast<Largeslab*>(p));
}
/***
* Method for callback object to perform lazy decommit.
**/
static void process(PalNotificationObject* p)
{
// Unsafe downcast here. Don't want vtable and RTTI.
auto self = reinterpret_cast<MemoryProviderStateMixin<PAL>*>(p);
self->lazy_decommit();
}
public:
/**
* Primitive allocator for structure that are required before
@@ -235,24 +247,6 @@ namespace snmalloc
return new (p) T(std::forward<Args...>(args)...);
}
/**
* Returns the number of low memory notifications that have been received
* (over the lifetime of this process). If the underlying system does not
* support low memory notifications, this will return 0.
*/
SNMALLOC_FAST_PATH
uint64_t low_memory_epoch()
{
if constexpr (pal_supports<LowMemoryNotification, PAL>)
{
return PAL::low_memory_epoch();
}
else
{
return 0;
}
}
template<bool committed>
void* reserve(size_t large_class) noexcept
{
@@ -316,41 +310,6 @@ namespace snmalloc
return result;
}
}
SNMALLOC_FAST_PATH void lazy_decommit_if_needed()
{
#ifdef TEST_LAZY_DECOMMIT
static_assert(
TEST_LAZY_DECOMMIT > 0,
"TEST_LAZY_DECOMMIT must be a positive integer value.");
static std::atomic<uint64_t> counter;
auto c = counter++;
if (c % TEST_LAZY_DECOMMIT == 0)
{
lazy_decommit();
}
#else
if constexpr (decommit_strategy == DecommitSuperLazy)
{
auto new_epoch = low_memory_epoch();
auto old_epoch = last_low_memory_epoch.load(std::memory_order_acquire);
if (new_epoch > old_epoch)
{
// Try to update the epoch to the value that we've seen. If
// another thread has seen a newer epoch than us (or done the same
// update) let them win.
do
{
if (last_low_memory_epoch.compare_exchange_strong(
old_epoch, new_epoch))
{
lazy_decommit();
}
} while (old_epoch < new_epoch);
}
}
#endif
}
};
using Stats = AllocStats<NUM_SIZECLASSES, NUM_LARGE_CLASSES>;
@@ -381,7 +340,6 @@ namespace snmalloc
size = rsize;
void* p = memory_provider.large_stack[large_class].pop();
memory_provider.lazy_decommit_if_needed();
if (p == nullptr)
{
@@ -392,8 +350,7 @@ namespace snmalloc
{
stats.superslab_pop();
// Cross-reference alloc.h's large_dealloc decommitment condition
// and lazy_decommit_if_needed.
// Cross-reference alloc.h's large_dealloc decommitment condition.
bool decommitted =
((decommit_strategy == DecommitSuperLazy) &&
(static_cast<Baseslab*>(p)->get_kind() == Decommitted)) ||
@@ -449,7 +406,6 @@ namespace snmalloc
stats.superslab_push();
memory_provider.large_stack[large_class].push(static_cast<Largeslab*>(p));
memory_provider.lazy_decommit_if_needed();
}
};

63
src/pal/pal_consts.h
View File

@@ -10,8 +10,8 @@ namespace snmalloc
{
/**
* This PAL supports low memory notifications. It must implement a
* `low_memory_epoch` method that returns a `uint64_t` of the number of
* times that a low-memory notification has been raised and an
* `register_for_low_memory_callback` method that allows callbacks to be
* registered when the platform detects low-memory and an
* `expensive_low_memory_check()` method that returns a `bool` indicating
* whether low memory conditions are still in effect.
*/
@@ -53,4 +53,63 @@ namespace snmalloc
* Default Tag ID for the Apple class
*/
static const int PALAnonDefaultID = 241;
/**
* This struct is used to represent callbacks for notification from the
* platform. It contains a next pointer as client is responsible for
* allocation as we cannot assume an allocator at this point.
**/
struct PalNotificationObject
{
std::atomic<PalNotificationObject*> pal_next;
void (*pal_notify)(PalNotificationObject* self);
};
/***
* Wrapper for managing notifications for PAL events
**/
class PalNotifier
{
/**
* List of callbacks to notify
**/
std::atomic<PalNotificationObject*> callbacks = nullptr;
public:
/**
* Register a callback object to be notified
*
* The object should never be deallocated by the client after calling
* this.
**/
void register_notification(PalNotificationObject* callback)
{
callback->pal_next = nullptr;
auto prev = &callbacks;
auto curr = prev->load();
do
{
while (curr != nullptr)
{
prev = &(curr->pal_next);
curr = prev->load();
}
} while (!prev->compare_exchange_weak(curr, callback));
}
/**
* Calls the pal_notify of all the registered objects.
**/
void notify_all()
{
PalNotificationObject* curr = callbacks;
while (curr != nullptr)
{
curr->pal_notify(curr);
curr = curr->pal_next;
}
}
};
} // namespace snmalloc

36
src/pal/pal_windows.h
View File

@@ -20,23 +20,25 @@ namespace snmalloc
{
class PALWindows
{
/**
* The number of times that the memory pressure notification has fired.
*/
static inline std::atomic<uint64_t> pressure_epoch;
/**
* A flag indicating that we have tried to register for low-memory
* notifications.
*/
static inline std::atomic<bool> registered_for_notifications;
static inline HANDLE lowMemoryObject;
/**
* List of callbacks for low-memory notification
**/
static inline PalNotifier low_memory_callbacks;
/**
* Callback, used when the system delivers a low-memory notification. This
* simply increments an atomic counter each time the notification is raised.
* calls all the handlers registered with the PAL.
*/
static void CALLBACK low_memory(_In_ PVOID, _In_ BOOLEAN)
{
pressure_epoch++;
low_memory_callbacks.notify_all();
}
public:
@@ -76,17 +78,6 @@ namespace snmalloc
# endif
;
/**
* Counter values for the number of times that a low-pressure notification
* has been delivered. Callers should compare this with a previous value
* to see if the low memory state has been triggered since they last
* checked.
*/
uint64_t low_memory_epoch()
{
return pressure_epoch.load(std::memory_order_acquire);
}
/**
* Check whether the low memory state is still in effect. This is an
* expensive operation and should not be on any fast paths.
@@ -98,6 +89,17 @@ namespace snmalloc
return result;
}
/**
* Register callback object for low-memory notifications.
* Client is responsible for allocation, and ensuring the object is live
* for the duration of the program.
**/
static void
register_for_low_memory_callback(PalNotificationObject* callback)
{
low_memory_callbacks.register_notification(callback);
}
static void error(const char* const str)
{
puts(str);

78
src/test/perf/low_memory/low-memory.cc
View File

@@ -1,6 +1,7 @@
#include <iostream>
#include <snmalloc.h>
#include <test/measuretime.h>
#include <test/opt.h>
#include <test/setup.h>
#include <unordered_set>
#include <vector>
@@ -37,29 +38,34 @@ public:
tail = tail->next;
}
void try_remove()
bool try_remove()
{
if (head->next == nullptr)
return;
return false;
Node* next = head->next;
ThreadAlloc::get()->dealloc(head);
head = next;
return true;
}
};
std::atomic<uint64_t> global_epoch = 0;
void advance(PalNotificationObject* unused)
{
UNUSED(unused);
global_epoch++;
}
PalNotificationObject update_epoch = {nullptr, &advance};
bool has_pressure()
{
static thread_local uint64_t epoch;
static thread_local uint64_t epoch = 0;
if constexpr (!pal_supports<LowMemoryNotification, GlobalVirtual>)
{
return false;
}
uint64_t current_epoch = default_memory_provider().low_memory_epoch();
bool result = epoch != current_epoch;
epoch = current_epoch;
bool result = epoch != global_epoch;
epoch = global_epoch;
return result;
}
@@ -79,7 +85,7 @@ void reach_pressure(Queue& allocations)
void reduce_pressure(Queue& allocations)
{
size_t size = 4096;
for (size_t n = 0; n < 1000; n++)
for (size_t n = 0; n < 10000; n++)
{
allocations.try_remove();
allocations.try_remove();
@@ -87,21 +93,51 @@ void reduce_pressure(Queue& allocations)
}
}
int main(int, char**)
/**
* Wrapper to handle Pals that don't have the method.
* Template parameter required to handle `if constexpr` always evaluating both
* sides.
**/
template<typename PAL>
void register_for_pal_notifications()
{
#ifndef NDEBUG
Queue allocations;
PAL::register_for_low_memory_callback(&update_epoch);
}
if constexpr (!pal_supports<LowMemoryNotification, GlobalVirtual>)
int main(int argc, char** argv)
{
opt::Opt opt(argc, argv);
if constexpr (pal_supports<LowMemoryNotification, GlobalVirtual>)
{
register_for_pal_notifications<GlobalVirtual>();
}
else
{
std::cout << "Pal does not support low-memory notification! Test not run"
<< std::endl;
return 0;
}
#ifdef NDEBUG
# if defined(WIN32) && !defined(SNMALLOC_VA_BITS_64)
std::cout << "32-bit windows not supported for this test." << std::endl;
# else
bool interactive = opt.has("--interactive");
Queue allocations;
std::cout
<< "Expected use:" << std::endl
<< " run first instances with --interactive. Wait for first to print "
<< std::endl
<< " 'No allocations left. Press any key to terminate'" << std::endl
<< "watch working set, and start second instance working set of first "
<< "should drop to almost zero," << std::endl
<< "and second should climb to physical ram." << std::endl
<< std::endl;
setup();
for (size_t i = 0; i < 10; i++)
@@ -111,6 +147,16 @@ int main(int, char**)
reduce_pressure(allocations);
}
// Deallocate everything
while (allocations.try_remove())
;
if (interactive)
{
std::cout << "No allocations left. Press any key to terminate" << std::endl;
getchar();
}
# endif
#else
std::cout << "Release test only." << std::endl;