Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add local caching to chunk allocator

Browse Source
This commit is contained in:
Matthew Parkinson
2021-10-20 16:55:08 +01:00
committed by Matthew Parkinson
parent 20a114cb62
commit 72ccb23d02
5 changed files with 174 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/mem/allocconfig.h
View File

@@ -68,8 +68,6 @@ namespace snmalloc
// Used to isolate values on cache lines to prevent false sharing.
static constexpr size_t CACHELINE_SIZE = 64;
static constexpr size_t PAGE_ALIGNED_SIZE = OS_PAGE_SIZE << INTERMEDIATE_BITS;
// Minimum allocation size is space for two pointers.
static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*));
static constexpr size_t MIN_ALLOC_SIZE = 2 * sizeof(void*);

166
src/mem/chunkallocator.h
View File

@@ -3,6 +3,7 @@
#include "../ds/mpmcstack.h"
#include "../mem/metaslab.h"
#include "../mem/sizeclasstable.h"
#include "../pal/pal_ds.h"
#ifdef SNMALLOC_TRACING
# include <iostream>
@@ -36,6 +37,35 @@ namespace snmalloc
static_assert(
sizeof(Metaslab) >= sizeof(ChunkRecord), "We conflate these two types.");
/**
* Number of free stacks per chunk size that each allocator will use.
* For performance ideally a power of 2. We will return to the central
* pool anything that has not be used in the last NUM_EPOCHS - 1, where
* each epoch is separated by DecayMemoryTimerObject::PERIOD.
* I.e. if period is 500ms and num of epochs is 4, then we will return to
* the central pool anything not used for the last 1500-2000ms.
*/
constexpr size_t NUM_EPOCHS = 4;
static_assert(bits::is_pow2(NUM_EPOCHS), "Code assumes power of two.");
class ChunkAllocatorLocalState
{
friend class ChunkAllocator;
/**
* Stack of slabs that have been returned for reuse.
*/
ModArray<
NUM_SLAB_SIZES,
ModArray<NUM_EPOCHS, MPMCStack<ChunkRecord, RequiresInit>>>
chunk_stack;
/**
* Used for list of all ChunkAllocatorLocalStates.
*/
std::atomic<ChunkAllocatorLocalState*> next{nullptr};
};
/**
* This is the global state required for the chunk allocator.
* It must be provided as a part of the shared state handle
@@ -47,7 +77,8 @@ namespace snmalloc
/**
* Stack of slabs that have been returned for reuse.
*/
ModArray<NUM_SLAB_SIZES, MPMCStack<ChunkRecord, RequiresInit>> chunk_stack;
ModArray<NUM_SLAB_SIZES, MPMCStack<ChunkRecord, RequiresInit>>
decommitted_chunk_stack;
/**
* All memory issued by this address space manager
@@ -56,6 +87,17 @@ namespace snmalloc
std::atomic<size_t> memory_in_stacks{0};
std::atomic<ChunkAllocatorLocalState*> all_local{nullptr};
/**
* Which is the current epoch to place dealloced chunks, and the
* first place we look for allocating chunks.
*/
std::atomic<size_t> epoch{0};
// Flag to ensure one-shot registration with the PAL for notifications.
std::atomic_flag register_decay{};
public:
size_t unused_memory()
{
@@ -78,10 +120,68 @@ namespace snmalloc
class ChunkAllocator
{
template<SNMALLOC_CONCEPT(ConceptPAL) Pal>
class DecayMemoryTimerObject : public PalTimerObject
{
ChunkAllocatorState* state;
/***
* Method for callback object to perform lazy decommit.
*/
static void process(PalTimerObject* p)
{
// Unsafe downcast here. Don't want vtable and RTTI.
auto self = reinterpret_cast<DecayMemoryTimerObject*>(p);
ChunkAllocator::handle_decay_tick(self->state);
}
// Specify that we notify the ChunkAllocator every 500ms.
static constexpr size_t PERIOD = 500;
public:
DecayMemoryTimerObject(ChunkAllocatorState* state)
: PalTimerObject(&process, PERIOD), state(state)
{}
};
static void handle_decay_tick(ChunkAllocatorState* state)
{
auto new_epoch = (state->epoch + 1) % NUM_EPOCHS;
// Flush old index for all threads.
ChunkAllocatorLocalState* curr = state->all_local;
while (curr != nullptr)
{
for (size_t sc = 0; sc < NUM_SLAB_SIZES; sc++)
{
auto& old_stack = curr->chunk_stack[sc][new_epoch];
ChunkRecord* record = old_stack.pop_all();
while (record != nullptr)
{
auto next = record->next.load();
// Disable pages for this
Pal::notify_not_using(
record->meta_common.chunk.unsafe_ptr(),
slab_sizeclass_to_size(sc));
// Add to global state
state->decommitted_chunk_stack[sc].push(record);
record = next;
}
}
curr = curr->next;
}
// Advance current index
state->epoch = new_epoch;
}
public:
template<SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle>
static std::pair<capptr::Chunk<void>, Metaslab*> alloc_chunk(
typename SharedStateHandle::LocalState& local_state,
ChunkAllocatorLocalState& chunk_alloc_local_state,
sizeclass_t sizeclass,
sizeclass_t slab_sizeclass, // TODO sizeclass_t
size_t slab_size,
@@ -96,8 +196,24 @@ namespace snmalloc
return {nullptr, nullptr};
}
// Pop a slab
auto chunk_record = state.chunk_stack[slab_sizeclass].pop();
ChunkRecord* chunk_record = nullptr;
// Try local cache of chunks first
for (size_t e = 0; e < NUM_EPOCHS && chunk_record == nullptr; e++)
{
chunk_record =
chunk_alloc_local_state
.chunk_stack[slab_sizeclass][(state.epoch - e) % NUM_EPOCHS]
.pop();
}
// Try global cache.
if (chunk_record == nullptr)
{
chunk_record = state.decommitted_chunk_stack[slab_sizeclass].pop();
if (chunk_record != nullptr)
Pal::notify_using<NoZero>(
chunk_record->meta_common.chunk.unsafe_ptr(), slab_size);
}
if (chunk_record != nullptr)
{
@@ -137,17 +253,21 @@ namespace snmalloc
template<SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle>
SNMALLOC_SLOW_PATH static void dealloc(
typename SharedStateHandle::LocalState& local_state,
ChunkAllocatorLocalState& chunk_alloc_local_state,
ChunkRecord* p,
size_t slab_sizeclass)
{
auto& state = SharedStateHandle::get_chunk_allocator_state(&local_state);
ChunkAllocatorState& state =
SharedStateHandle::get_chunk_allocator_state(&local_state);
#ifdef SNMALLOC_TRACING
std::cout << "Return slab:" << p->meta_common.chunk.unsafe_ptr()
<< " slab_sizeclass " << slab_sizeclass << " size "
<< slab_sizeclass_to_size(slab_sizeclass)
<< " memory in stacks " << state.memory_in_stacks << std::endl;
#endif
state.chunk_stack[slab_sizeclass].push(p);
chunk_alloc_local_state.chunk_stack[slab_sizeclass][state.epoch].push(p);
state.memory_in_stacks += slab_sizeclass_to_size(slab_sizeclass);
}
@@ -175,5 +295,41 @@ namespace snmalloc
return new (p.unsafe_ptr()) U(std::forward<Args>(args)...);
}
template<SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle>
static void register_local_state(
typename SharedStateHandle::LocalState& local_state,
ChunkAllocatorLocalState& chunk_alloc_local_state)
{
ChunkAllocatorState& state =
SharedStateHandle::get_chunk_allocator_state(&local_state);
// Register with the Pal to receive notifications.
if (!state.register_decay.test_and_set())
{
auto timer = alloc_meta_data<
DecayMemoryTimerObject<typename SharedStateHandle::Pal>,
SharedStateHandle>(&local_state, &state);
if (timer != nullptr)
{
SharedStateHandle::Pal::register_timer(timer);
}
else
{
// We failed to register the notification.
// This is not catarophic, but if we can't allocate this
// state something else will fail shortly.
state.register_decay.clear();
}
}
// Add to the list of local states.
auto* head = state.all_local.load();
do
{
chunk_alloc_local_state.next = head;
} while (!state.all_local.compare_exchange_strong(
head, &chunk_alloc_local_state));
}
};
} // namespace snmalloc

10
src/mem/corealloc.h
View File

@@ -58,6 +58,11 @@ namespace snmalloc
*/
MetaslabCache alloc_classes[NUM_SIZECLASSES];
/**
* Local cache for the Chunk allocator.
*/
ChunkAllocatorLocalState chunk_local_state;
/**
* Local entropy source and current version of keys for
* this thread
@@ -376,6 +381,7 @@ namespace snmalloc
auto chunk_record = clear_slab(meta, sizeclass);
ChunkAllocator::dealloc<SharedStateHandle>(
get_backend_local_state(),
chunk_local_state,
chunk_record,
sizeclass_to_slab_sizeclass(sizeclass));
@@ -548,6 +554,9 @@ namespace snmalloc
message_queue().invariant();
}
ChunkAllocator::register_local_state<SharedStateHandle>(
get_backend_local_state(), chunk_local_state);
#ifndef NDEBUG
for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++)
{
@@ -746,6 +755,7 @@ namespace snmalloc
auto [slab, meta] =
snmalloc::ChunkAllocator::alloc_chunk<SharedStateHandle>(
get_backend_local_state(),
chunk_local_state,
sizeclass,
slab_sizeclass,
slab_size,

2
src/mem/localalloc.h
View File

@@ -182,6 +182,7 @@ namespace snmalloc
// Set remote as large allocator remote.
auto [chunk, meta] = ChunkAllocator::alloc_chunk<SharedStateHandle>(
core_alloc->get_backend_local_state(),
core_alloc->chunk_local_state,
bits::next_pow2_bits(size), // TODO
large_size_to_chunk_sizeclass(size),
large_size_to_chunk_size(size),
@@ -547,6 +548,7 @@ namespace snmalloc
size_t slab_sizeclass) {
ChunkAllocator::dealloc<SharedStateHandle>(
core_alloc->get_backend_local_state(),
core_alloc->chunk_local_state,
slab_record,
slab_sizeclass);
return nullptr;

2
src/pal/pal_timer_default.h
View File

@@ -29,4 +29,4 @@ namespace snmalloc
timers.register_timer(timer);
}
};
}
} // namespace snmalloc