Major refactor of snmalloc (#343)
# Pagemap The Pagemap now stores all the meta-data for the object allocation. The meta-data in the pagemap is effectively a triple of the sizeclass, the remote allocator, and a pointer to a 64 byte block of meta-data for this chunk of memory. By storing the pointer to a block, it allows the pagemap to handle multiple slab sizes without branching on the fast path. There is one entry in the pagemap per 16KiB of address space, but by using the same entry in the pagemap for 4 adjacent entries, then we can treat a 64KiB range can be treated as a single slab of allocations. This change also means there is almost no capability amplification required by the implementation on CHERI for finding meta-data. The only amplification is required, when we change the way a chunk is used to a size of object allocation. # Backend There is a second major aspect of the refactor that there is now a narrow API that abstracts the Pagemap, PAL and address space management. This should better enable the compartmentalisation and makes it easier to produce alternative backends for various research directions. This is a template parameter that can be used to specialised by the front-end in different ways. # Thread local state The thread local state has been refactored into two components, one (called 'localalloc') that is stored directly in the TLS and is constant initialised, and one that is allocated in the address space (called 'coreallloc') which is lazily created and pooled. # Difference This removes Superslabs/Medium slabs as there meta-data is now part of the pagemap.
This commit is contained in:
Matthew Parkinson
committed by
GitHub
GitHub
parent
18d7cc99b6
commit
f0e2ab702a
@@ -3,9 +3,10 @@
|
||||
#include "../ds/mpscq.h"
|
||||
#include "../mem/allocconfig.h"
|
||||
#include "../mem/freelist.h"
|
||||
#include "../mem/sizeclass.h"
|
||||
#include "../mem/superslab.h"
|
||||
#include "../mem/metaslab.h"
|
||||
#include "../mem/sizeclasstable.h"
|
||||
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
|
||||
#ifdef CHECK_CLIENT
|
||||
@@ -28,75 +29,7 @@ namespace snmalloc
|
||||
AtomicCapPtr<Remote, CBAlloc> next{nullptr};
|
||||
};
|
||||
|
||||
#ifdef SNMALLOC_DONT_CACHE_ALLOCATOR_PTR
|
||||
/**
|
||||
* Cache the size class of the object to improve performance.
|
||||
*
|
||||
* This implementation does not cache the allocator id due to security
|
||||
* concerns. Alternative implementations may store the allocator
|
||||
* id, so that amplification costs can be mitigated on CHERI with MTE.
|
||||
*/
|
||||
sizeclass_t sizeclasscache;
|
||||
#else
|
||||
/* This implementation assumes that storing the allocator ID in a freed
|
||||
* object is not a security concern. Either we trust the code running on
|
||||
* top of the allocator, or additional security measure are in place such
|
||||
* as MTE + CHERI.
|
||||
*
|
||||
* We embed the size class in the bottom 8 bits of an allocator ID (i.e.,
|
||||
* the address of an Alloc's remote_alloc's message_queue; in practice we
|
||||
* only need 7 bits, but using 8 is conjectured to be faster). The hashing
|
||||
* algorithm of the Alloc's RemoteCache already ignores the bottom
|
||||
* "initial_shift" bits, which is, in practice, well above 8. There's a
|
||||
* static_assert() over there that helps ensure this stays true.
|
||||
*
|
||||
* This does mean that we might have message_queues that always collide in
|
||||
* the hash algorithm, if they're within "initial_shift" of each other. Such
|
||||
* pairings will substantially decrease performance and so we prohibit them
|
||||
* and use SNMALLOC_ASSERT to verify that they do not exist in debug builds.
|
||||
*/
|
||||
alloc_id_t alloc_id_and_sizeclass;
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Set up a remote object. Potentially cache sizeclass and allocator id.
|
||||
*/
|
||||
void set_info(alloc_id_t id, sizeclass_t sc)
|
||||
{
|
||||
#ifdef SNMALLOC_DONT_CACHE_ALLOCATOR_PTR
|
||||
UNUSED(id);
|
||||
sizeclasscache = sc;
|
||||
#else
|
||||
alloc_id_and_sizeclass = (id & ~SIZECLASS_MASK) | sc;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Return allocator for this object. This may perform amplification.
|
||||
*/
|
||||
template<typename LargeAlloc>
|
||||
static alloc_id_t
|
||||
trunc_target_id(CapPtr<Remote, CBAlloc> r, LargeAlloc* large_allocator)
|
||||
{
|
||||
#ifdef SNMALLOC_DONT_CACHE_ALLOCATOR_PTR
|
||||
// Rederive allocator id.
|
||||
auto r_auth = large_allocator->template capptr_amplify<Remote>(r);
|
||||
auto super = Superslab::get(r_auth);
|
||||
return super->get_allocator()->trunc_id();
|
||||
#else
|
||||
UNUSED(large_allocator);
|
||||
return r->alloc_id_and_sizeclass & ~SIZECLASS_MASK;
|
||||
#endif
|
||||
}
|
||||
|
||||
sizeclass_t sizeclass()
|
||||
{
|
||||
#ifdef SNMALLOC_DONT_CACHE_ALLOCATOR_PTR
|
||||
return sizeclasscache;
|
||||
#else
|
||||
return alloc_id_and_sizeclass & SIZECLASS_MASK;
|
||||
#endif
|
||||
}
|
||||
constexpr Remote() : next(nullptr) {}
|
||||
|
||||
/** Zero out a Remote tracking structure, return pointer to object base */
|
||||
template<capptr_bounds B>
|
||||
@@ -111,13 +44,18 @@ namespace snmalloc
|
||||
sizeof(Remote) <= MIN_ALLOC_SIZE,
|
||||
"Needs to be able to fit in smallest allocation.");
|
||||
|
||||
struct RemoteAllocator
|
||||
// Remotes need to be aligned enough that all the
|
||||
// small size classes can fit in the bottom bits.
|
||||
static constexpr size_t REMOTE_MIN_ALIGN = bits::min<size_t>(
|
||||
CACHELINE_SIZE, bits::next_pow2_const(NUM_SIZECLASSES + 1));
|
||||
|
||||
struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator
|
||||
{
|
||||
using alloc_id_t = Remote::alloc_id_t;
|
||||
// Store the message queue on a separate cacheline. It is mutable data that
|
||||
// is read by other threads.
|
||||
alignas(CACHELINE_SIZE)
|
||||
MPSCQ<Remote, CapPtrCBAlloc, AtomicCapPtrCBAlloc> message_queue;
|
||||
|
||||
MPSCQ<Remote, CapPtrCBAlloc, AtomicCapPtrCBAlloc> message_queue;
|
||||
|
||||
alloc_id_t trunc_id()
|
||||
{
|
||||
@@ -126,138 +64,4 @@ namespace snmalloc
|
||||
~SIZECLASS_MASK;
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
* A singly-linked list of Remote objects, supporting append and
|
||||
* take-all operations. Intended only for the private use of this
|
||||
* allocator; the Remote objects here will later be taken and pushed
|
||||
* to the inter-thread message queues.
|
||||
*/
|
||||
struct RemoteList
|
||||
{
|
||||
/*
|
||||
* A stub Remote object that will always be the head of this list;
|
||||
* never taken for further processing.
|
||||
*/
|
||||
Remote head{};
|
||||
|
||||
CapPtr<Remote, CBAlloc> last{&head};
|
||||
|
||||
void clear()
|
||||
{
|
||||
last = CapPtr<Remote, CBAlloc>(&head);
|
||||
}
|
||||
|
||||
bool empty()
|
||||
{
|
||||
return address_cast(last) == address_cast(&head);
|
||||
}
|
||||
};
|
||||
|
||||
struct RemoteCache
|
||||
{
|
||||
/**
|
||||
* The total amount of memory we are waiting for before we will dispatch
|
||||
* to other allocators. Zero or negative mean we should dispatch on the
|
||||
* next remote deallocation. This is initialised to the 0 so that we
|
||||
* always hit a slow path to start with, when we hit the slow path and
|
||||
* need to dispatch everything, we can check if we are a real allocator
|
||||
* and lazily provide a real allocator.
|
||||
*/
|
||||
int64_t capacity{0};
|
||||
std::array<RemoteList, REMOTE_SLOTS> list{};
|
||||
|
||||
/// Used to find the index into the array of queues for remote
|
||||
/// deallocation
|
||||
/// r is used for which round of sending this is.
|
||||
template<typename Alloc>
|
||||
inline size_t get_slot(size_t id, size_t r)
|
||||
{
|
||||
constexpr size_t allocator_size = sizeof(Alloc);
|
||||
constexpr size_t initial_shift =
|
||||
bits::next_pow2_bits_const(allocator_size);
|
||||
static_assert(
|
||||
initial_shift >= 8,
|
||||
"Can't embed sizeclass_t into allocator ID low bits");
|
||||
SNMALLOC_ASSERT((initial_shift + (r * REMOTE_SLOT_BITS)) < 64);
|
||||
return (id >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK;
|
||||
}
|
||||
|
||||
template<typename Alloc>
|
||||
SNMALLOC_FAST_PATH void dealloc(
|
||||
Remote::alloc_id_t target_id,
|
||||
CapPtr<void, CBAlloc> p,
|
||||
sizeclass_t sizeclass)
|
||||
{
|
||||
this->capacity -= sizeclass_to_size(sizeclass);
|
||||
auto r = p.template as_reinterpret<Remote>();
|
||||
|
||||
r->set_info(target_id, sizeclass);
|
||||
|
||||
RemoteList* l = &list[get_slot<Alloc>(target_id, 0)];
|
||||
l->last->non_atomic_next = r;
|
||||
l->last = r;
|
||||
}
|
||||
|
||||
template<typename Alloc>
|
||||
void post(Alloc* allocator, Remote::alloc_id_t id)
|
||||
{
|
||||
// When the cache gets big, post lists to their target allocators.
|
||||
capacity = REMOTE_CACHE;
|
||||
|
||||
size_t post_round = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
auto my_slot = get_slot<Alloc>(id, post_round);
|
||||
|
||||
for (size_t i = 0; i < REMOTE_SLOTS; i++)
|
||||
{
|
||||
if (i == my_slot)
|
||||
continue;
|
||||
|
||||
RemoteList* l = &list[i];
|
||||
CapPtr<Remote, CBAlloc> first = l->head.non_atomic_next;
|
||||
|
||||
if (!l->empty())
|
||||
{
|
||||
// Send all slots to the target at the head of the list.
|
||||
auto first_auth =
|
||||
allocator->large_allocator.template capptr_amplify<Remote>(first);
|
||||
auto super = Superslab::get(first_auth);
|
||||
super->get_allocator()->message_queue.enqueue(first, l->last);
|
||||
l->clear();
|
||||
}
|
||||
}
|
||||
|
||||
RemoteList* resend = &list[my_slot];
|
||||
if (resend->empty())
|
||||
break;
|
||||
|
||||
// Entries could map back onto the "resend" list,
|
||||
// so take copy of the head, mark the last element,
|
||||
// and clear the original list.
|
||||
CapPtr<Remote, CBAlloc> r = resend->head.non_atomic_next;
|
||||
resend->last->non_atomic_next = nullptr;
|
||||
resend->clear();
|
||||
|
||||
post_round++;
|
||||
|
||||
while (r != nullptr)
|
||||
{
|
||||
// Use the next N bits to spread out remote deallocs in our own
|
||||
// slot.
|
||||
size_t slot = get_slot<Alloc>(
|
||||
Remote::trunc_target_id(r, &allocator->large_allocator),
|
||||
post_round);
|
||||
RemoteList* l = &list[slot];
|
||||
l->last->non_atomic_next = r;
|
||||
l->last = r;
|
||||
|
||||
r = r->non_atomic_next;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace snmalloc
|
||||
|
||||
Reference in New Issue
Block a user