Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Collect freelist things in a namespace

Browse Source 
Motivated by renaming `FreeObject::{Head,Queue,AtomicQueue}Ptr` to
`freelist::...Ptr`, in fact go further, moving `FreeObject` itself to
`freelist::Object` and `FreeListBuilder` to `freelist::Builder` and
`FreeListIter` to `freelist::Iter`
This commit is contained in:
Nathaniel Wesley Filardo
2021-10-08 16:45:00 +01:00
committed by Nathaniel Wesley Filardo
parent bc365e0abb
commit 96155db640
7 changed files with 637 additions and 617 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/mem/corealloc.h
View File

@@ -162,7 +162,7 @@ namespace snmalloc
// Manufacture an allocation to prime the queue
// Using an actual allocation removes a conditional from a critical path.
auto dummy = capptr::Alloc<void>(small_alloc_one(MIN_ALLOC_SIZE))
.template as_static<FreeObject::T<>>();
.template as_static<freelist::Object::T<>>();
if (dummy == nullptr)
{
error("Critical error: Out-of-memory during initialisation.");
@@ -180,12 +180,12 @@ namespace snmalloc
{
SNMALLOC_ASSERT(attached_cache != nullptr);
auto domesticate =
[this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
[this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(backend_state_ptr(), p);
};
// Use attached cache, and fill it if it is empty.
return attached_cache->template alloc<NoZero, SharedStateHandle>(
domesticate, size, [&](sizeclass_t sizeclass, FreeListIter<>* fl) {
domesticate, size, [&](sizeclass_t sizeclass, freelist::Iter<>* fl) {
return small_alloc<NoZero>(sizeclass, *fl);
});
}
@@ -250,7 +250,7 @@ namespace snmalloc
{
b.add(
// Here begins our treatment of the heap as containing Wild pointers
FreeObject::make<capptr::bounds::AllocWild>(
freelist::Object::make<capptr::bounds::AllocWild>(
capptr_to_user_address_control(curr_ptr.as_void())),
key,
entropy);
@@ -262,7 +262,7 @@ namespace snmalloc
{
b.add(
// Here begins our treatment of the heap as containing Wild pointers
FreeObject::make<capptr::bounds::AllocWild>(
freelist::Object::make<capptr::bounds::AllocWild>(
capptr_to_user_address_control(
Aal::capptr_bound<void, capptr::bounds::AllocFull>(
p.as_void(), rsize))),
@@ -277,12 +277,12 @@ namespace snmalloc
ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass)
{
auto& key = entropy.get_free_list_key();
FreeListIter<> fl;
freelist::Iter<> fl;
auto more = meta->free_queue.close(fl, key);
UNUSED(more);
auto local_state = backend_state_ptr();
auto domesticate =
[local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
[local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass);
@@ -415,7 +415,7 @@ namespace snmalloc
bool need_post = false;
auto local_state = backend_state_ptr();
auto domesticate =
[local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
[local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
for (size_t i = 0; i < REMOTE_BATCH; i++)
@@ -608,7 +608,7 @@ namespace snmalloc
Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)),
"Not deallocating start of an object");
auto cp = p.as_static<FreeObject::T<>>();
auto cp = p.as_static<freelist::Object::T<>>();
auto& key = entropy.get_free_list_key();
@@ -620,7 +620,7 @@ namespace snmalloc
template<ZeroMem zero_mem>
SNMALLOC_SLOW_PATH void*
small_alloc(sizeclass_t sizeclass, FreeListIter<>& fast_free_list)
small_alloc(sizeclass_t sizeclass, freelist::Iter<>& fast_free_list)
{
size_t rsize = sizeclass_to_size(sizeclass);
@@ -645,7 +645,7 @@ namespace snmalloc
alloc_classes[sizeclass].unused--;
auto domesticate = [this](
FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(backend_state_ptr(), p);
};
auto [p, still_active] = Metaslab::alloc_free_list(
@@ -682,7 +682,7 @@ namespace snmalloc
template<ZeroMem zero_mem>
SNMALLOC_SLOW_PATH void* small_alloc_slow(
sizeclass_t sizeclass, FreeListIter<>& fast_free_list, size_t rsize)
sizeclass_t sizeclass, freelist::Iter<>& fast_free_list, size_t rsize)
{
// No existing free list get a new slab.
size_t slab_size = sizeclass_to_slab_size(sizeclass);
@@ -713,7 +713,7 @@ namespace snmalloc
alloc_new_list(slab, meta, rsize, slab_size, entropy);
auto domesticate =
[this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
[this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(backend_state_ptr(), p);
};
auto [p, still_active] = Metaslab::alloc_free_list(
@@ -738,7 +738,7 @@ namespace snmalloc
SNMALLOC_ASSERT(attached_cache != nullptr);
auto local_state = backend_state_ptr();
auto domesticate =
[local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
[local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};

1162
src/mem/freelist.h
View File

File diff suppressed because it is too large Load Diff

6
src/mem/localalloc.h
View File

@@ -211,21 +211,21 @@ namespace snmalloc
SNMALLOC_FAST_PATH void* small_alloc(size_t size)
{
// SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES));
auto domesticate = [this](FreeObject::QueuePtr p)
auto domesticate = [this](freelist::QueuePtr p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(
core_alloc->backend_state_ptr(), p);
};
auto slowpath = [&](
sizeclass_t sizeclass,
FreeListIter<>* fl) SNMALLOC_FAST_PATH_LAMBDA {
freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA {
if (likely(core_alloc != nullptr))
{
return core_alloc->handle_message_queue(
[](
CoreAlloc* core_alloc,
sizeclass_t sizeclass,
FreeListIter<>* fl) {
freelist::Iter<>* fl) {
return core_alloc->template small_alloc<zero_mem>(sizeclass, *fl);
},
core_alloc,

10
src/mem/localcache.h
View File

@@ -13,7 +13,7 @@ namespace snmalloc
using Stats = AllocStats<NUM_SIZECLASSES, NUM_LARGE_CLASSES>;
inline static SNMALLOC_FAST_PATH void*
finish_alloc_no_zero(FreeObject::HeadPtr p, sizeclass_t sizeclass)
finish_alloc_no_zero(freelist::HeadPtr p, sizeclass_t sizeclass)
{
SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p)));
UNUSED(sizeclass);
@@ -25,14 +25,14 @@ namespace snmalloc
template<ZeroMem zero_mem, typename SharedStateHandle>
inline static SNMALLOC_FAST_PATH void*
finish_alloc(FreeObject::HeadPtr p, sizeclass_t sizeclass)
finish_alloc(freelist::HeadPtr p, sizeclass_t sizeclass)
{
auto r = finish_alloc_no_zero(p, sizeclass);
if constexpr (zero_mem == YesZero)
SharedStateHandle::Pal::zero(r, sizeclass_to_size(sizeclass));
// TODO: Should this be zeroing the FreeObject state, in the non-zeroing
// TODO: Should this be zeroing the free Object state, in the non-zeroing
// case?
return r;
@@ -46,7 +46,7 @@ namespace snmalloc
// Free list per small size class. These are used for
// allocation on the fast path. This part of the code is inspired by
// mimalloc.
FreeListIter<> small_fast_free_lists[NUM_SIZECLASSES] = {};
freelist::Iter<> small_fast_free_lists[NUM_SIZECLASSES] = {};
// This is the entropy for a particular thread.
LocalEntropy entropy;
@@ -80,7 +80,7 @@ namespace snmalloc
{
auto& key = entropy.get_free_list_key();
auto domesticate =
[local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
[local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};

8
src/mem/metaslab.h
View File

@@ -25,9 +25,9 @@ namespace snmalloc
* Data-structure for building the free list for this slab.
*/
#ifdef SNMALLOC_CHECK_CLIENT
FreeListBuilder<true> free_queue;
freelist::Builder<true> free_queue;
#else
FreeListBuilder<false> free_queue;
freelist::Builder<false> free_queue;
#endif
/**
@@ -153,11 +153,11 @@ namespace snmalloc
* available objects for this metaslab.
*/
template<typename Domesticator>
static SNMALLOC_FAST_PATH std::pair<FreeObject::HeadPtr, bool>
static SNMALLOC_FAST_PATH std::pair<freelist::HeadPtr, bool>
alloc_free_list(
Domesticator domesticate,
Metaslab* meta,
FreeListIter<>& fast_free_list,
freelist::Iter<>& fast_free_list,
LocalEntropy& entropy,
sizeclass_t sizeclass)
{

32
src/mem/remoteallocator.h
View File

@@ -35,10 +35,10 @@ namespace snmalloc
// Store the message queue on a separate cacheline. It is mutable data that
// is read by other threads.
alignas(CACHELINE_SIZE) FreeObject::AtomicQueuePtr back{nullptr};
alignas(CACHELINE_SIZE) freelist::AtomicQueuePtr back{nullptr};
// Store the two ends on different cache lines as access by different
// threads.
alignas(CACHELINE_SIZE) FreeObject::QueuePtr front{nullptr};
alignas(CACHELINE_SIZE) freelist::QueuePtr front{nullptr};
constexpr RemoteAllocator() = default;
@@ -48,17 +48,17 @@ namespace snmalloc
SNMALLOC_ASSERT(front != nullptr);
}
void init(FreeObject::HeadPtr stub)
void init(freelist::HeadPtr stub)
{
FreeObject::atomic_store_null(stub, key_global);
freelist::Object::atomic_store_null(stub, key_global);
front = capptr_rewild(stub);
back.store(front, std::memory_order_relaxed);
invariant();
}
FreeObject::QueuePtr destroy()
freelist::QueuePtr destroy()
{
FreeObject::QueuePtr fnt = front;
freelist::QueuePtr fnt = front;
back.store(nullptr, std::memory_order_relaxed);
front = nullptr;
return fnt;
@@ -66,7 +66,7 @@ namespace snmalloc
inline bool is_empty()
{
FreeObject::QueuePtr bk = back.load(std::memory_order_relaxed);
freelist::QueuePtr bk = back.load(std::memory_order_relaxed);
return bk == front;
}
@@ -77,22 +77,22 @@ namespace snmalloc
*/
template<typename Domesticator>
void enqueue(
FreeObject::HeadPtr first,
FreeObject::HeadPtr last,
freelist::HeadPtr first,
freelist::HeadPtr last,
const FreeListKey& key,
Domesticator domesticate)
{
invariant();
FreeObject::atomic_store_null(last, key);
freelist::Object::atomic_store_null(last, key);
// exchange needs to be a release, so nullptr in next is visible.
FreeObject::QueuePtr prev =
freelist::QueuePtr prev =
back.exchange(capptr_rewild(last), std::memory_order_release);
FreeObject::atomic_store_next(domesticate(prev), first, key);
freelist::Object::atomic_store_next(domesticate(prev), first, key);
}
FreeObject::QueuePtr peek()
freelist::QueuePtr peek()
{
return front;
}
@@ -101,12 +101,12 @@ namespace snmalloc
* Returns the front message, or null if not possible to return a message.
*/
template<typename Domesticator>
std::pair<FreeObject::HeadPtr, bool>
std::pair<freelist::HeadPtr, bool>
dequeue(const FreeListKey& key, Domesticator domesticate)
{
invariant();
FreeObject::HeadPtr first = domesticate(front);
FreeObject::HeadPtr next = first->atomic_read_next(key, domesticate);
freelist::HeadPtr first = domesticate(front);
freelist::HeadPtr next = first->atomic_read_next(key, domesticate);
if (next != nullptr)
{

8
src/mem/remotecache.h
View File

@@ -16,7 +16,7 @@ namespace snmalloc
*/
struct RemoteDeallocCache
{
std::array<FreeListBuilder<false, false>, REMOTE_SLOTS> list;
std::array<freelist::Builder<false, false>, REMOTE_SLOTS> list;
/**
* The total amount of memory we are waiting for before we will dispatch
@@ -70,7 +70,7 @@ namespace snmalloc
const FreeListKey& key)
{
SNMALLOC_ASSERT(initialised);
auto r = p.template as_reinterpret<FreeObject::T<>>();
auto r = p.template as_reinterpret<freelist::Object::T<>>();
list[get_slot<allocator_size>(target_id, 0)].add(r, key);
}
@@ -85,7 +85,7 @@ namespace snmalloc
size_t post_round = 0;
bool sent_something = false;
auto domesticate =
[local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
[local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
@@ -114,7 +114,7 @@ namespace snmalloc
// Entries could map back onto the "resend" list,
// so take copy of the head, mark the last element,
// and clear the original list.
FreeListIter<> resend;
freelist::Iter<> resend;
list[my_slot].close(resend, key);
post_round++;