Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Default template args for FreeObject & friends

Browse Source 
Now that explicit annotations have gotten us through the refactoring, it's time
for the scaffolding to disappear.  src/mem/freelist.h is left generic for any
future machinations, but `FreeObject::T<>`, the several `FreeObject::...Ptr<>`s,
`FreeListIter<>`, and `FreeListBuilder<>` are given default parameters and all
uses are shortened to use defaults where possible.
This commit is contained in:
Nathaniel Wesley Filardo
2021-10-08 13:57:48 +01:00
committed by Nathaniel Wesley Filardo
parent 7deb3b61da
commit bc365e0abb
7 changed files with 143 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
src/mem/corealloc.h
View File

@@ -161,9 +161,8 @@ namespace snmalloc
{
// Manufacture an allocation to prime the queue
// Using an actual allocation removes a conditional from a critical path.
auto dummy =
capptr::Alloc<void>(small_alloc_one(MIN_ALLOC_SIZE))
.template as_static<FreeObject::T<capptr::bounds::AllocWild>>();
auto dummy = capptr::Alloc<void>(small_alloc_one(MIN_ALLOC_SIZE))
.template as_static<FreeObject::T<>>();
if (dummy == nullptr)
{
error("Critical error: Out-of-memory during initialisation.");
@@ -181,18 +180,12 @@ namespace snmalloc
{
SNMALLOC_ASSERT(attached_cache != nullptr);
auto domesticate =
[this](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(
backend_state_ptr(), p);
};
[this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(backend_state_ptr(), p);
};
// Use attached cache, and fill it if it is empty.
return attached_cache->template alloc<NoZero, SharedStateHandle>(
domesticate,
size,
[&](
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>* fl) {
domesticate, size, [&](sizeclass_t sizeclass, FreeListIter<>* fl) {
return small_alloc<NoZero>(sizeclass, *fl);
});
}
@@ -284,15 +277,14 @@ namespace snmalloc
ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass)
{
auto& key = entropy.get_free_list_key();
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild> fl;
FreeListIter<> fl;
auto more = meta->free_queue.close(fl, key);
UNUSED(more);
auto local_state = backend_state_ptr();
auto domesticate =
[local_state](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
[local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass);
#ifdef SNMALLOC_CHECK_CLIENT
@@ -423,10 +415,9 @@ namespace snmalloc
bool need_post = false;
auto local_state = backend_state_ptr();
auto domesticate =
[local_state](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
[local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
for (size_t i = 0; i < REMOTE_BATCH; i++)
{
auto p = message_queue().peek();
@@ -617,7 +608,7 @@ namespace snmalloc
Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)),
"Not deallocating start of an object");
auto cp = p.as_static<FreeObject::T<capptr::bounds::AllocWild>>();
auto cp = p.as_static<FreeObject::T<>>();
auto& key = entropy.get_free_list_key();
@@ -628,10 +619,8 @@ namespace snmalloc
}
template<ZeroMem zero_mem>
SNMALLOC_SLOW_PATH void* small_alloc(
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>&
fast_free_list)
SNMALLOC_SLOW_PATH void*
small_alloc(sizeclass_t sizeclass, FreeListIter<>& fast_free_list)
{
size_t rsize = sizeclass_to_size(sizeclass);
@@ -655,12 +644,10 @@ namespace snmalloc
if (meta->needed() == 0)
alloc_classes[sizeclass].unused--;
auto domesticate =
[this](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(
backend_state_ptr(), p);
};
auto domesticate = [this](
FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(backend_state_ptr(), p);
};
auto [p, still_active] = Metaslab::alloc_free_list(
domesticate, meta, fast_free_list, entropy, sizeclass);
@@ -695,10 +682,7 @@ namespace snmalloc
template<ZeroMem zero_mem>
SNMALLOC_SLOW_PATH void* small_alloc_slow(
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>&
fast_free_list,
size_t rsize)
sizeclass_t sizeclass, FreeListIter<>& fast_free_list, size_t rsize)
{
// No existing free list get a new slab.
size_t slab_size = sizeclass_to_slab_size(sizeclass);
@@ -729,11 +713,9 @@ namespace snmalloc
alloc_new_list(slab, meta, rsize, slab_size, entropy);
auto domesticate =
[this](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(
backend_state_ptr(), p);
};
[this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(backend_state_ptr(), p);
};
auto [p, still_active] = Metaslab::alloc_free_list(
domesticate, meta, fast_free_list, entropy, sizeclass);
@@ -756,10 +738,9 @@ namespace snmalloc
SNMALLOC_ASSERT(attached_cache != nullptr);
auto local_state = backend_state_ptr();
auto domesticate =
[local_state](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
[local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
if (destroy_queue)
{

105
src/mem/freelist.h
View File

@@ -55,22 +55,31 @@ namespace snmalloc
class FreeObject
{
public:
template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
template<
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild>
class T;
/**
* This "inductive step" type -- a queue-annotated pointer to a FreeObject
* containing a queue-annotated pointer -- shows up all over the place.
* Give it a shorter name (FreeObject::QueuePtr<BQueue>) for convenience.
* Give it a shorter name (FreeObject::BQueuePtr<BQueue>) for convenience.
*/
template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
using QueuePtr = CapPtr<FreeObject::T<BQueue>, BQueue>;
using BQueuePtr = CapPtr<FreeObject::T<BQueue>, BQueue>;
/**
* As with QueuePtr, but atomic.
* In particular, external code almost always uses AllocWild as the queue
* annotation. Make their lives easier.
*/
using QueuePtr = BQueuePtr<capptr::bounds::AllocWild>;
/**
* As with BQueuePtr, but atomic.
*/
template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
using AtomicQueuePtr = AtomicCapPtr<FreeObject::T<BQueue>, BQueue>;
using BAtomicQueuePtr = AtomicCapPtr<FreeObject::T<BQueue>, BQueue>;
using AtomicQueuePtr = BAtomicQueuePtr<capptr::bounds::AllocWild>;
/**
* This is the "base case" of that induction. While we can't get rid of the
@@ -81,7 +90,9 @@ namespace snmalloc
template<
SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
using HeadPtr = CapPtr<FreeObject::T<BQueue>, BView>;
using BHeadPtr = CapPtr<FreeObject::T<BQueue>, BView>;
using HeadPtr = BHeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>;
/**
* As with HeadPtr, but atomic.
@@ -115,9 +126,9 @@ namespace snmalloc
union
{
QueuePtr<BQueue> next_object;
BQueuePtr<BQueue> next_object;
// TODO: Should really use C++20 atomic_ref rather than a union.
AtomicQueuePtr<BQueue> atomic_next_object;
BAtomicQueuePtr<BQueue> atomic_next_object;
};
#ifdef SNMALLOC_CHECK_CLIENT
// Encoded representation of a back pointer.
@@ -131,7 +142,7 @@ namespace snmalloc
SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue::
template with_wildness<capptr::dimension::Wildness::Tame>,
typename Domesticator>
HeadPtr<BView, BQueue>
BHeadPtr<BView, BQueue>
atomic_read_next(const FreeListKey& key, Domesticator domesticate)
{
auto n_wild = FreeObject::decode_next(
@@ -156,7 +167,7 @@ namespace snmalloc
SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue::
template with_wildness<capptr::dimension::Wildness::Tame>,
typename Domesticator>
HeadPtr<BView, BQueue>
BHeadPtr<BView, BQueue>
read_next(const FreeListKey& key, Domesticator domesticate)
{
return domesticate(FreeObject::decode_next(
@@ -179,7 +190,7 @@ namespace snmalloc
template<
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue,
SNMALLOC_CONCEPT(capptr::ConceptBound) BView>
static HeadPtr<BView, BQueue> make(CapPtr<void, BView> p)
static BHeadPtr<BView, BQueue> make(CapPtr<void, BView> p)
{
return p.template as_static<FreeObject::T<BQueue>>();
}
@@ -241,10 +252,10 @@ namespace snmalloc
template<
SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
inline static QueuePtr<BQueue> encode_next(
address_t curr, HeadPtr<BView, BQueue> next, const FreeListKey& key)
inline static BQueuePtr<BQueue> encode_next(
address_t curr, BHeadPtr<BView, BQueue> next, const FreeListKey& key)
{
return QueuePtr<BQueue>(code_next(curr, next.unsafe_ptr(), key));
return BQueuePtr<BQueue>(code_next(curr, next.unsafe_ptr(), key));
}
/**
@@ -264,10 +275,10 @@ namespace snmalloc
template<
SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
inline static HeadPtr<BView, BQueue> decode_next(
address_t curr, HeadPtr<BView, BQueue> next, const FreeListKey& key)
inline static BHeadPtr<BView, BQueue> decode_next(
address_t curr, BHeadPtr<BView, BQueue> next, const FreeListKey& key)
{
return HeadPtr<BView, BQueue>(code_next(curr, next.unsafe_ptr(), key));
return BHeadPtr<BView, BQueue>(code_next(curr, next.unsafe_ptr(), key));
}
template<
@@ -299,9 +310,9 @@ namespace snmalloc
template<
SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
static QueuePtr<BQueue>* store_next(
QueuePtr<BQueue>* curr,
HeadPtr<BView, BQueue> next,
static BQueuePtr<BQueue>* store_next(
BQueuePtr<BQueue>* curr,
BHeadPtr<BView, BQueue> next,
const FreeListKey& key)
{
assert_view_queue_bounds<BView, BQueue>();
@@ -317,9 +328,9 @@ namespace snmalloc
}
template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
static void store_null(QueuePtr<BQueue>* curr, const FreeListKey& key)
static void store_null(BQueuePtr<BQueue>* curr, const FreeListKey& key)
{
*curr = encode_next(address_cast(curr), QueuePtr<BQueue>(nullptr), key);
*curr = encode_next(address_cast(curr), BQueuePtr<BQueue>(nullptr), key);
}
/**
@@ -331,8 +342,8 @@ namespace snmalloc
SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
static void atomic_store_next(
HeadPtr<BView, BQueue> curr,
HeadPtr<BView, BQueue> next,
BHeadPtr<BView, BQueue> curr,
BHeadPtr<BView, BQueue> next,
const FreeListKey& key)
{
static_assert(BView::wildness == capptr::dimension::Wildness::Tame);
@@ -354,13 +365,13 @@ namespace snmalloc
SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
static void
atomic_store_null(HeadPtr<BView, BQueue> curr, const FreeListKey& key)
atomic_store_null(BHeadPtr<BView, BQueue> curr, const FreeListKey& key)
{
static_assert(BView::wildness == capptr::dimension::Wildness::Tame);
curr->atomic_next_object.store(
encode_next(
address_cast(&curr->next_object), QueuePtr<BQueue>(nullptr), key),
address_cast(&curr->next_object), BQueuePtr<BQueue>(nullptr), key),
std::memory_order_relaxed);
}
};
@@ -375,18 +386,18 @@ namespace snmalloc
* Checks signing of pointers
*/
template<
SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild>
class FreeListIter
{
FreeObject::HeadPtr<BView, BQueue> curr{nullptr};
FreeObject::BHeadPtr<BView, BQueue> curr{nullptr};
#ifdef SNMALLOC_CHECK_CLIENT
address_t prev{0};
#endif
public:
constexpr FreeListIter(
FreeObject::HeadPtr<BView, BQueue> head, address_t prev_value)
FreeObject::BHeadPtr<BView, BQueue> head, address_t prev_value)
: curr(head)
{
#ifdef SNMALLOC_CHECK_CLIENT
@@ -408,7 +419,7 @@ namespace snmalloc
/**
* Returns current head without affecting the iterator.
*/
FreeObject::HeadPtr<BView, BQueue> peek()
FreeObject::BHeadPtr<BView, BQueue> peek()
{
return curr;
}
@@ -417,7 +428,7 @@ namespace snmalloc
* Moves the iterator on, and returns the current value.
*/
template<typename Domesticator>
FreeObject::HeadPtr<BView, BQueue>
FreeObject::BHeadPtr<BView, BQueue>
take(const FreeListKey& key, Domesticator domesticate)
{
auto c = curr;
@@ -455,9 +466,9 @@ namespace snmalloc
*/
template<
bool RANDOM,
SNMALLOC_CONCEPT(capptr::ConceptBound) BView,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue,
bool INIT = true>
bool INIT = true,
SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild>
class FreeListBuilder
{
static constexpr size_t LENGTH = RANDOM ? 2 : 1;
@@ -466,7 +477,7 @@ namespace snmalloc
* We use native pointers below so that we don't run afoul of strict
* aliasing rules. head is a FreeObject::HeadPtr<BView, BQueue> -- that
* is, a known-domesticated pointer to a queue of wild pointers -- and
* it's usually the case that end is a FreeObject::QueuePtr<BQueue>* --
* it's usually the case that end is a FreeObject::BQueuePtr<BQueue>* --
* that is, a known-domesticated pointer to a wild pointer to a queue of
* wild pointers. However, in order to do branchless inserts, we set end
* = &head, which breaks strict aliasing rules with the types as given.
@@ -481,19 +492,19 @@ namespace snmalloc
// This enables branch free enqueuing.
std::array<void**, LENGTH> end{nullptr};
FreeObject::QueuePtr<BQueue>* cast_end(uint32_t ix)
FreeObject::BQueuePtr<BQueue>* cast_end(uint32_t ix)
{
return reinterpret_cast<FreeObject::QueuePtr<BQueue>*>(end[ix]);
return reinterpret_cast<FreeObject::BQueuePtr<BQueue>*>(end[ix]);
}
void set_end(uint32_t ix, FreeObject::QueuePtr<BQueue>* p)
void set_end(uint32_t ix, FreeObject::BQueuePtr<BQueue>* p)
{
end[ix] = reinterpret_cast<void**>(p);
}
FreeObject::HeadPtr<BView, BQueue> cast_head(uint32_t ix)
FreeObject::BHeadPtr<BView, BQueue> cast_head(uint32_t ix)
{
return FreeObject::HeadPtr<BView, BQueue>(
return FreeObject::BHeadPtr<BView, BQueue>(
static_cast<FreeObject::T<BQueue>*>(head[ix]));
}
@@ -527,7 +538,7 @@ namespace snmalloc
* Adds an element to the builder
*/
void add(
FreeObject::HeadPtr<BView, BQueue> n,
FreeObject::BHeadPtr<BView, BQueue> n,
const FreeListKey& key,
LocalEntropy& entropy)
{
@@ -554,7 +565,7 @@ namespace snmalloc
*/
template<bool RANDOM_ = RANDOM>
std::enable_if_t<!RANDOM_>
add(FreeObject::HeadPtr<BView, BQueue> n, const FreeListKey& key)
add(FreeObject::BHeadPtr<BView, BQueue> n, const FreeListKey& key)
{
static_assert(RANDOM_ == RANDOM, "Don't set template parameter");
set_end(0, FreeObject::store_next(cast_end(0), n, key));
@@ -578,7 +589,7 @@ namespace snmalloc
* and is thus subject to encoding if the next_object pointers
* encoded.
*/
FreeObject::HeadPtr<BView, BQueue>
FreeObject::BHeadPtr<BView, BQueue>
read_head(uint32_t index, const FreeListKey& key)
{
return FreeObject::decode_next(
@@ -652,8 +663,8 @@ namespace snmalloc
std::enable_if_t<
!RANDOM_,
std::pair<
FreeObject::HeadPtr<BView, BQueue>,
FreeObject::HeadPtr<BView, BQueue>>>
FreeObject::BHeadPtr<BView, BQueue>,
FreeObject::BHeadPtr<BView, BQueue>>>
extract_segment(const FreeListKey& key)
{
static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!");
@@ -664,7 +675,7 @@ namespace snmalloc
// this is doing a CONTAINING_RECORD like cast to get back
// to the actual object. This isn't true if the builder is
// empty, but you are not allowed to call this in the empty case.
auto last = FreeObject::HeadPtr<BView, BQueue>(
auto last = FreeObject::BHeadPtr<BView, BQueue>(
FreeObject::from_next_ptr(cast_end(0)));
init();
return {first, last};

57
src/mem/localalloc.h
View File

@@ -211,38 +211,33 @@ namespace snmalloc
SNMALLOC_FAST_PATH void* small_alloc(size_t size)
{
// SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES));
auto domesticate =
[this](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(
core_alloc->backend_state_ptr(), p);
};
auto slowpath =
[&](
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>*
fl) SNMALLOC_FAST_PATH_LAMBDA {
if (likely(core_alloc != nullptr))
{
return core_alloc->handle_message_queue(
[](
CoreAlloc* core_alloc,
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>*
fl) {
return core_alloc->template small_alloc<zero_mem>(
sizeclass, *fl);
},
core_alloc,
sizeclass,
fl);
}
return lazy_init(
[&](CoreAlloc*, sizeclass_t sizeclass) {
return small_alloc<zero_mem>(sizeclass_to_size(sizeclass));
auto domesticate = [this](FreeObject::QueuePtr p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(
core_alloc->backend_state_ptr(), p);
};
auto slowpath = [&](
sizeclass_t sizeclass,
FreeListIter<>* fl) SNMALLOC_FAST_PATH_LAMBDA {
if (likely(core_alloc != nullptr))
{
return core_alloc->handle_message_queue(
[](
CoreAlloc* core_alloc,
sizeclass_t sizeclass,
FreeListIter<>* fl) {
return core_alloc->template small_alloc<zero_mem>(sizeclass, *fl);
},
sizeclass);
};
core_alloc,
sizeclass,
fl);
}
return lazy_init(
[&](CoreAlloc*, sizeclass_t sizeclass) {
return small_alloc<zero_mem>(sizeclass_to_size(sizeclass));
},
sizeclass);
};
return local_cache.template alloc<zero_mem, SharedStateHandle>(
domesticate, size, slowpath);

24
src/mem/localcache.h
View File

@@ -12,9 +12,8 @@ namespace snmalloc
{
using Stats = AllocStats<NUM_SIZECLASSES, NUM_LARGE_CLASSES>;
template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero(
FreeObject::HeadPtr<capptr::bounds::Alloc, BQueue> p, sizeclass_t sizeclass)
inline static SNMALLOC_FAST_PATH void*
finish_alloc_no_zero(FreeObject::HeadPtr p, sizeclass_t sizeclass)
{
SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p)));
UNUSED(sizeclass);
@@ -24,12 +23,9 @@ namespace snmalloc
return r;
}
template<
ZeroMem zero_mem,
typename SharedStateHandle,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
inline static SNMALLOC_FAST_PATH void* finish_alloc(
FreeObject::HeadPtr<capptr::bounds::Alloc, BQueue> p, sizeclass_t sizeclass)
template<ZeroMem zero_mem, typename SharedStateHandle>
inline static SNMALLOC_FAST_PATH void*
finish_alloc(FreeObject::HeadPtr p, sizeclass_t sizeclass)
{
auto r = finish_alloc_no_zero(p, sizeclass);
@@ -50,8 +46,7 @@ namespace snmalloc
// Free list per small size class. These are used for
// allocation on the fast path. This part of the code is inspired by
// mimalloc.
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>
small_fast_free_lists[NUM_SIZECLASSES] = {};
FreeListIter<> small_fast_free_lists[NUM_SIZECLASSES] = {};
// This is the entropy for a particular thread.
LocalEntropy entropy;
@@ -85,10 +80,9 @@ namespace snmalloc
{
auto& key = entropy.get_free_list_key();
auto domesticate =
[local_state](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
[local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
for (size_t i = 0; i < NUM_SIZECLASSES; i++)
{

13
src/mem/metaslab.h
View File

@@ -25,11 +25,9 @@ namespace snmalloc
* Data-structure for building the free list for this slab.
*/
#ifdef SNMALLOC_CHECK_CLIENT
FreeListBuilder<true, capptr::bounds::Alloc, capptr::bounds::AllocWild>
free_queue;
FreeListBuilder<true> free_queue;
#else
FreeListBuilder<false, capptr::bounds::Alloc, capptr::bounds::AllocWild>
free_queue;
FreeListBuilder<false> free_queue;
#endif
/**
@@ -155,14 +153,11 @@ namespace snmalloc
* available objects for this metaslab.
*/
template<typename Domesticator>
static SNMALLOC_FAST_PATH std::pair<
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>,
bool>
static SNMALLOC_FAST_PATH std::pair<FreeObject::HeadPtr, bool>
alloc_free_list(
Domesticator domesticate,
Metaslab* meta,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>&
fast_free_list,
FreeListIter<>& fast_free_list,
LocalEntropy& entropy,
sizeclass_t sizeclass)
{

37
src/mem/remoteallocator.h
View File

@@ -35,12 +35,10 @@ namespace snmalloc
// Store the message queue on a separate cacheline. It is mutable data that
// is read by other threads.
alignas(CACHELINE_SIZE)
FreeObject::AtomicQueuePtr<capptr::bounds::AllocWild> back{nullptr};
alignas(CACHELINE_SIZE) FreeObject::AtomicQueuePtr back{nullptr};
// Store the two ends on different cache lines as access by different
// threads.
alignas(CACHELINE_SIZE)
FreeObject::QueuePtr<capptr::bounds::AllocWild> front{nullptr};
alignas(CACHELINE_SIZE) FreeObject::QueuePtr front{nullptr};
constexpr RemoteAllocator() = default;
@@ -50,9 +48,7 @@ namespace snmalloc
SNMALLOC_ASSERT(front != nullptr);
}
void
init(FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>
stub)
void init(FreeObject::HeadPtr stub)
{
FreeObject::atomic_store_null(stub, key_global);
front = capptr_rewild(stub);
@@ -60,9 +56,9 @@ namespace snmalloc
invariant();
}
FreeObject::QueuePtr<capptr::bounds::AllocWild> destroy()
FreeObject::QueuePtr destroy()
{
FreeObject::QueuePtr<capptr::bounds::AllocWild> fnt = front;
FreeObject::QueuePtr fnt = front;
back.store(nullptr, std::memory_order_relaxed);
front = nullptr;
return fnt;
@@ -70,8 +66,7 @@ namespace snmalloc
inline bool is_empty()
{
FreeObject::QueuePtr<capptr::bounds::AllocWild> bk =
back.load(std::memory_order_relaxed);
FreeObject::QueuePtr bk = back.load(std::memory_order_relaxed);
return bk == front;
}
@@ -82,10 +77,8 @@ namespace snmalloc
*/
template<typename Domesticator>
void enqueue(
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>
first,
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>
last,
FreeObject::HeadPtr first,
FreeObject::HeadPtr last,
const FreeListKey& key,
Domesticator domesticate)
{
@@ -93,13 +86,13 @@ namespace snmalloc
FreeObject::atomic_store_null(last, key);
// exchange needs to be a release, so nullptr in next is visible.
FreeObject::QueuePtr<capptr::bounds::AllocWild> prev =
FreeObject::QueuePtr prev =
back.exchange(capptr_rewild(last), std::memory_order_release);
FreeObject::atomic_store_next(domesticate(prev), first, key);
}
FreeObject::QueuePtr<capptr::bounds::AllocWild> peek()
FreeObject::QueuePtr peek()
{
return front;
}
@@ -108,16 +101,12 @@ namespace snmalloc
* Returns the front message, or null if not possible to return a message.
*/
template<typename Domesticator>
std::pair<
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>,
bool>
std::pair<FreeObject::HeadPtr, bool>
dequeue(const FreeListKey& key, Domesticator domesticate)
{
invariant();
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>
first = domesticate(front);
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>
next = first->atomic_read_next(key, domesticate);
FreeObject::HeadPtr first = domesticate(front);
FreeObject::HeadPtr next = first->atomic_read_next(key, domesticate);
if (next != nullptr)
{

21
src/mem/remotecache.h
View File

@@ -16,14 +16,7 @@ namespace snmalloc
*/
struct RemoteDeallocCache
{
std::array<
FreeListBuilder<
false,
capptr::bounds::Alloc,
capptr::bounds::AllocWild,
false>,
REMOTE_SLOTS>
list;
std::array<FreeListBuilder<false, false>, REMOTE_SLOTS> list;
/**
* The total amount of memory we are waiting for before we will dispatch
@@ -77,8 +70,7 @@ namespace snmalloc
const FreeListKey& key)
{
SNMALLOC_ASSERT(initialised);
auto r =
p.template as_reinterpret<FreeObject::T<capptr::bounds::AllocWild>>();
auto r = p.template as_reinterpret<FreeObject::T<>>();
list[get_slot<allocator_size>(target_id, 0)].add(r, key);
}
@@ -93,10 +85,9 @@ namespace snmalloc
size_t post_round = 0;
bool sent_something = false;
auto domesticate =
[local_state](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
[local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
while (true)
{
@@ -123,7 +114,7 @@ namespace snmalloc
// Entries could map back onto the "resend" list,
// so take copy of the head, mark the last element,
// and clear the original list.
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild> resend;
FreeListIter<> resend;
list[my_slot].close(resend, key);
post_round++;