Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remote queues hold Wild pointers

Browse Source
This commit is contained in:
Nathaniel Wesley Filardo
2021-09-28 15:47:15 +01:00
committed by Nathaniel Wesley Filardo
parent 94ab856ff5
commit 501c14661f
3 changed files with 66 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/mem/corealloc.h
View File

@@ -163,7 +163,7 @@ namespace snmalloc
// Using an actual allocation removes a conditional from a critical path.
auto dummy =
capptr::Alloc<void>(small_alloc_one(MIN_ALLOC_SIZE))
.template as_static<FreeObject::T<capptr::bounds::Alloc>>();
.template as_static<FreeObject::T<capptr::bounds::AllocWild>>();
if (dummy == nullptr)
{
error("Critical error: Out-of-memory during initialisation.");
@@ -421,13 +421,17 @@ namespace snmalloc
handle_message_queue_inner(Action action, Args... args)
{
bool need_post = false;
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
auto local_state = backend_state_ptr();
auto domesticate =
[local_state](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
for (size_t i = 0; i < REMOTE_BATCH; i++)
{
auto p = message_queue().peek();
auto& entry = SharedStateHandle::Pagemap::get_metaentry(
backend_state_ptr(), snmalloc::address_cast(p));
local_state, snmalloc::address_cast(p));
auto r = message_queue().dequeue(key_global, domesticate);
@@ -750,24 +754,26 @@ namespace snmalloc
bool flush(bool destroy_queue = false)
{
SNMALLOC_ASSERT(attached_cache != nullptr);
// TODO: Placeholder
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
auto local_state = backend_state_ptr();
auto domesticate =
[local_state](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
if (destroy_queue)
{
auto p = message_queue().destroy();
auto p_wild = message_queue().destroy();
auto p_tame = domesticate(p_wild);
while (p != nullptr)
while (p_tame != nullptr)
{
bool need_post = true; // Always going to post, so ignore.
auto n = p->atomic_read_next(key_global, domesticate);
auto n_tame = p_tame->atomic_read_next(key_global, domesticate);
auto& entry = SharedStateHandle::Pagemap::get_metaentry(
backend_state_ptr(), snmalloc::address_cast(p));
handle_dealloc_remote(entry, p.as_void(), need_post);
// TODO p = SharedStateHandle::capptr_domesticate(backend_state_ptr(),
// n);
p = n;
backend_state_ptr(), snmalloc::address_cast(p_tame));
handle_dealloc_remote(entry, p_tame.as_void(), need_post);
p_tame = n_tame;
}
}
else

59
src/mem/remoteallocator.h
View File

@@ -36,11 +36,11 @@ namespace snmalloc
// Store the message queue on a separate cacheline. It is mutable data that
// is read by other threads.
alignas(CACHELINE_SIZE)
FreeObject::AtomicQueuePtr<capptr::bounds::Alloc> back{nullptr};
FreeObject::AtomicQueuePtr<capptr::bounds::AllocWild> back{nullptr};
// Store the two ends on different cache lines as access by different
// threads.
alignas(CACHELINE_SIZE) FreeObject::QueuePtr<capptr::bounds::Alloc> front{
nullptr};
alignas(CACHELINE_SIZE)
FreeObject::QueuePtr<capptr::bounds::AllocWild> front{nullptr};
constexpr RemoteAllocator() = default;
@@ -50,17 +50,19 @@ namespace snmalloc
SNMALLOC_ASSERT(front != nullptr);
}
void init(FreeObject::QueuePtr<capptr::bounds::Alloc> stub)
void
init(FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>
stub)
{
FreeObject::atomic_store_null(stub, key_global);
front = stub;
back.store(stub, std::memory_order_relaxed);
front = capptr_rewild(stub);
back.store(front, std::memory_order_relaxed);
invariant();
}
FreeObject::QueuePtr<capptr::bounds::Alloc> destroy()
FreeObject::QueuePtr<capptr::bounds::AllocWild> destroy()
{
FreeObject::QueuePtr<capptr::bounds::Alloc> fnt = front;
FreeObject::QueuePtr<capptr::bounds::AllocWild> fnt = front;
back.store(nullptr, std::memory_order_relaxed);
front = nullptr;
return fnt;
@@ -68,7 +70,7 @@ namespace snmalloc
inline bool is_empty()
{
FreeObject::QueuePtr<capptr::bounds::Alloc> bk =
FreeObject::QueuePtr<capptr::bounds::AllocWild> bk =
back.load(std::memory_order_relaxed);
return bk == front;
@@ -78,23 +80,26 @@ namespace snmalloc
* Pushes a list of messages to the queue. Each message from first to
* last should be linked together through their next pointers.
*/
template<typename Domesticator>
void enqueue(
FreeObject::QueuePtr<capptr::bounds::Alloc> first,
FreeObject::QueuePtr<capptr::bounds::Alloc> last,
const FreeListKey& key)
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>
first,
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>
last,
const FreeListKey& key,
Domesticator domesticate)
{
invariant();
FreeObject::atomic_store_null(last, key);
// exchange needs to be a release, so nullptr in next is visible.
FreeObject::QueuePtr<capptr::bounds::Alloc> prev =
back.exchange(last, std::memory_order_release);
FreeObject::QueuePtr<capptr::bounds::AllocWild> prev =
back.exchange(capptr_rewild(last), std::memory_order_release);
// XXX prev is not known to be domesticated
FreeObject::atomic_store_next(prev, first, key);
FreeObject::atomic_store_next(domesticate(prev), first, key);
}
FreeObject::QueuePtr<capptr::bounds::Alloc> peek()
FreeObject::QueuePtr<capptr::bounds::AllocWild> peek()
{
return front;
}
@@ -104,19 +109,27 @@ namespace snmalloc
*/
template<typename Domesticator>
std::pair<
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc>,
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>,
bool>
dequeue(const FreeListKey& key, Domesticator domesticate)
{
invariant();
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc> first =
domesticate(front);
FreeObject::QueuePtr<capptr::bounds::Alloc> next =
first->atomic_read_next(key, domesticate);
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>
first = domesticate(front);
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>
next = first->atomic_read_next(key, domesticate);
if (next != nullptr)
{
front = next;
/*
* We've domesticate_queue-d next so that we can read through it, but
* we're storing it back into client-accessible memory in
* !QueueHeadsAreTame builds, so go ahead and consider it Wild again.
* On QueueHeadsAreTame builds, the subsequent domesticate_head call
* above will also be a type-level sleight of hand, but we can still
* justify it by the domesticate_queue that happened in this dequeue().
*/
front = capptr_rewild(next);
invariant();
return {first, true};
}

15
src/mem/remotecache.h
View File

@@ -20,7 +20,7 @@ namespace snmalloc
FreeListBuilder<
false,
capptr::bounds::Alloc,
capptr::bounds::Alloc,
capptr::bounds::AllocWild,
false>,
REMOTE_SLOTS>
list;
@@ -78,7 +78,7 @@ namespace snmalloc
{
SNMALLOC_ASSERT(initialised);
auto r =
p.template as_reinterpret<FreeObject::T<capptr::bounds::Alloc>>();
p.template as_reinterpret<FreeObject::T<capptr::bounds::AllocWild>>();
list[get_slot<allocator_size>(target_id, 0)].add(r, key);
}
@@ -92,8 +92,11 @@ namespace snmalloc
SNMALLOC_ASSERT(initialised);
size_t post_round = 0;
bool sent_something = false;
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
auto domesticate =
[local_state](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
while (true)
{
@@ -109,7 +112,7 @@ namespace snmalloc
auto [first, last] = list[i].extract_segment(key);
MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry(
local_state, address_cast(first));
entry.get_remote()->enqueue(first, last, key);
entry.get_remote()->enqueue(first, last, key, domesticate);
sent_something = true;
}
}
@@ -120,7 +123,7 @@ namespace snmalloc
// Entries could map back onto the "resend" list,
// so take copy of the head, mark the last element,
// and clear the original list.
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc> resend;
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild> resend;
list[my_slot].close(resend, key);
post_round++;