diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 77363f6..8d84c91 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -163,7 +163,7 @@ namespace snmalloc // Using an actual allocation removes a conditional from a critical path. auto dummy = capptr::Alloc(small_alloc_one(MIN_ALLOC_SIZE)) - .template as_static>(); + .template as_static>(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -421,13 +421,17 @@ namespace snmalloc handle_message_queue_inner(Action action, Args... args) { bool need_post = false; - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto local_state = backend_state_ptr(); + auto domesticate = + [local_state](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; for (size_t i = 0; i < REMOTE_BATCH; i++) { auto p = message_queue().peek(); auto& entry = SharedStateHandle::Pagemap::get_metaentry( - backend_state_ptr(), snmalloc::address_cast(p)); + local_state, snmalloc::address_cast(p)); auto r = message_queue().dequeue(key_global, domesticate); @@ -750,24 +754,26 @@ namespace snmalloc bool flush(bool destroy_queue = false) { SNMALLOC_ASSERT(attached_cache != nullptr); - // TODO: Placeholder - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto local_state = backend_state_ptr(); + auto domesticate = + [local_state](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; if (destroy_queue) { - auto p = message_queue().destroy(); + auto p_wild = message_queue().destroy(); + auto p_tame = domesticate(p_wild); - while (p != nullptr) + while (p_tame != nullptr) { bool need_post = true; // Always going to post, so ignore. - auto n = p->atomic_read_next(key_global, domesticate); + auto n_tame = p_tame->atomic_read_next(key_global, domesticate); auto& entry = SharedStateHandle::Pagemap::get_metaentry( - backend_state_ptr(), snmalloc::address_cast(p)); - handle_dealloc_remote(entry, p.as_void(), need_post); - // TODO p = SharedStateHandle::capptr_domesticate(backend_state_ptr(), - // n); - p = n; + backend_state_ptr(), snmalloc::address_cast(p_tame)); + handle_dealloc_remote(entry, p_tame.as_void(), need_post); + p_tame = n_tame; } } else diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index bce6352..c658c2e 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -36,11 +36,11 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. alignas(CACHELINE_SIZE) - FreeObject::AtomicQueuePtr back{nullptr}; + FreeObject::AtomicQueuePtr back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) FreeObject::QueuePtr front{ - nullptr}; + alignas(CACHELINE_SIZE) + FreeObject::QueuePtr front{nullptr}; constexpr RemoteAllocator() = default; @@ -50,17 +50,19 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void init(FreeObject::QueuePtr stub) + void + init(FreeObject::HeadPtr + stub) { FreeObject::atomic_store_null(stub, key_global); - front = stub; - back.store(stub, std::memory_order_relaxed); + front = capptr_rewild(stub); + back.store(front, std::memory_order_relaxed); invariant(); } - FreeObject::QueuePtr destroy() + FreeObject::QueuePtr destroy() { - FreeObject::QueuePtr fnt = front; + FreeObject::QueuePtr fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -68,7 +70,7 @@ namespace snmalloc inline bool is_empty() { - FreeObject::QueuePtr bk = + FreeObject::QueuePtr bk = back.load(std::memory_order_relaxed); return bk == front; @@ -78,23 +80,26 @@ namespace snmalloc * Pushes a list of messages to the queue. Each message from first to * last should be linked together through their next pointers. */ + template void enqueue( - FreeObject::QueuePtr first, - FreeObject::QueuePtr last, - const FreeListKey& key) + FreeObject::HeadPtr + first, + FreeObject::HeadPtr + last, + const FreeListKey& key, + Domesticator domesticate) { invariant(); FreeObject::atomic_store_null(last, key); // exchange needs to be a release, so nullptr in next is visible. - FreeObject::QueuePtr prev = - back.exchange(last, std::memory_order_release); + FreeObject::QueuePtr prev = + back.exchange(capptr_rewild(last), std::memory_order_release); - // XXX prev is not known to be domesticated - FreeObject::atomic_store_next(prev, first, key); + FreeObject::atomic_store_next(domesticate(prev), first, key); } - FreeObject::QueuePtr peek() + FreeObject::QueuePtr peek() { return front; } @@ -104,19 +109,27 @@ namespace snmalloc */ template std::pair< - FreeObject::HeadPtr, + FreeObject::HeadPtr, bool> dequeue(const FreeListKey& key, Domesticator domesticate) { invariant(); - FreeObject::HeadPtr first = - domesticate(front); - FreeObject::QueuePtr next = - first->atomic_read_next(key, domesticate); + FreeObject::HeadPtr + first = domesticate(front); + FreeObject::HeadPtr + next = first->atomic_read_next(key, domesticate); if (next != nullptr) { - front = next; + /* + * We've domesticate_queue-d next so that we can read through it, but + * we're storing it back into client-accessible memory in + * !QueueHeadsAreTame builds, so go ahead and consider it Wild again. + * On QueueHeadsAreTame builds, the subsequent domesticate_head call + * above will also be a type-level sleight of hand, but we can still + * justify it by the domesticate_queue that happened in this dequeue(). + */ + front = capptr_rewild(next); invariant(); return {first, true}; } diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 039d83d..0d20d89 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -20,7 +20,7 @@ namespace snmalloc FreeListBuilder< false, capptr::bounds::Alloc, - capptr::bounds::Alloc, + capptr::bounds::AllocWild, false>, REMOTE_SLOTS> list; @@ -78,7 +78,7 @@ namespace snmalloc { SNMALLOC_ASSERT(initialised); auto r = - p.template as_reinterpret>(); + p.template as_reinterpret>(); list[get_slot(target_id, 0)].add(r, key); } @@ -92,8 +92,11 @@ namespace snmalloc SNMALLOC_ASSERT(initialised); size_t post_round = 0; bool sent_something = false; - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [local_state](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; while (true) { @@ -109,7 +112,7 @@ namespace snmalloc auto [first, last] = list[i].extract_segment(key); MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( local_state, address_cast(first)); - entry.get_remote()->enqueue(first, last, key); + entry.get_remote()->enqueue(first, last, key, domesticate); sent_something = true; } } @@ -120,7 +123,7 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - FreeListIter resend; + FreeListIter resend; list[my_slot].close(resend, key); post_round++;