From d9822036b2b07cff3455234b84331c13acaaabfd Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Mon, 8 Mar 2021 12:12:02 +0000 Subject: [PATCH] SP: add bounds placeholders to message queues Like all the annotations so far, leave these as CBArena-bounded. However, we can start to do a little better than we were doing before: for not-large deallocations, we know that the internal superslab pointer is sufficiently authoritative that we can use it to get to data structures rather than also passing an amplified pointer to the allocation in question. This, then, partially reverts some of the earlier changes made while plumbing CapPtr<>s through the system. While here, avoid quite as many void*-s in favor of Remote* or SlabNext*. --- src/mem/alloc.h | 56 ++++++++++++++++++++++----------------- src/mem/remoteallocator.h | 16 ++++++++--- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 85c8236..a907836 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -517,16 +517,16 @@ namespace snmalloc */ Remote head{}; - Remote* last{&head}; + CapPtr last{&head}; void clear() { - last = &head; + last = CapPtr(&head); } bool empty() { - return last == &head; + return address_cast(last) == address_cast(&head); } }; @@ -563,12 +563,14 @@ namespace snmalloc return (id >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK; } - SNMALLOC_FAST_PATH void - dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass) + SNMALLOC_FAST_PATH void dealloc( + alloc_id_t target_id, + CapPtr p, + sizeclass_t sizeclass) { this->capacity -= sizeclass_to_size(sizeclass); + auto r = p.template as_reinterpret(); - Remote* r = static_cast(p); r->set_info(target_id, sizeclass); RemoteList* l = &list[get_slot(target_id, 0)]; @@ -595,12 +597,12 @@ namespace snmalloc continue; RemoteList* l = &list[i]; - Remote* first = l->head.non_atomic_next; + CapPtr first = l->head.non_atomic_next; if (!l->empty()) { // Send all slots to the target at the head of the list. - auto super = Superslab::get(CapPtr(first)); + auto super = Superslab::get(first); super->get_allocator()->message_queue.enqueue(first, l->last); l->clear(); } @@ -613,7 +615,7 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - Remote* r = resend->head.non_atomic_next; + CapPtr r = resend->head.non_atomic_next; resend->last->non_atomic_next = nullptr; resend->clear(); @@ -759,11 +761,11 @@ namespace snmalloc // Destroy the message queue so that it has no stub message. { - Remote* p = message_queue().destroy(); + CapPtr p = message_queue().destroy(); while (p != nullptr) { - Remote* n = p->non_atomic_next; + auto n = p->non_atomic_next; handle_dealloc_remote(p); p = n; } @@ -844,7 +846,8 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - Remote* dummy = reinterpret_cast(alloc(MIN_ALLOC_SIZE)); + auto dummy = CapPtr(alloc(MIN_ALLOC_SIZE)) + .template as_static(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -853,7 +856,7 @@ namespace snmalloc message_queue().init(dummy); } - SNMALLOC_FAST_PATH void handle_dealloc_remote(Remote* p) + SNMALLOC_FAST_PATH void handle_dealloc_remote(CapPtr p) { if (likely(p->trunc_target_id() == get_trunc_id())) { @@ -865,13 +868,16 @@ namespace snmalloc p->trunc_target_id() == super->get_allocator()->trunc_id(), "Detected memory corruption. Potential use-after-free"); - auto start_auth = remove_cache_friendly_offset(p_auth, p->sizeclass()); - dealloc_not_large_local(super, start_auth, p_auth, p->sizeclass()); + dealloc_not_large_local(super, p, p->sizeclass()); } else { - // Merely routing - remote.dealloc(p->trunc_target_id(), p, p->sizeclass()); + // Merely routing; despite the cast here, p is going to be cast right + // back to a Remote. + remote.dealloc( + p->trunc_target_id(), + p.template as_reinterpret(), + p->sizeclass()); } } @@ -885,7 +891,7 @@ namespace snmalloc auto super = Superslab::get(p_auth); auto offseted = apply_cache_friendly_offset(p_auth, sizeclass) .template as_reinterpret(); - dealloc_not_large_local(super, p_auth, offseted, sizeclass); + dealloc_not_large_local(super, offseted, sizeclass); } else { @@ -895,7 +901,6 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_not_large_local( CapPtr super, - CapPtr p_auth, CapPtr p_auth_offseted, sizeclass_t sizeclass) { @@ -905,7 +910,7 @@ namespace snmalloc if (likely(sizeclass < NUM_SMALL_CLASSES)) { SNMALLOC_ASSERT(super->get_kind() == Super); - auto slab = Metaslab::get_slab(p_auth); + auto slab = Metaslab::get_slab(p_auth_offseted); small_dealloc_offseted( super, slab, FreeObject::make(p_auth_offseted), sizeclass); } @@ -913,7 +918,9 @@ namespace snmalloc { SNMALLOC_ASSERT(super->get_kind() == Medium); medium_dealloc_local( - super.template as_reinterpret(), p_auth, sizeclass); + super.template as_reinterpret(), + Remote::clear(p_auth_offseted, sizeclass), + sizeclass); } } @@ -1070,7 +1077,7 @@ namespace snmalloc pal_zero( p, sizeclass_to_size(sizeclass)); } - return CapPtr(p); + return p; } if (likely(!has_messages())) @@ -1615,7 +1622,8 @@ namespace snmalloc if (remote.capacity > 0) { stats().remote_free(sizeclass); - remote.dealloc(target->trunc_id(), p.unsafe_capptr, sizeclass); + auto offseted = apply_cache_friendly_offset(p, sizeclass); + remote.dealloc(target->trunc_id(), offseted, sizeclass); return; } @@ -1654,7 +1662,7 @@ namespace snmalloc stats().remote_free(sizeclass); auto offseted = apply_cache_friendly_offset(p_auth, sizeclass); - remote.dealloc(target->trunc_id(), offseted.unsafe_capptr, sizeclass); + remote.dealloc(target->trunc_id(), offseted, sizeclass); stats().remote_post(); remote.post(&large_allocator, get_trunc_id()); diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 7440a0b..764cd0f 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -18,8 +18,8 @@ namespace snmalloc using alloc_id_t = size_t; union { - Remote* non_atomic_next; - std::atomic next{nullptr}; + CapPtr non_atomic_next; + AtomicCapPtr next{nullptr}; }; /* @@ -51,6 +51,15 @@ namespace snmalloc { return alloc_id_and_sizeclass & SIZECLASS_MASK; } + + /** Zero out a Remote tracking structure, return pointer to object base */ + template + SNMALLOC_FAST_PATH static CapPtr + clear(CapPtr self, sizeclass_t sizeclass) + { + pal_zero(self, sizeof(Remote)); + return remove_cache_friendly_offset(self, sizeclass); + } }; static_assert( @@ -62,7 +71,8 @@ namespace snmalloc using alloc_id_t = Remote::alloc_id_t; // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) MPSCQ message_queue; + alignas(CACHELINE_SIZE) + MPSCQ message_queue; alloc_id_t trunc_id() {