From 7c04a9dad660dc68e086a862c6f2af72a6607f63 Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Mon, 14 Dec 2020 18:34:08 +0000 Subject: [PATCH] Remote: store the sizeclass, too Squeeze some bits out of allocator IDs so that we can land the sizeclass in each Remote object. The intent is that, on StrictProvenance architectures like CHERI, we will be able to route Remote messages through RemoteCache-s without needing to amplify back to read the sizeclass metadata field out of the slab headers. --- src/mem/alloc.h | 68 +++++++++++++++++++-------------------- src/mem/globalalloc.h | 2 +- src/mem/remoteallocator.h | 33 +++++++++++++++---- src/mem/sizeclass.h | 2 ++ 4 files changed, 62 insertions(+), 43 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 03ad7e8..efaba52 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -510,9 +510,19 @@ namespace snmalloc #endif } - size_t get_id() + /** + * Return this allocator's "truncated" ID, an integer useful as a hash + * value of this allocator. + * + * Specifically, this is the address of this allocator's message queue + * with the least significant bits missing, masked by SIZECLASS_MASK. + * This will be unique for Allocs with inline queues; Allocs with + * out-of-line queues must ensure that no two queues' addresses collide + * under this masking. + */ + size_t get_trunc_id() { - return id(); + return public_state()->trunc_id(); } private: @@ -571,30 +581,26 @@ namespace snmalloc IsQueueInline>); constexpr size_t initial_shift = bits::next_pow2_bits_const(allocator_size); + static_assert( + initial_shift >= 8, + "Can't embed sizeclass_t into allocator ID low bits"); SNMALLOC_ASSERT((initial_shift + (r * REMOTE_SLOT_BITS)) < 64); return (id >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK; } SNMALLOC_FAST_PATH void - dealloc_sized(alloc_id_t target_id, void* p, size_t objectsize) + dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass) { - this->capacity -= objectsize; + this->capacity -= sizeclass_to_size(sizeclass); Remote* r = static_cast(p); - r->set_target_id(target_id); - SNMALLOC_ASSERT(r->target_id() == target_id); + r->set_info(target_id, sizeclass); RemoteList* l = &list[get_slot(target_id, 0)]; l->last->non_atomic_next = r; l->last = r; } - SNMALLOC_FAST_PATH void - dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass) - { - dealloc_sized(target_id, p, sizeclass_to_size(sizeclass)); - } - void post(alloc_id_t id) { // When the cache gets big, post lists to their target allocators. @@ -640,7 +646,7 @@ namespace snmalloc { // Use the next N bits to spread out remote deallocs in our own // slot. - size_t slot = get_slot(r->target_id(), post_round); + size_t slot = get_slot(r->trunc_target_id(), post_round); RemoteList* l = &list[slot]; l->last->non_atomic_next = r; l->last = r; @@ -694,11 +700,6 @@ namespace snmalloc } } - alloc_id_t id() - { - return public_state()->id(); - } - auto& message_queue() { return public_state()->message_queue; @@ -725,9 +726,6 @@ namespace snmalloc remote_alloc = r; } - if (id() >= static_cast(-1)) - error("Id should not be -1"); - // If this is fake, don't do any of the bits of initialisation that may // allocate memory. if (isFake) @@ -863,7 +861,7 @@ namespace snmalloc { error("Critical error: Out-of-memory during initialisation."); } - dummy->set_target_id(id()); + dummy->set_info(get_trunc_id(), size_to_sizeclass_const(MIN_ALLOC_SIZE)); message_queue().init(dummy); } @@ -872,14 +870,14 @@ namespace snmalloc Superslab* super = Superslab::get(p); #ifdef CHECK_CLIENT - if (p->target_id() != super->get_allocator()->id()) + if (p->trunc_target_id() != (super->get_allocator()->trunc_id())) error("Detected memory corruption. Potential use-after-free"); #endif if (likely(super->get_kind() == Super)) { Slab* slab = Metaslab::get_slab(p); Metaslab& meta = super->get_meta(slab); - if (likely(p->target_id() == id())) + if (likely(super->get_allocator() == public_state())) { small_dealloc_offseted(super, p, meta.sizeclass); return; @@ -894,7 +892,7 @@ namespace snmalloc if (likely(super->get_kind() == Medium)) { Mediumslab* slab = Mediumslab::get(p); - if (p->target_id() == id()) + if (likely(super->get_allocator() == public_state())) { sizeclass_t sizeclass = slab->get_sizeclass(); void* start = remove_cache_friendly_offset(p, sizeclass); @@ -903,16 +901,17 @@ namespace snmalloc else { // Queue for remote dealloc elsewhere. - remote.dealloc(p->target_id(), p, slab->get_sizeclass()); + remote.dealloc(p->trunc_target_id(), p, slab->get_sizeclass()); } } else { - SNMALLOC_ASSERT(likely(p->target_id() != id())); + SNMALLOC_ASSERT(likely(p->trunc_target_id() != get_trunc_id())); + SNMALLOC_ASSERT(likely(super->get_allocator() != public_state())); Slab* slab = Metaslab::get_slab(p); Metaslab& meta = super->get_meta(slab); // Queue for remote dealloc elsewhere. - remote.dealloc(p->target_id(), p, meta.sizeclass); + remote.dealloc(p->trunc_target_id(), p, meta.sizeclass); } } @@ -933,7 +932,7 @@ namespace snmalloc return; stats().remote_post(); - remote.post(id()); + remote.post(get_trunc_id()); } /** @@ -1468,17 +1467,16 @@ namespace snmalloc void remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) { MEASURE_TIME(remote_dealloc, 4, 16); - SNMALLOC_ASSERT(target->id() != id()); + SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id()); // Check whether this will overflow the cache first. If we are a fake // allocator, then our cache will always be full and so we will never hit // this path. - size_t sz = sizeclass_to_size(sizeclass); if (remote.capacity > 0) { void* offseted = apply_cache_friendly_offset(p, sizeclass); stats().remote_free(sizeclass); - remote.dealloc_sized(target->id(), offseted, sz); + remote.dealloc(target->trunc_id(), offseted, sizeclass); return; } @@ -1488,7 +1486,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH void remote_dealloc_slow(RemoteAllocator* target, void* p, sizeclass_t sizeclass) { - SNMALLOC_ASSERT(target->id() != id()); + SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id()); // Now that we've established that we're in the slow path (if we're a // real allocator, we will have to empty our cache now), check if we are @@ -1506,10 +1504,10 @@ namespace snmalloc stats().remote_free(sizeclass); void* offseted = apply_cache_friendly_offset(p, sizeclass); - remote.dealloc(target->id(), offseted, sizeclass); + remote.dealloc(target->trunc_id(), offseted, sizeclass); stats().remote_post(); - remote.post(id()); + remote.post(get_trunc_id()); } ChunkMap& chunkmap() diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index f94ad45..16cf29a 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -126,7 +126,7 @@ namespace snmalloc if (alloc->remote.capacity < REMOTE_CACHE) { alloc->stats().remote_post(); - alloc->remote.post(alloc->id()); + alloc->remote.post(alloc->get_trunc_id()); done = false; } diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index d833a25..7440a0b 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -22,16 +22,34 @@ namespace snmalloc std::atomic next{nullptr}; }; - alloc_id_t allocator_id; + /* + * We embed the size class in the bottom 8 bits of an allocator ID (i.e., + * the address of an Alloc's remote_alloc's message_queue; in practice we + * only need 7 bits, but using 8 is conjectured to be faster). The hashing + * algorithm of the Alloc's RemoteCache already ignores the bottom + * "initial_shift" bits, which is, in practice, well above 8. There's a + * static_assert() over there that helps ensure this stays true. + * + * This does mean that we might have message_queues that always collide in + * the hash algorithm, if they're within "initial_shift" of each other. Such + * pairings will substantially decrease performance and so we prohibit them + * and use SNMALLOC_ASSERT to verify that they do not exist in debug builds. + */ + alloc_id_t alloc_id_and_sizeclass; - void set_target_id(alloc_id_t id) + void set_info(alloc_id_t id, sizeclass_t sc) { - allocator_id = id; + alloc_id_and_sizeclass = (id & ~SIZECLASS_MASK) | sc; } - alloc_id_t target_id() + alloc_id_t trunc_target_id() { - return allocator_id; + return alloc_id_and_sizeclass & ~SIZECLASS_MASK; + } + + sizeclass_t sizeclass() + { + return alloc_id_and_sizeclass & SIZECLASS_MASK; } }; @@ -46,10 +64,11 @@ namespace snmalloc // is read by other threads. alignas(CACHELINE_SIZE) MPSCQ message_queue; - alloc_id_t id() + alloc_id_t trunc_id() { return static_cast( - reinterpret_cast(&message_queue)); + reinterpret_cast(&message_queue)) & + ~SIZECLASS_MASK; } }; } // namespace snmalloc diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 5321aaf..0d39db3 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -11,6 +11,8 @@ namespace snmalloc // using sizeclass_t = uint8_t; using sizeclass_compress_t = uint8_t; + constexpr static uintptr_t SIZECLASS_MASK = 0xFF; + constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short); constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); constexpr static size_t