From 95871ff8a132180af91ea3497ebb138b42104ab5 Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Sat, 13 Mar 2021 18:21:35 +0000 Subject: [PATCH] SP: free lists and remote queues are CBAlloc Continue tightening the screws on pointer bounds. Notably, pointers in remote queues are bounded to the free objects. While we believe that something like MTE is required to make in-band metadata safe, this is a kind of defense in depth for StrictProvenance architectures: UAF for small and medium objects expose mostly other (free) small or medium objects and not allocator metadata (modulo some potential aliasing when Superslabs and Mediumslabs interconvert). This might shift the burdon on an attacker from simply holding a UAF pointer to having had to farm several heap pointers. The policy of bounding remote queue pointers may make the allocator's behavior for small objects unexpected: while initial object construction during allocation (that is, when the free list is empty) continues to cleave out exportable pointers from elevated pointers to internal slabs, reuse pulls from free lists of *already-bounded* objects. These objects are queued by the deallocation side, of course, but these paths now include "parallel reconstruction" of a pointer to the free object from the amplified view of the returned pointer, rather than queueing amplified pointers and leaving reconstruction to the allocation side. Medium objects are possibly similarly mysterious with the added twist that medium slabs do not store pointers but rather always cleave from their self-reference (but their interface has always operated using pointers). Nevertheless, pointers to medium objects end up in remote queues, so we continue to engage in "parallel reconstruction" in the deallocation paths. --- src/mem/alloc.h | 89 +++++++++++++++++++++------------------ src/mem/freelist.h | 44 +++++++++---------- src/mem/mediumslab.h | 2 +- src/mem/metaslab.h | 9 ++-- src/mem/remoteallocator.h | 6 +-- src/mem/slab.h | 38 ++++++++++------- 6 files changed, 101 insertions(+), 87 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index fcdf501..04eeccb 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -518,11 +518,11 @@ namespace snmalloc */ Remote head{}; - CapPtr last{&head}; + CapPtr last{&head}; void clear() { - last = CapPtr(&head); + last = CapPtr(&head); } bool empty() @@ -566,7 +566,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc( alloc_id_t target_id, - CapPtr p, + CapPtr p, sizeclass_t sizeclass) { this->capacity -= sizeclass_to_size(sizeclass); @@ -581,8 +581,6 @@ namespace snmalloc void post(LargeAlloc* large_allocator, alloc_id_t id) { - UNUSED(large_allocator); - // When the cache gets big, post lists to their target allocators. capacity = REMOTE_CACHE; @@ -598,12 +596,14 @@ namespace snmalloc continue; RemoteList* l = &list[i]; - CapPtr first = l->head.non_atomic_next; + CapPtr first = l->head.non_atomic_next; if (!l->empty()) { // Send all slots to the target at the head of the list. - auto super = Superslab::get(first); + auto first_auth = + large_allocator->template capptr_amplify(first); + auto super = Superslab::get(first_auth); super->get_allocator()->message_queue.enqueue(first, l->last); l->clear(); } @@ -616,7 +616,7 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - CapPtr r = resend->head.non_atomic_next; + CapPtr r = resend->head.non_atomic_next; resend->last->non_atomic_next = nullptr; resend->clear(); @@ -762,7 +762,7 @@ namespace snmalloc // Destroy the message queue so that it has no stub message. { - CapPtr p = message_queue().destroy(); + CapPtr p = message_queue().destroy(); while (p != nullptr) { @@ -796,8 +796,9 @@ namespace snmalloc if (!small_fast_free_lists[i].empty()) { auto head = small_fast_free_lists[i].peek(); - auto super = Superslab::get(head); - auto slab = Metaslab::get_slab(head); + auto head_auth = large_allocator.capptr_amplify(head); + auto super = Superslab::get(head_auth); + auto slab = Metaslab::get_slab(head_auth); do { auto curr = small_fast_free_lists[i].take(entropy); @@ -847,7 +848,7 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - auto dummy = CapPtr(alloc(MIN_ALLOC_SIZE)) + auto dummy = CapPtr(alloc(MIN_ALLOC_SIZE)) .template as_static(); if (dummy == nullptr) { @@ -857,12 +858,12 @@ namespace snmalloc message_queue().init(dummy); } - SNMALLOC_FAST_PATH void handle_dealloc_remote(CapPtr p) + SNMALLOC_FAST_PATH void handle_dealloc_remote(CapPtr p) { if (likely(p->trunc_target_id() == get_trunc_id())) { // Destined for my slabs - auto p_auth = CapPtr(p); + auto p_auth = large_allocator.template capptr_amplify(p); auto super = Superslab::get(p_auth); check_client( @@ -883,26 +884,25 @@ namespace snmalloc } SNMALLOC_SLOW_PATH void dealloc_not_large( - RemoteAllocator* target, - CapPtr p_auth, - sizeclass_t sizeclass) + RemoteAllocator* target, CapPtr p, sizeclass_t sizeclass) { if (likely(target->trunc_id() == get_trunc_id())) { + auto p_auth = large_allocator.capptr_amplify(p); auto super = Superslab::get(p_auth); - auto offseted = apply_cache_friendly_offset(p_auth, sizeclass) + auto offseted = apply_cache_friendly_offset(p, sizeclass) .template as_reinterpret(); dealloc_not_large_local(super, offseted, sizeclass); } else { - remote_dealloc_and_post(target, p_auth, sizeclass); + remote_dealloc_and_post(target, p, sizeclass); } } SNMALLOC_FAST_PATH void dealloc_not_large_local( CapPtr super, - CapPtr p_auth_offseted, + CapPtr p_offseted, sizeclass_t sizeclass) { // Guard against remote queues that have colliding IDs @@ -911,16 +911,17 @@ namespace snmalloc if (likely(sizeclass < NUM_SMALL_CLASSES)) { SNMALLOC_ASSERT(super->get_kind() == Super); - auto slab = Metaslab::get_slab(p_auth_offseted); + auto slab = + Metaslab::get_slab(Aal::capptr_rebound(super.as_void(), p_offseted)); small_dealloc_offseted( - super, slab, FreeObject::make(p_auth_offseted), sizeclass); + super, slab, FreeObject::make(p_offseted), sizeclass); } else { SNMALLOC_ASSERT(super->get_kind() == Medium); medium_dealloc_local( super.template as_reinterpret(), - Remote::clear(p_auth_offseted, sizeclass), + Remote::clear(p_offseted, sizeclass), sizeclass); } } @@ -1079,10 +1080,7 @@ namespace snmalloc p, sizeclass_to_size(sizeclass)); } - // TODO: This goes away once our free lists are bounded - auto ret = Aal::capptr_bound(p, size); - - return capptr_export(ret); + return capptr_export(p); } if (likely(!has_messages())) @@ -1206,10 +1204,7 @@ namespace snmalloc pal_zero(p, sizeclass_to_size(sizeclass)); } - // TODO: This goes away once our free lists are bounded - auto p_bounded = Aal::capptr_bound(p, rsize); - - return capptr_export(p_bounded); + return capptr_export(p); } /** @@ -1285,19 +1280,22 @@ namespace snmalloc RemoteAllocator* target = super->get_allocator(); + auto p = + Aal::capptr_bound(p_auth, sizeclass_to_size(sizeclass)); + if (likely(target == public_state())) { - auto offseted = apply_cache_friendly_offset(p_auth, sizeclass); + auto offseted = apply_cache_friendly_offset(p, sizeclass); small_dealloc_offseted(super, slab, offseted, sizeclass); } else - remote_dealloc(target, p_auth, sizeclass); + remote_dealloc(target, p, sizeclass); } SNMALLOC_FAST_PATH void small_dealloc_offseted( CapPtr super, CapPtr slab, - CapPtr p, + CapPtr p, sizeclass_t sizeclass) { stats().sizeclass_dealloc(sizeclass); @@ -1308,7 +1306,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( CapPtr super, CapPtr slab, - CapPtr p, + CapPtr p, sizeclass_t sizeclass) { if (likely(Slab::dealloc_fast(slab, super, p, entropy))) @@ -1320,7 +1318,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow( CapPtr super, CapPtr slab, - CapPtr p, + CapPtr p, sizeclass_t sizeclass) { bool was_full = super->is_full(); @@ -1495,16 +1493,23 @@ namespace snmalloc RemoteAllocator* target = slab->get_allocator(); + // TODO: This bound is perhaps superfluous in the local case, as + // mediumslabs store free objects by offset rather than pointer. + auto p = + Aal::capptr_bound(p_auth, sizeclass_to_size(sizeclass)); + if (likely(target == public_state())) - medium_dealloc_local(slab, p_auth, sizeclass); + medium_dealloc_local(slab, p, sizeclass); else - remote_dealloc(target, p_auth, sizeclass); + { + remote_dealloc(target, p, sizeclass); + } } SNMALLOC_FAST_PATH void medium_dealloc_local( CapPtr slab, - CapPtr p, + CapPtr p, sizeclass_t sizeclass) { stats().sizeclass_dealloc(sizeclass); @@ -1644,7 +1649,7 @@ namespace snmalloc // Clang. SNMALLOC_FAST_PATH void remote_dealloc( - RemoteAllocator* target, CapPtr p, sizeclass_t sizeclass) + RemoteAllocator* target, CapPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id()); @@ -1664,7 +1669,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH void remote_dealloc_slow( RemoteAllocator* target, - CapPtr p_auth, + CapPtr p_auth, sizeclass_t sizeclass) { SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id()); @@ -1687,7 +1692,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH void remote_dealloc_and_post( RemoteAllocator* target, - CapPtr p_auth, + CapPtr p_auth, sizeclass_t sizeclass) { handle_message_queue(); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index d7987df..7843379 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -48,14 +48,14 @@ namespace snmalloc } template - static inline bool different_slab(address_t p1, CapPtr p2) + static inline bool different_slab(address_t p1, CapPtr p2) { return different_slab(p1, address_cast(p2)); } template static inline bool - different_slab(CapPtr p1, CapPtr p2) + different_slab(CapPtr p1, CapPtr p2) { return different_slab(address_cast(p1), address_cast(p2)); } @@ -64,7 +64,7 @@ namespace snmalloc class EncodeFreeObjectReference { - CapPtr reference; + CapPtr reference; /** * On architectures which use IntegerPointers, we can obfuscate our free @@ -82,8 +82,8 @@ namespace snmalloc public: #ifdef CHECK_CLIENT template - static std::enable_if_t> encode( - uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) + static std::enable_if_t> encode( + uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) { // Simple involutional encoding. The bottom half of each word is // multiplied by a function of both global and local keys (the latter, @@ -96,13 +96,13 @@ namespace snmalloc address_t key = (local_key + 1) * entropy.get_constant_key(); next ^= (((next & MASK) + 1) * key) & ~(bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1); - return CapPtr(reinterpret_cast(next)); + return CapPtr(reinterpret_cast(next)); } #endif template - static std::enable_if_t> encode( - uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) + static std::enable_if_t> encode( + uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) { UNUSED(local_key); UNUSED(entropy); @@ -110,14 +110,14 @@ namespace snmalloc } void store( - CapPtr value, + CapPtr value, uint16_t local_key, LocalEntropy& entropy) { reference = encode(local_key, value, entropy); } - CapPtr read(uint16_t local_key, LocalEntropy& entropy) + CapPtr read(uint16_t local_key, LocalEntropy& entropy) { return encode(local_key, reference, entropy); } @@ -136,7 +136,7 @@ namespace snmalloc public: EncodeFreeObjectReference next_object; - static CapPtr make(CapPtr p) + static CapPtr make(CapPtr p) { return p.template as_static(); } @@ -144,7 +144,7 @@ namespace snmalloc /** * Construct a FreeObject for local slabs from a Remote message. */ - static CapPtr make(CapPtr p) + static CapPtr make(CapPtr p) { // TODO: Zero the difference between a FreeObject and a Remote return p.template as_reinterpret(); @@ -153,7 +153,7 @@ namespace snmalloc /** * Read the next pointer handling any required decoding of the pointer */ - CapPtr read_next(uint16_t key, LocalEntropy& entropy) + CapPtr read_next(uint16_t key, LocalEntropy& entropy) { return next_object.read(key, entropy); } @@ -166,7 +166,7 @@ namespace snmalloc */ class FreeListIter { - CapPtr curr = nullptr; + CapPtr curr = nullptr; #ifdef CHECK_CLIENT address_t prev = 0; #endif @@ -186,7 +186,7 @@ namespace snmalloc * Currently this is just the value of current before this call. * Other schemes could be used. */ - void update_cursor(CapPtr next) + void update_cursor(CapPtr next) { #ifdef CHECK_CLIENT # ifndef NDEBUG @@ -203,7 +203,7 @@ namespace snmalloc } public: - FreeListIter(CapPtr head) + FreeListIter(CapPtr head) : curr(head) #ifdef CHECK_CLIENT , @@ -226,7 +226,7 @@ namespace snmalloc /** * Returns current head without affecting the iterator. */ - CapPtr peek() + CapPtr peek() { return curr; } @@ -234,7 +234,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - CapPtr take(LocalEntropy& entropy) + CapPtr take(LocalEntropy& entropy) { #ifdef CHECK_CLIENT check_client( @@ -354,7 +354,7 @@ namespace snmalloc return true; } - bool debug_different_slab(CapPtr n) + bool debug_different_slab(CapPtr n) { for (size_t i = 0; i < LENGTH; i++) { @@ -367,7 +367,7 @@ namespace snmalloc /** * Adds an element to the builder */ - void add(CapPtr n, LocalEntropy& entropy) + void add(CapPtr n, LocalEntropy& entropy) { SNMALLOC_ASSERT(!debug_different_slab(n) || empty()); @@ -394,11 +394,11 @@ namespace snmalloc { uint16_t local_prev = HEAD_KEY; EncodeFreeObjectReference* iter = &head[i]; - CapPtr prev_obj = iter->read(local_prev, entropy); + CapPtr prev_obj = iter->read(local_prev, entropy); uint16_t local_curr = initial_key(prev_obj) & 0xffff; while (end[i] != iter) { - CapPtr next = iter->read(local_prev, entropy); + CapPtr next = iter->read(local_prev, entropy); check_client(!different_slab(next, prev_obj), "Heap corruption"); local_prev = local_curr; local_curr = address_cast(next) & 0xffff; diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index 1877961..c1ea9be 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -122,7 +122,7 @@ namespace snmalloc } static bool - dealloc(CapPtr self, CapPtr p) + dealloc(CapPtr self, CapPtr p) { SNMALLOC_ASSERT(self->head > 0); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 2173786..7dc5fcd 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -178,31 +178,32 @@ namespace snmalloc self->free_queue.close(fast_free_list, entropy); auto n = fast_free_list.take(entropy); auto n_slab = Aal::capptr_rebound(self.as_void(), n); + auto meta = Metaslab::get_slab(n_slab); entropy.refresh_bits(); // Treat stealing the free list as allocating it all. self->remove(); - self->set_full(Metaslab::get_slab(n_slab)); + self->set_full(meta); auto p = remove_cache_friendly_offset(n, self->sizeclass()); SNMALLOC_ASSERT(is_start_of_object(self, address_cast(p))); - self->debug_slab_invariant(Metaslab::get_slab(n_slab), entropy); + self->debug_slab_invariant(meta, entropy); if constexpr (zero_mem == YesZero) { if (rsize < PAGE_ALIGNED_SIZE) pal_zero(p, rsize); else - pal_zero(p, rsize); + pal_zero(Aal::capptr_rebound(self.as_void(), p), rsize); } else { UNUSED(rsize); } - return capptr_export(Aal::capptr_bound(p, rsize)); + return capptr_export(p); } template diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 764cd0f..0a5d86d 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -18,8 +18,8 @@ namespace snmalloc using alloc_id_t = size_t; union { - CapPtr non_atomic_next; - AtomicCapPtr next{nullptr}; + CapPtr non_atomic_next; + AtomicCapPtr next{nullptr}; }; /* @@ -72,7 +72,7 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. alignas(CACHELINE_SIZE) - MPSCQ message_queue; + MPSCQ message_queue; alloc_id_t trunc_id() { diff --git a/src/mem/slab.h b/src/mem/slab.h index b398c74..42694ac 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -50,36 +50,42 @@ namespace snmalloc // Structure to represent the temporary list elements struct PreAllocObject { - CapPtr next; + CapPtr next; }; // The following code implements Sattolo's algorithm for generating // random cyclic permutations. This implementation is in the opposite // direction, so that the original space does not need initialising. This // is described as outside-in without citation on Wikipedia, appears to be // Folklore algorithm. - auto curr = + + // Note the wide bounds on curr relative to each of the ->next fields; + // curr is not persisted once the list is built. + CapPtr curr = pointer_offset(bumpptr, 0).template as_static(); - curr->next = curr; + curr->next = Aal::capptr_bound(curr, rsize); + uint16_t count = 1; - for (auto p = pointer_offset(bumpptr, rsize) - .template as_static(); - p.as_void() < slab_end; - p = pointer_offset(p, rsize).template as_static()) + for (curr = + pointer_offset(curr, rsize).template as_static(); + curr.as_void() < slab_end; + curr = + pointer_offset(curr, rsize).template as_static()) { size_t insert_index = entropy.sample(count); - p->next = std::exchange( + curr->next = std::exchange( pointer_offset(bumpptr, insert_index * rsize) .template as_static() ->next, - p); + Aal::capptr_bound(curr, rsize)); count++; } // Pick entry into space, and then build linked list by traversing cycle - // to the start. + // to the start. Use ->next to jump from CBArena to CBAlloc. auto start_index = entropy.sample(count); auto start_ptr = pointer_offset(bumpptr, start_index * rsize) - .template as_static(); + .template as_static() + ->next; auto curr_ptr = start_ptr; do { @@ -89,7 +95,7 @@ namespace snmalloc #else for (auto p = bumpptr; p < slab_end; p = pointer_offset(p, rsize)) { - b.add(FreeObject::make(p.as_void()), entropy); + b.add(Aal::capptr_bound(p, rsize), entropy); } #endif // This code consumes everything up to slab_end. @@ -105,7 +111,7 @@ namespace snmalloc static SNMALLOC_FAST_PATH bool dealloc_fast( CapPtr self, CapPtr super, - CapPtr p, + CapPtr p, LocalEntropy& entropy) { auto meta = super->get_meta(self); @@ -128,7 +134,7 @@ namespace snmalloc CapPtr self, SlabList* sl, CapPtr super, - CapPtr p, + CapPtr p, LocalEntropy& entropy) { auto meta = super->get_meta(self); @@ -137,7 +143,9 @@ namespace snmalloc if (meta->is_full()) { auto allocated = get_slab_capacity( - meta->sizeclass(), Metaslab::is_short(Metaslab::get_slab(p))); + meta->sizeclass(), + Metaslab::is_short( + Metaslab::get_slab(Aal::capptr_rebound(super.as_void(), p)))); // We are not on the sizeclass list. if (allocated == 1) {