diff --git a/src/mem/alloc.h b/src/mem/alloc.h index fcdf501..04eeccb 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -518,11 +518,11 @@ namespace snmalloc */ Remote head{}; - CapPtr last{&head}; + CapPtr last{&head}; void clear() { - last = CapPtr(&head); + last = CapPtr(&head); } bool empty() @@ -566,7 +566,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc( alloc_id_t target_id, - CapPtr p, + CapPtr p, sizeclass_t sizeclass) { this->capacity -= sizeclass_to_size(sizeclass); @@ -581,8 +581,6 @@ namespace snmalloc void post(LargeAlloc* large_allocator, alloc_id_t id) { - UNUSED(large_allocator); - // When the cache gets big, post lists to their target allocators. capacity = REMOTE_CACHE; @@ -598,12 +596,14 @@ namespace snmalloc continue; RemoteList* l = &list[i]; - CapPtr first = l->head.non_atomic_next; + CapPtr first = l->head.non_atomic_next; if (!l->empty()) { // Send all slots to the target at the head of the list. - auto super = Superslab::get(first); + auto first_auth = + large_allocator->template capptr_amplify(first); + auto super = Superslab::get(first_auth); super->get_allocator()->message_queue.enqueue(first, l->last); l->clear(); } @@ -616,7 +616,7 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - CapPtr r = resend->head.non_atomic_next; + CapPtr r = resend->head.non_atomic_next; resend->last->non_atomic_next = nullptr; resend->clear(); @@ -762,7 +762,7 @@ namespace snmalloc // Destroy the message queue so that it has no stub message. { - CapPtr p = message_queue().destroy(); + CapPtr p = message_queue().destroy(); while (p != nullptr) { @@ -796,8 +796,9 @@ namespace snmalloc if (!small_fast_free_lists[i].empty()) { auto head = small_fast_free_lists[i].peek(); - auto super = Superslab::get(head); - auto slab = Metaslab::get_slab(head); + auto head_auth = large_allocator.capptr_amplify(head); + auto super = Superslab::get(head_auth); + auto slab = Metaslab::get_slab(head_auth); do { auto curr = small_fast_free_lists[i].take(entropy); @@ -847,7 +848,7 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - auto dummy = CapPtr(alloc(MIN_ALLOC_SIZE)) + auto dummy = CapPtr(alloc(MIN_ALLOC_SIZE)) .template as_static(); if (dummy == nullptr) { @@ -857,12 +858,12 @@ namespace snmalloc message_queue().init(dummy); } - SNMALLOC_FAST_PATH void handle_dealloc_remote(CapPtr p) + SNMALLOC_FAST_PATH void handle_dealloc_remote(CapPtr p) { if (likely(p->trunc_target_id() == get_trunc_id())) { // Destined for my slabs - auto p_auth = CapPtr(p); + auto p_auth = large_allocator.template capptr_amplify(p); auto super = Superslab::get(p_auth); check_client( @@ -883,26 +884,25 @@ namespace snmalloc } SNMALLOC_SLOW_PATH void dealloc_not_large( - RemoteAllocator* target, - CapPtr p_auth, - sizeclass_t sizeclass) + RemoteAllocator* target, CapPtr p, sizeclass_t sizeclass) { if (likely(target->trunc_id() == get_trunc_id())) { + auto p_auth = large_allocator.capptr_amplify(p); auto super = Superslab::get(p_auth); - auto offseted = apply_cache_friendly_offset(p_auth, sizeclass) + auto offseted = apply_cache_friendly_offset(p, sizeclass) .template as_reinterpret(); dealloc_not_large_local(super, offseted, sizeclass); } else { - remote_dealloc_and_post(target, p_auth, sizeclass); + remote_dealloc_and_post(target, p, sizeclass); } } SNMALLOC_FAST_PATH void dealloc_not_large_local( CapPtr super, - CapPtr p_auth_offseted, + CapPtr p_offseted, sizeclass_t sizeclass) { // Guard against remote queues that have colliding IDs @@ -911,16 +911,17 @@ namespace snmalloc if (likely(sizeclass < NUM_SMALL_CLASSES)) { SNMALLOC_ASSERT(super->get_kind() == Super); - auto slab = Metaslab::get_slab(p_auth_offseted); + auto slab = + Metaslab::get_slab(Aal::capptr_rebound(super.as_void(), p_offseted)); small_dealloc_offseted( - super, slab, FreeObject::make(p_auth_offseted), sizeclass); + super, slab, FreeObject::make(p_offseted), sizeclass); } else { SNMALLOC_ASSERT(super->get_kind() == Medium); medium_dealloc_local( super.template as_reinterpret(), - Remote::clear(p_auth_offseted, sizeclass), + Remote::clear(p_offseted, sizeclass), sizeclass); } } @@ -1079,10 +1080,7 @@ namespace snmalloc p, sizeclass_to_size(sizeclass)); } - // TODO: This goes away once our free lists are bounded - auto ret = Aal::capptr_bound(p, size); - - return capptr_export(ret); + return capptr_export(p); } if (likely(!has_messages())) @@ -1206,10 +1204,7 @@ namespace snmalloc pal_zero(p, sizeclass_to_size(sizeclass)); } - // TODO: This goes away once our free lists are bounded - auto p_bounded = Aal::capptr_bound(p, rsize); - - return capptr_export(p_bounded); + return capptr_export(p); } /** @@ -1285,19 +1280,22 @@ namespace snmalloc RemoteAllocator* target = super->get_allocator(); + auto p = + Aal::capptr_bound(p_auth, sizeclass_to_size(sizeclass)); + if (likely(target == public_state())) { - auto offseted = apply_cache_friendly_offset(p_auth, sizeclass); + auto offseted = apply_cache_friendly_offset(p, sizeclass); small_dealloc_offseted(super, slab, offseted, sizeclass); } else - remote_dealloc(target, p_auth, sizeclass); + remote_dealloc(target, p, sizeclass); } SNMALLOC_FAST_PATH void small_dealloc_offseted( CapPtr super, CapPtr slab, - CapPtr p, + CapPtr p, sizeclass_t sizeclass) { stats().sizeclass_dealloc(sizeclass); @@ -1308,7 +1306,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( CapPtr super, CapPtr slab, - CapPtr p, + CapPtr p, sizeclass_t sizeclass) { if (likely(Slab::dealloc_fast(slab, super, p, entropy))) @@ -1320,7 +1318,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow( CapPtr super, CapPtr slab, - CapPtr p, + CapPtr p, sizeclass_t sizeclass) { bool was_full = super->is_full(); @@ -1495,16 +1493,23 @@ namespace snmalloc RemoteAllocator* target = slab->get_allocator(); + // TODO: This bound is perhaps superfluous in the local case, as + // mediumslabs store free objects by offset rather than pointer. + auto p = + Aal::capptr_bound(p_auth, sizeclass_to_size(sizeclass)); + if (likely(target == public_state())) - medium_dealloc_local(slab, p_auth, sizeclass); + medium_dealloc_local(slab, p, sizeclass); else - remote_dealloc(target, p_auth, sizeclass); + { + remote_dealloc(target, p, sizeclass); + } } SNMALLOC_FAST_PATH void medium_dealloc_local( CapPtr slab, - CapPtr p, + CapPtr p, sizeclass_t sizeclass) { stats().sizeclass_dealloc(sizeclass); @@ -1644,7 +1649,7 @@ namespace snmalloc // Clang. SNMALLOC_FAST_PATH void remote_dealloc( - RemoteAllocator* target, CapPtr p, sizeclass_t sizeclass) + RemoteAllocator* target, CapPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id()); @@ -1664,7 +1669,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH void remote_dealloc_slow( RemoteAllocator* target, - CapPtr p_auth, + CapPtr p_auth, sizeclass_t sizeclass) { SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id()); @@ -1687,7 +1692,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH void remote_dealloc_and_post( RemoteAllocator* target, - CapPtr p_auth, + CapPtr p_auth, sizeclass_t sizeclass) { handle_message_queue(); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index d7987df..7843379 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -48,14 +48,14 @@ namespace snmalloc } template - static inline bool different_slab(address_t p1, CapPtr p2) + static inline bool different_slab(address_t p1, CapPtr p2) { return different_slab(p1, address_cast(p2)); } template static inline bool - different_slab(CapPtr p1, CapPtr p2) + different_slab(CapPtr p1, CapPtr p2) { return different_slab(address_cast(p1), address_cast(p2)); } @@ -64,7 +64,7 @@ namespace snmalloc class EncodeFreeObjectReference { - CapPtr reference; + CapPtr reference; /** * On architectures which use IntegerPointers, we can obfuscate our free @@ -82,8 +82,8 @@ namespace snmalloc public: #ifdef CHECK_CLIENT template - static std::enable_if_t> encode( - uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) + static std::enable_if_t> encode( + uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) { // Simple involutional encoding. The bottom half of each word is // multiplied by a function of both global and local keys (the latter, @@ -96,13 +96,13 @@ namespace snmalloc address_t key = (local_key + 1) * entropy.get_constant_key(); next ^= (((next & MASK) + 1) * key) & ~(bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1); - return CapPtr(reinterpret_cast(next)); + return CapPtr(reinterpret_cast(next)); } #endif template - static std::enable_if_t> encode( - uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) + static std::enable_if_t> encode( + uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) { UNUSED(local_key); UNUSED(entropy); @@ -110,14 +110,14 @@ namespace snmalloc } void store( - CapPtr value, + CapPtr value, uint16_t local_key, LocalEntropy& entropy) { reference = encode(local_key, value, entropy); } - CapPtr read(uint16_t local_key, LocalEntropy& entropy) + CapPtr read(uint16_t local_key, LocalEntropy& entropy) { return encode(local_key, reference, entropy); } @@ -136,7 +136,7 @@ namespace snmalloc public: EncodeFreeObjectReference next_object; - static CapPtr make(CapPtr p) + static CapPtr make(CapPtr p) { return p.template as_static(); } @@ -144,7 +144,7 @@ namespace snmalloc /** * Construct a FreeObject for local slabs from a Remote message. */ - static CapPtr make(CapPtr p) + static CapPtr make(CapPtr p) { // TODO: Zero the difference between a FreeObject and a Remote return p.template as_reinterpret(); @@ -153,7 +153,7 @@ namespace snmalloc /** * Read the next pointer handling any required decoding of the pointer */ - CapPtr read_next(uint16_t key, LocalEntropy& entropy) + CapPtr read_next(uint16_t key, LocalEntropy& entropy) { return next_object.read(key, entropy); } @@ -166,7 +166,7 @@ namespace snmalloc */ class FreeListIter { - CapPtr curr = nullptr; + CapPtr curr = nullptr; #ifdef CHECK_CLIENT address_t prev = 0; #endif @@ -186,7 +186,7 @@ namespace snmalloc * Currently this is just the value of current before this call. * Other schemes could be used. */ - void update_cursor(CapPtr next) + void update_cursor(CapPtr next) { #ifdef CHECK_CLIENT # ifndef NDEBUG @@ -203,7 +203,7 @@ namespace snmalloc } public: - FreeListIter(CapPtr head) + FreeListIter(CapPtr head) : curr(head) #ifdef CHECK_CLIENT , @@ -226,7 +226,7 @@ namespace snmalloc /** * Returns current head without affecting the iterator. */ - CapPtr peek() + CapPtr peek() { return curr; } @@ -234,7 +234,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - CapPtr take(LocalEntropy& entropy) + CapPtr take(LocalEntropy& entropy) { #ifdef CHECK_CLIENT check_client( @@ -354,7 +354,7 @@ namespace snmalloc return true; } - bool debug_different_slab(CapPtr n) + bool debug_different_slab(CapPtr n) { for (size_t i = 0; i < LENGTH; i++) { @@ -367,7 +367,7 @@ namespace snmalloc /** * Adds an element to the builder */ - void add(CapPtr n, LocalEntropy& entropy) + void add(CapPtr n, LocalEntropy& entropy) { SNMALLOC_ASSERT(!debug_different_slab(n) || empty()); @@ -394,11 +394,11 @@ namespace snmalloc { uint16_t local_prev = HEAD_KEY; EncodeFreeObjectReference* iter = &head[i]; - CapPtr prev_obj = iter->read(local_prev, entropy); + CapPtr prev_obj = iter->read(local_prev, entropy); uint16_t local_curr = initial_key(prev_obj) & 0xffff; while (end[i] != iter) { - CapPtr next = iter->read(local_prev, entropy); + CapPtr next = iter->read(local_prev, entropy); check_client(!different_slab(next, prev_obj), "Heap corruption"); local_prev = local_curr; local_curr = address_cast(next) & 0xffff; diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index 1877961..c1ea9be 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -122,7 +122,7 @@ namespace snmalloc } static bool - dealloc(CapPtr self, CapPtr p) + dealloc(CapPtr self, CapPtr p) { SNMALLOC_ASSERT(self->head > 0); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 2173786..7dc5fcd 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -178,31 +178,32 @@ namespace snmalloc self->free_queue.close(fast_free_list, entropy); auto n = fast_free_list.take(entropy); auto n_slab = Aal::capptr_rebound(self.as_void(), n); + auto meta = Metaslab::get_slab(n_slab); entropy.refresh_bits(); // Treat stealing the free list as allocating it all. self->remove(); - self->set_full(Metaslab::get_slab(n_slab)); + self->set_full(meta); auto p = remove_cache_friendly_offset(n, self->sizeclass()); SNMALLOC_ASSERT(is_start_of_object(self, address_cast(p))); - self->debug_slab_invariant(Metaslab::get_slab(n_slab), entropy); + self->debug_slab_invariant(meta, entropy); if constexpr (zero_mem == YesZero) { if (rsize < PAGE_ALIGNED_SIZE) pal_zero(p, rsize); else - pal_zero(p, rsize); + pal_zero(Aal::capptr_rebound(self.as_void(), p), rsize); } else { UNUSED(rsize); } - return capptr_export(Aal::capptr_bound(p, rsize)); + return capptr_export(p); } template diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 764cd0f..0a5d86d 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -18,8 +18,8 @@ namespace snmalloc using alloc_id_t = size_t; union { - CapPtr non_atomic_next; - AtomicCapPtr next{nullptr}; + CapPtr non_atomic_next; + AtomicCapPtr next{nullptr}; }; /* @@ -72,7 +72,7 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. alignas(CACHELINE_SIZE) - MPSCQ message_queue; + MPSCQ message_queue; alloc_id_t trunc_id() { diff --git a/src/mem/slab.h b/src/mem/slab.h index b398c74..42694ac 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -50,36 +50,42 @@ namespace snmalloc // Structure to represent the temporary list elements struct PreAllocObject { - CapPtr next; + CapPtr next; }; // The following code implements Sattolo's algorithm for generating // random cyclic permutations. This implementation is in the opposite // direction, so that the original space does not need initialising. This // is described as outside-in without citation on Wikipedia, appears to be // Folklore algorithm. - auto curr = + + // Note the wide bounds on curr relative to each of the ->next fields; + // curr is not persisted once the list is built. + CapPtr curr = pointer_offset(bumpptr, 0).template as_static(); - curr->next = curr; + curr->next = Aal::capptr_bound(curr, rsize); + uint16_t count = 1; - for (auto p = pointer_offset(bumpptr, rsize) - .template as_static(); - p.as_void() < slab_end; - p = pointer_offset(p, rsize).template as_static()) + for (curr = + pointer_offset(curr, rsize).template as_static(); + curr.as_void() < slab_end; + curr = + pointer_offset(curr, rsize).template as_static()) { size_t insert_index = entropy.sample(count); - p->next = std::exchange( + curr->next = std::exchange( pointer_offset(bumpptr, insert_index * rsize) .template as_static() ->next, - p); + Aal::capptr_bound(curr, rsize)); count++; } // Pick entry into space, and then build linked list by traversing cycle - // to the start. + // to the start. Use ->next to jump from CBArena to CBAlloc. auto start_index = entropy.sample(count); auto start_ptr = pointer_offset(bumpptr, start_index * rsize) - .template as_static(); + .template as_static() + ->next; auto curr_ptr = start_ptr; do { @@ -89,7 +95,7 @@ namespace snmalloc #else for (auto p = bumpptr; p < slab_end; p = pointer_offset(p, rsize)) { - b.add(FreeObject::make(p.as_void()), entropy); + b.add(Aal::capptr_bound(p, rsize), entropy); } #endif // This code consumes everything up to slab_end. @@ -105,7 +111,7 @@ namespace snmalloc static SNMALLOC_FAST_PATH bool dealloc_fast( CapPtr self, CapPtr super, - CapPtr p, + CapPtr p, LocalEntropy& entropy) { auto meta = super->get_meta(self); @@ -128,7 +134,7 @@ namespace snmalloc CapPtr self, SlabList* sl, CapPtr super, - CapPtr p, + CapPtr p, LocalEntropy& entropy) { auto meta = super->get_meta(self); @@ -137,7 +143,9 @@ namespace snmalloc if (meta->is_full()) { auto allocated = get_slab_capacity( - meta->sizeclass(), Metaslab::is_short(Metaslab::get_slab(p))); + meta->sizeclass(), + Metaslab::is_short( + Metaslab::get_slab(Aal::capptr_rebound(super.as_void(), p)))); // We are not on the sizeclass list. if (allocated == 1) {