From 005f5787ef9a138ce849205ebdbee496679392cd Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Fri, 5 Mar 2021 16:46:25 +0000 Subject: [PATCH] SP: start plumbing CapPtr<>s --- src/mem/alloc.h | 412 +++++++++++++++++++------------ src/mem/chunkmap.h | 36 ++- src/mem/freelist.h | 66 ++--- src/mem/largealloc.h | 10 +- src/mem/mediumslab.h | 37 +-- src/mem/metaslab.h | 42 ++-- src/mem/slab.h | 86 ++++--- src/mem/superslab.h | 27 +- src/pal/pal.h | 17 +- src/test/func/sandbox/sandbox.cc | 2 +- 10 files changed, 431 insertions(+), 304 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 8422ad4..50aed78 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -105,7 +105,7 @@ namespace snmalloc * If aligned to a SLAB start, then it is empty, and a new * slab is required. */ - void* bump_ptrs[NUM_SMALL_CLASSES] = {nullptr}; + CapPtr bump_ptrs[NUM_SMALL_CLASSES] = {nullptr}; public: Stats& stats() @@ -210,29 +210,32 @@ namespace snmalloc * external pointer. */ template - void dealloc(void* p) + void dealloc(void* p_raw) { #ifdef SNMALLOC_PASS_THROUGH UNUSED(size); - return external_alloc::free(p); + return external_alloc::free(p_raw); #else constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size); + auto p_ret = CapPtr(p_raw); + auto p_auth = CapPtr(p_raw); + if (sizeclass < NUM_SMALL_CLASSES) { - Superslab* super = Superslab::get(p); + auto super = Superslab::get(p_auth); - small_dealloc_unchecked(super, p, sizeclass); + small_dealloc_unchecked(super, p_auth, p_ret, sizeclass); } else if (sizeclass < NUM_SIZECLASSES) { - Mediumslab* slab = Mediumslab::get(p); + auto slab = Mediumslab::get(p_auth); - medium_dealloc_unchecked(slab, p, sizeclass); + medium_dealloc_unchecked(slab, p_auth, p_ret, sizeclass); } else { - large_dealloc_unchecked(p, size); + large_dealloc_unchecked(p_auth, p_ret, size); } #endif } @@ -241,53 +244,59 @@ namespace snmalloc * Free memory of a dynamically known size. Must be called with an * external pointer. */ - SNMALLOC_FAST_PATH void dealloc(void* p, size_t size) + SNMALLOC_FAST_PATH void dealloc(void* p_raw, size_t size) { #ifdef SNMALLOC_PASS_THROUGH UNUSED(size); - return external_alloc::free(p); + return external_alloc::free(p_raw); #else - SNMALLOC_ASSERT(p != nullptr); + SNMALLOC_ASSERT(p_raw != nullptr); + + auto p_ret = CapPtr(p_raw); + auto p_auth = CapPtr(p_raw); + if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1))) { - Superslab* super = Superslab::get(p); + auto super = Superslab::get(p_auth); sizeclass_t sizeclass = size_to_sizeclass(size); - small_dealloc_unchecked(super, p, sizeclass); + small_dealloc_unchecked(super, p_auth, p_ret, sizeclass); return; } - dealloc_sized_slow(p, size); + dealloc_sized_slow(p_auth, p_ret, size); #endif } - SNMALLOC_SLOW_PATH void dealloc_sized_slow(void* p, size_t size) + SNMALLOC_SLOW_PATH void dealloc_sized_slow( + CapPtr p_auth, CapPtr p_ret, size_t size) { if (size == 0) - return dealloc(p, 1); + return dealloc(p_ret.unsafe_capptr, 1); if (likely(size <= sizeclass_to_size(NUM_SIZECLASSES - 1))) { - Mediumslab* slab = Mediumslab::get(p); + auto slab = Mediumslab::get(p_auth); sizeclass_t sizeclass = size_to_sizeclass(size); - medium_dealloc_unchecked(slab, p, sizeclass); + medium_dealloc_unchecked(slab, p_auth, p_ret, sizeclass); return; } - large_dealloc_unchecked(p, size); + large_dealloc_unchecked(p_auth, p_ret, size); } /* * Free memory of an unknown size. Must be called with an external * pointer. */ - SNMALLOC_FAST_PATH void dealloc(void* p) + SNMALLOC_FAST_PATH void dealloc(void* p_raw) { #ifdef SNMALLOC_PASS_THROUGH - return external_alloc::free(p); + return external_alloc::free(p_raw); #else - uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p)); + uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p_raw)); - Superslab* super = Superslab::get(p); + auto p_ret = CapPtr(p_raw); + auto p_auth = CapPtr(p_raw); if (likely(chunkmap_slab_kind == CMSuperslab)) { @@ -305,22 +314,25 @@ namespace snmalloc * through the guard in small_dealloc_start(), we must treat this as * possibly stale and suspect. */ - Slab* slab = Metaslab::get_slab(p); - Metaslab& meta = super->get_meta(slab); - sizeclass_t sizeclass = meta.sizeclass(); + auto super = Superslab::get(p_auth); + auto slab = Metaslab::get_slab(p_auth); + auto meta = super->get_meta(slab); + sizeclass_t sizeclass = meta->sizeclass(); - small_dealloc_checked_sizeclass(super, slab, p, sizeclass); + small_dealloc_checked_sizeclass(super, slab, p_auth, p_ret, sizeclass); return; } - dealloc_not_small(p, chunkmap_slab_kind); + dealloc_not_small(p_auth, p_ret, chunkmap_slab_kind); } - SNMALLOC_SLOW_PATH void - dealloc_not_small(void* p, uint8_t chunkmap_slab_kind) + SNMALLOC_SLOW_PATH void dealloc_not_small( + CapPtr p_auth, + CapPtr p_ret, + uint8_t chunkmap_slab_kind) { handle_message_queue(); - if (p == nullptr) + if (p_ret == nullptr) return; if (chunkmap_slab_kind == CMMediumslab) @@ -330,10 +342,10 @@ namespace snmalloc * values are suspect until we complete the double-free check in * medium_dealloc_smart(). */ - Mediumslab* slab = Mediumslab::get(p); + auto slab = Mediumslab::get(p_auth); sizeclass_t sizeclass = slab->get_sizeclass(); - medium_dealloc_checked_sizeclass(slab, p, sizeclass); + medium_dealloc_checked_sizeclass(slab, p_auth, p_ret, sizeclass); return; } @@ -343,41 +355,46 @@ namespace snmalloc } large_dealloc_checked_sizeclass( - p, bits::one_at_bit(chunkmap_slab_kind), chunkmap_slab_kind); + p_auth, + p_ret, + bits::one_at_bit(chunkmap_slab_kind), + chunkmap_slab_kind); #endif } template - void* external_pointer(void* p) + void* external_pointer(void* p_raw) { #ifdef SNMALLOC_PASS_THROUGH error("Unsupported"); - UNUSED(p); + UNUSED(p_raw); #else - uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p)); + uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p_raw)); + auto p_ret = CapPtr(p_raw); + auto p_auth = CapPtr(p_raw); - Superslab* super = Superslab::get(p); + auto super = Superslab::get(p_auth); if (chunkmap_slab_kind == CMSuperslab) { - Slab* slab = Metaslab::get_slab(p); - Metaslab& meta = super->get_meta(slab); + auto slab = Metaslab::get_slab(p_auth); + auto meta = super->get_meta(slab); - sizeclass_t sc = meta.sizeclass(); - void* slab_end = pointer_offset(slab, SLAB_SIZE); + sizeclass_t sc = meta->sizeclass(); + auto slab_end = pointer_offset(slab, SLAB_SIZE); - return external_pointer(p, sc, slab_end); + return capptr_reveal(external_pointer(p_ret, sc, slab_end)); } if (chunkmap_slab_kind == CMMediumslab) { - Mediumslab* slab = Mediumslab::get(p); + auto slab = Mediumslab::get(p_auth); sizeclass_t sc = slab->get_sizeclass(); - void* slab_end = pointer_offset(slab, SUPERSLAB_SIZE); + auto slab_end = pointer_offset(slab, SUPERSLAB_SIZE); - return external_pointer(p, sc, slab_end); + return capptr_reveal(external_pointer(p_ret, sc, slab_end)); } - auto ss = super; + auto ss = super.unsafe_capptr; while (chunkmap_slab_kind >= CMLargeRangeMin) { @@ -386,7 +403,7 @@ namespace snmalloc ss, -(static_cast(1) << (chunkmap_slab_kind - CMLargeRangeMin + SUPERSLAB_BITS))); - chunkmap_slab_kind = chunkmap().get(ss); + chunkmap_slab_kind = chunkmap().get(address_cast(ss)); } if (chunkmap_slab_kind == CMNotOurs) @@ -420,29 +437,30 @@ namespace snmalloc } public: - SNMALLOC_FAST_PATH size_t alloc_size(const void* p) + SNMALLOC_FAST_PATH size_t alloc_size(const void* p_raw) { #ifdef SNMALLOC_PASS_THROUGH - return external_alloc::malloc_usable_size(const_cast(p)); + return external_alloc::malloc_usable_size(const_cast(p_raw)); #else // This must be called on an external pointer. - size_t chunkmap_slab_kind = chunkmap().get(address_cast(p)); + size_t chunkmap_slab_kind = chunkmap().get(address_cast(p_raw)); + auto p_auth = CapPtr(const_cast(p_raw)); if (likely(chunkmap_slab_kind == CMSuperslab)) { - Superslab* super = Superslab::get(p); + auto super = Superslab::get(p_auth); // Reading a remote sizeclass won't fail, since the other allocator // can't reuse the slab, as we have no yet deallocated this pointer. - Slab* slab = Metaslab::get_slab(p); - Metaslab& meta = super->get_meta(slab); + auto slab = Metaslab::get_slab(p_auth); + auto meta = super->get_meta(slab); - return sizeclass_to_size(meta.sizeclass()); + return sizeclass_to_size(meta->sizeclass()); } if (likely(chunkmap_slab_kind == CMMediumslab)) { - Mediumslab* slab = Mediumslab::get(p); + auto slab = Mediumslab::get(p_auth); // Reading a remote sizeclass won't fail, since the other allocator // can't reuse the slab, as we have no yet deallocated this pointer. return sizeclass_to_size(slab->get_sizeclass()); @@ -576,7 +594,7 @@ namespace snmalloc if (!l->empty()) { // Send all slots to the target at the head of the list. - Superslab* super = Superslab::get(first); + auto super = Superslab::get(CapPtr(first)); super->get_allocator()->message_queue.enqueue(first, l->last); l->clear(); } @@ -611,7 +629,7 @@ namespace snmalloc }; SlabList small_classes[NUM_SMALL_CLASSES]; - DLList medium_classes[NUM_MEDIUM_CLASSES]; + DLList medium_classes[NUM_MEDIUM_CLASSES]; DLList super_available; DLList super_only_short_available; @@ -747,8 +765,8 @@ namespace snmalloc auto rsize = sizeclass_to_size(i); FreeListIter ffl; - Superslab* super = Superslab::get(bp); - Slab* slab = Metaslab::get_slab(bp); + auto super = Superslab::get(bp); + auto slab = Metaslab::get_slab(bp); while (pointer_align_up(bp, SLAB_SIZE) != bp) { Slab::alloc_new_list(bp, ffl, rsize, entropy); @@ -789,24 +807,28 @@ namespace snmalloc } template - static void* - external_pointer(void* p, sizeclass_t sizeclass, void* end_point) + static CapPtr external_pointer( + CapPtr p_ret, + sizeclass_t sizeclass, + CapPtr end_point) { size_t rsize = sizeclass_to_size(sizeclass); - void* end_point_correction = location == End ? + auto end_point_correction = location == End ? pointer_offset_signed(end_point, -1) : (location == OnePastEnd ? end_point : pointer_offset_signed(end_point, -static_cast(rsize))); size_t offset_from_end = - pointer_diff(p, pointer_offset_signed(end_point, -1)); + pointer_diff(p_ret, pointer_offset_signed(end_point, -1)); size_t end_to_end = round_by_sizeclass(sizeclass, offset_from_end); - return pointer_offset_signed( + auto ret_auth = pointer_offset_signed( end_point_correction, -static_cast(end_to_end)); + + return CapPtr(ret_auth.unsafe_capptr); } void init_message_queue() @@ -827,14 +849,17 @@ namespace snmalloc if (likely(p->trunc_target_id() == get_trunc_id())) { // Destined for my slabs - Superslab* super = Superslab::get(p); + auto p_auth = CapPtr(p); + auto super = Superslab::get(p_auth); check_client( p->trunc_target_id() == super->get_allocator()->trunc_id(), "Detected memory corruption. Potential use-after-free"); - void* start = remove_cache_friendly_offset(p, p->sizeclass()); - dealloc_not_large_local(super, start, p, p->sizeclass()); + auto start_auth = CapPtr( + remove_cache_friendly_offset(p_auth.unsafe_capptr, p->sizeclass())); + dealloc_not_large_local( + super, start_auth, p_auth.as_void(), p->sizeclass()); } else { @@ -843,14 +868,17 @@ namespace snmalloc } } - SNMALLOC_SLOW_PATH void - dealloc_not_large(RemoteAllocator* target, void* p, sizeclass_t sizeclass) + SNMALLOC_SLOW_PATH void dealloc_not_large( + RemoteAllocator* target, + CapPtr p_auth, + sizeclass_t sizeclass) { - void* offseted = apply_cache_friendly_offset(p, sizeclass); + auto offseted = CapPtr( + apply_cache_friendly_offset(p_auth.unsafe_capptr, sizeclass)); if (likely(target->trunc_id() == get_trunc_id())) { - Superslab* super = Superslab::get(p); - dealloc_not_large_local(super, p, offseted, sizeclass); + auto super = Superslab::get(p_auth); + dealloc_not_large_local(super, p_auth, offseted, sizeclass); } else { @@ -859,7 +887,10 @@ namespace snmalloc } SNMALLOC_FAST_PATH void dealloc_not_large_local( - Superslab* super, void* p, void* offseted, sizeclass_t sizeclass) + CapPtr super, + CapPtr p_auth, + CapPtr p_auth_offseted, + sizeclass_t sizeclass) { // Guard against remote queues that have colliding IDs SNMALLOC_ASSERT(super->get_allocator() == public_state()); @@ -867,13 +898,15 @@ namespace snmalloc if (likely(sizeclass < NUM_SMALL_CLASSES)) { SNMALLOC_ASSERT(super->get_kind() == Super); - Slab* slab = Metaslab::get_slab(p); - small_dealloc_offseted(super, slab, offseted, sizeclass); + auto slab = Metaslab::get_slab(p_auth); + small_dealloc_offseted( + super, slab, FreeObject::make(p_auth_offseted), sizeclass); } else { SNMALLOC_ASSERT(super->get_kind() == Medium); - medium_dealloc_local(Mediumslab::get(p), p, sizeclass); + medium_dealloc_local( + super.template as_reinterpret(), p_auth, sizeclass); } } @@ -930,7 +963,7 @@ namespace snmalloc return super; super->init(public_state()); - chunkmap().set_slab(super); + chunkmap().set_slab(CapPtr(super)); super_available.insert(super); return super; } @@ -969,7 +1002,7 @@ namespace snmalloc } } - SNMALLOC_SLOW_PATH Slab* alloc_slab(sizeclass_t sizeclass) + SNMALLOC_SLOW_PATH CapPtr alloc_slab(sizeclass_t sizeclass) { stats().sizeclass_alloc_slab(sizeclass); if (Superslab::is_short_sizeclass(sizeclass)) @@ -982,7 +1015,7 @@ namespace snmalloc { auto slab = Superslab::alloc_short_slab(super, sizeclass); SNMALLOC_ASSERT(super->is_full()); - return slab; + return CapPtr(slab); } super = get_superslab(); @@ -992,7 +1025,7 @@ namespace snmalloc auto slab = Superslab::alloc_short_slab(super, sizeclass); reposition_superslab(super); - return slab; + return CapPtr(slab); } Superslab* super = get_superslab(); @@ -1002,7 +1035,7 @@ namespace snmalloc auto slab = Superslab::alloc_slab(super, sizeclass); reposition_superslab(super); - return slab; + return CapPtr(slab); } template @@ -1023,7 +1056,8 @@ namespace snmalloc { stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); - void* p = remove_cache_friendly_offset(fl.take(entropy), sizeclass); + void* p = remove_cache_friendly_offset( + fl.take(entropy).unsafe_capptr, sizeclass); if constexpr (zero_mem == YesZero) { pal_zero( @@ -1066,7 +1100,7 @@ namespace snmalloc stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); - auto meta = reinterpret_cast(sl.get_next()); + auto meta = sl.get_next().template as_static(); auto& ffl = small_fast_free_lists[sizeclass]; return Metaslab::alloc( meta, ffl, rsize, entropy); @@ -1135,7 +1169,8 @@ namespace snmalloc SNMALLOC_ASSERT(ffl.empty()); Slab::alloc_new_list(bp, ffl, rsize, entropy); - auto p = remove_cache_friendly_offset(ffl.take(entropy), sizeclass); + auto p = remove_cache_friendly_offset( + ffl.take(entropy).unsafe_capptr, sizeclass); if constexpr (zero_mem == YesZero) { @@ -1154,64 +1189,87 @@ namespace snmalloc { auto& bp = bump_ptrs[sizeclass]; // Fetch new slab - Slab* slab = alloc_slab(sizeclass); + auto slab = alloc_slab(sizeclass); if (slab == nullptr) return nullptr; - bp = pointer_offset( + bp = pointer_offset( slab, get_initial_offset(sizeclass, Metaslab::is_short(slab))); return small_alloc_build_free_list(sizeclass); } - SNMALLOC_FAST_PATH void - small_dealloc_unchecked(Superslab* super, void* p, sizeclass_t sizeclass) + SNMALLOC_FAST_PATH void small_dealloc_unchecked( + CapPtr super, + CapPtr p_auth, + CapPtr p_ret, + sizeclass_t sizeclass) { check_client( - chunkmap().get(address_cast(p)) == CMSuperslab, + chunkmap().get(address_cast(p_ret)) == CMSuperslab, "Claimed small deallocation is not in a Superslab"); - small_dealloc_checked_chunkmap(super, p, sizeclass); + small_dealloc_checked_chunkmap(super, p_auth, p_ret, sizeclass); } SNMALLOC_FAST_PATH void small_dealloc_checked_chunkmap( - Superslab* super, void* p, sizeclass_t sizeclass) + CapPtr super, + CapPtr p_auth, + CapPtr p_ret, + sizeclass_t sizeclass) { - Slab* slab = Metaslab::get_slab(p); + auto slab = Metaslab::get_slab(p_auth); check_client( - sizeclass == super->get_meta(slab).sizeclass(), + sizeclass == super->get_meta(slab)->sizeclass(), "Claimed small deallocation with mismatching size class"); - small_dealloc_checked_sizeclass(super, slab, p, sizeclass); + small_dealloc_checked_sizeclass(super, slab, p_auth, p_ret, sizeclass); } SNMALLOC_FAST_PATH void small_dealloc_checked_sizeclass( - Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass) + CapPtr super, + CapPtr slab, + CapPtr p_auth, + CapPtr p_ret, + sizeclass_t sizeclass) { check_client( - Metaslab::is_start_of_object(&Slab::get_meta(slab), p), + Metaslab::is_start_of_object(Slab::get_meta(slab), address_cast(p_ret)), "Not deallocating start of an object"); - small_dealloc_start(super, slab, p, sizeclass); + small_dealloc_start(super, slab, p_auth, p_ret, sizeclass); } SNMALLOC_FAST_PATH void small_dealloc_start( - Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass) + CapPtr super, + CapPtr slab, + CapPtr p_auth, + CapPtr p_ret, + sizeclass_t sizeclass) { // TODO: with SSM/MTE, guard against double-frees + UNUSED(p_ret); RemoteAllocator* target = super->get_allocator(); if (likely(target == public_state())) { - void* offseted = apply_cache_friendly_offset(p, sizeclass); - small_dealloc_offseted(super, slab, offseted, sizeclass); + void* offseted = + apply_cache_friendly_offset(p_auth.unsafe_capptr, sizeclass); + small_dealloc_offseted( + super, + slab, + FreeObject::make(CapPtr(offseted)), + sizeclass); } else - remote_dealloc(target, p, sizeclass); + remote_dealloc(target, p_auth, sizeclass); } SNMALLOC_FAST_PATH void small_dealloc_offseted( - Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass) + CapPtr super, + CapPtr slab, + CapPtr p, + sizeclass_t sizeclass) { stats().sizeclass_dealloc(sizeclass); @@ -1219,7 +1277,10 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( - Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass) + CapPtr super, + CapPtr slab, + CapPtr p, + sizeclass_t sizeclass) { if (likely(Slab::dealloc_fast(slab, super, p, entropy))) return; @@ -1228,7 +1289,10 @@ namespace snmalloc } SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow( - Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass) + CapPtr super, + CapPtr slab, + CapPtr p, + sizeclass_t sizeclass) { bool was_full = super->is_full(); SlabList* sl = &small_classes[sizeclass]; @@ -1252,28 +1316,29 @@ namespace snmalloc { if (was_full) { - super_available.insert(super); + super_available.insert(super.unsafe_capptr); } else { - super_only_short_available.remove(super); - super_available.insert(super); + super_only_short_available.remove(super.unsafe_capptr); + super_available.insert(super.unsafe_capptr); } break; } case Superslab::OnlyShortSlabAvailable: { - super_only_short_available.insert(super); + super_only_short_available.insert(super.unsafe_capptr); break; } case Superslab::Empty: { - super_available.remove(super); + super_available.remove(super.unsafe_capptr); chunkmap().clear_slab(super); - large_allocator.dealloc(super, 0); + large_allocator.dealloc( + super.template as_reinterpret(), 0); stats().superslab_push(); break; } @@ -1285,9 +1350,9 @@ namespace snmalloc { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; - DLList* sc = &medium_classes[medium_class]; - Mediumslab* slab = sc->get_head(); - void* p; + auto sc = &medium_classes[medium_class]; + auto slab = sc->get_head(); + CapPtr p; if (slab != nullptr) { @@ -1306,9 +1371,9 @@ namespace snmalloc sizeclass, rsize, size); }); } - slab = - reinterpret_cast(large_allocator.template alloc( - 0, SUPERSLAB_SIZE, SUPERSLAB_SIZE)); + slab = CapPtr(large_allocator.template alloc( + 0, SUPERSLAB_SIZE, SUPERSLAB_SIZE)) + .template as_static(); if (slab == nullptr) return nullptr; @@ -1324,58 +1389,75 @@ namespace snmalloc stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); - return p; + return p.unsafe_capptr; } SNMALLOC_FAST_PATH - void - medium_dealloc_unchecked(Mediumslab* slab, void* p, sizeclass_t sizeclass) + void medium_dealloc_unchecked( + CapPtr slab, + CapPtr p_auth, + CapPtr p_ret, + sizeclass_t sizeclass) { check_client( - chunkmap().get(address_cast(p)) == CMMediumslab, + chunkmap().get(address_cast(p_ret)) == CMMediumslab, "Claimed medium deallocation is not in a Mediumslab"); - medium_dealloc_checked_chunkmap(slab, p, sizeclass); + medium_dealloc_checked_chunkmap(slab, p_auth, p_ret, sizeclass); } SNMALLOC_FAST_PATH void medium_dealloc_checked_chunkmap( - Mediumslab* slab, void* p, sizeclass_t sizeclass) + CapPtr slab, + CapPtr p_auth, + CapPtr p_ret, + sizeclass_t sizeclass) { check_client( slab->get_sizeclass() == sizeclass, "Claimed medium deallocation of the wrong sizeclass"); - medium_dealloc_checked_sizeclass(slab, p, sizeclass); + medium_dealloc_checked_sizeclass(slab, p_auth, p_ret, sizeclass); } SNMALLOC_FAST_PATH void medium_dealloc_checked_sizeclass( - Mediumslab* slab, void* p, sizeclass_t sizeclass) + CapPtr slab, + CapPtr p_auth, + CapPtr p_ret, + sizeclass_t sizeclass) { check_client( is_multiple_of_sizeclass( - sizeclass, pointer_diff(p, pointer_offset(slab, SUPERSLAB_SIZE))), + sizeclass, address_cast(slab) + SUPERSLAB_SIZE - address_cast(p_ret)), "Not deallocating start of an object"); - medium_dealloc_start(slab, p, sizeclass); + medium_dealloc_start(slab, p_auth, p_ret, sizeclass); } SNMALLOC_FAST_PATH - void medium_dealloc_start(Mediumslab* slab, void* p, sizeclass_t sizeclass) + void medium_dealloc_start( + CapPtr slab, + CapPtr p_auth, + CapPtr p_ret, + sizeclass_t sizeclass) { // TODO: with SSM/MTE, guard against double-frees + UNUSED(p_ret); RemoteAllocator* target = slab->get_allocator(); if (likely(target == public_state())) - medium_dealloc_local(slab, p, sizeclass); + medium_dealloc_local(slab, p_auth, sizeclass); else - remote_dealloc(target, p, sizeclass); + remote_dealloc(target, p_auth, sizeclass); } SNMALLOC_FAST_PATH - void medium_dealloc_local(Mediumslab* slab, void* p, sizeclass_t sizeclass) + void medium_dealloc_local( + CapPtr slab, + CapPtr p, + sizeclass_t sizeclass) { stats().sizeclass_dealloc(sizeclass); bool was_full = Mediumslab::dealloc(slab, p); @@ -1385,18 +1467,18 @@ namespace snmalloc if (!was_full) { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; - DLList* sc = &medium_classes[medium_class]; + auto sc = &medium_classes[medium_class]; sc->remove(slab); } chunkmap().clear_slab(slab); - large_allocator.dealloc(slab, 0); + large_allocator.dealloc(slab.template as_reinterpret(), 0); stats().superslab_push(); } else if (was_full) { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; - DLList* sc = &medium_classes[medium_class]; + auto sc = &medium_classes[medium_class]; sc->insert(slab); } } @@ -1433,7 +1515,8 @@ namespace snmalloc return p; } - void large_dealloc_unchecked(void* p, size_t size) + void large_dealloc_unchecked( + CapPtr p_auth, CapPtr p_ret, size_t size) { uint8_t claimed_chunkmap_slab_kind = static_cast(bits::next_pow2_bits(size)); @@ -1442,48 +1525,57 @@ namespace snmalloc // we're so far from the start that our actual chunkmap slab kind is not a // legitimate large class check_client( - chunkmap().get(address_cast(p)) == claimed_chunkmap_slab_kind, + chunkmap().get(address_cast(p_ret)) == claimed_chunkmap_slab_kind, "Claimed large deallocation with wrong size class"); // round up as we would if we had had to look up the chunkmap_slab_kind size_t rsize = bits::one_at_bit(claimed_chunkmap_slab_kind); - large_dealloc_checked_sizeclass(p, rsize, claimed_chunkmap_slab_kind); + large_dealloc_checked_sizeclass( + p_auth, p_ret, rsize, claimed_chunkmap_slab_kind); } void large_dealloc_checked_sizeclass( - void* p, size_t size, uint8_t chunkmap_slab_kind) + CapPtr p_auth, + CapPtr p_ret, + size_t size, + uint8_t chunkmap_slab_kind) { check_client( - address_cast(Superslab::get(p)) == address_cast(p), + address_cast(Superslab::get(p_auth)) == address_cast(p_ret), "Not deallocating start of an object"); SNMALLOC_ASSERT(bits::one_at_bit(chunkmap_slab_kind) >= SUPERSLAB_SIZE); - large_dealloc_start(p, size, chunkmap_slab_kind); + large_dealloc_start(p_auth, p_ret, size, chunkmap_slab_kind); } - void large_dealloc_start(void* p, size_t size, uint8_t chunkmap_slab_kind) + void large_dealloc_start( + CapPtr p_auth, + CapPtr p_ret, + size_t size, + uint8_t chunkmap_slab_kind) { // TODO: with SSM/MTE, guard against double-frees if (NeedsInitialisation(this)) { - InitThreadAllocator([p, size, chunkmap_slab_kind](void* alloc) { - reinterpret_cast(alloc)->large_dealloc_start( - p, size, chunkmap_slab_kind); - return nullptr; - }); + InitThreadAllocator( + [p_auth, p_ret, size, chunkmap_slab_kind](void* alloc) { + reinterpret_cast(alloc)->large_dealloc_start( + p_auth, p_ret, size, chunkmap_slab_kind); + return nullptr; + }); return; } size_t large_class = chunkmap_slab_kind - SUPERSLAB_BITS; - chunkmap().clear_large_size(p, size); + chunkmap().clear_large_size(p_auth, size); stats().large_dealloc(large_class); // Initialise in order to set the correct SlabKind. - Largeslab* slab = static_cast(p); + auto slab = p_auth.template as_static(); slab->init(); large_allocator.dealloc(slab, large_class); } @@ -1492,7 +1584,8 @@ namespace snmalloc // called in its slow path. This leads to one fewer unconditional jump in // Clang. SNMALLOC_FAST_PATH - void remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) + void remote_dealloc( + RemoteAllocator* target, CapPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id()); @@ -1501,7 +1594,8 @@ namespace snmalloc // this path. if (remote.capacity > 0) { - void* offseted = apply_cache_friendly_offset(p, sizeclass); + void* offseted = + apply_cache_friendly_offset(p.unsafe_capptr, sizeclass); stats().remote_free(sizeclass); remote.dealloc(target->trunc_id(), offseted, sizeclass); return; @@ -1510,8 +1604,10 @@ namespace snmalloc remote_dealloc_slow(target, p, sizeclass); } - SNMALLOC_SLOW_PATH void - remote_dealloc_slow(RemoteAllocator* target, void* p, sizeclass_t sizeclass) + SNMALLOC_SLOW_PATH void remote_dealloc_slow( + RemoteAllocator* target, + CapPtr p_auth, + sizeclass_t sizeclass) { SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id()); @@ -1520,24 +1616,26 @@ namespace snmalloc // a real allocator and construct one if we aren't. if (NeedsInitialisation(this)) { - InitThreadAllocator([target, p, sizeclass](void* alloc) { + InitThreadAllocator([target, p_auth, sizeclass](void* alloc) { reinterpret_cast(alloc)->dealloc_not_large( - target, p, sizeclass); + target, p_auth, sizeclass); return nullptr; }); return; } - remote_dealloc_and_post(target, p, sizeclass); + remote_dealloc_and_post(target, p_auth, sizeclass); } SNMALLOC_SLOW_PATH void remote_dealloc_and_post( - RemoteAllocator* target, void* offseted, sizeclass_t sizeclass) + RemoteAllocator* target, + CapPtr offseted, + sizeclass_t sizeclass) { handle_message_queue(); stats().remote_free(sizeclass); - remote.dealloc(target->trunc_id(), offseted, sizeclass); + remote.dealloc(target->trunc_id(), offseted.unsafe_capptr, sizeclass); stats().remote_post(); remote.post(&large_allocator, get_trunc_id()); diff --git a/src/mem/chunkmap.h b/src/mem/chunkmap.h index 1924d02..c8e462b 100644 --- a/src/mem/chunkmap.h +++ b/src/mem/chunkmap.h @@ -114,44 +114,36 @@ namespace snmalloc return PagemapProvider::pagemap().get(p); } - /** - * Get the pagemap entry corresponding to a specific address. - */ - static uint8_t get(void* p) - { - return get(address_cast(p)); - } - /** * Set a pagemap entry indicating that there is a superslab at the * specified index. */ - static void set_slab(Superslab* slab) + static void set_slab(CapPtr slab) { - set(slab, static_cast(CMSuperslab)); + set(address_cast(slab), static_cast(CMSuperslab)); } /** * Add a pagemap entry indicating that a medium slab has been allocated. */ - static void set_slab(Mediumslab* slab) + static void set_slab(CapPtr slab) { - set(slab, static_cast(CMMediumslab)); + set(address_cast(slab), static_cast(CMMediumslab)); } /** * Remove an entry from the pagemap corresponding to a superslab. */ - static void clear_slab(Superslab* slab) + static void clear_slab(CapPtr slab) { - SNMALLOC_ASSERT(get(slab) == CMSuperslab); - set(slab, static_cast(CMNotOurs)); + SNMALLOC_ASSERT(get(address_cast(slab)) == CMSuperslab); + set(address_cast(slab), static_cast(CMNotOurs)); } /** * Remove an entry corresponding to a medium slab. */ - static void clear_slab(Mediumslab* slab) + static void clear_slab(CapPtr slab) { - SNMALLOC_ASSERT(get(slab) == CMMediumslab); - set(slab, static_cast(CMNotOurs)); + SNMALLOC_ASSERT(get(address_cast(slab)) == CMMediumslab); + set(address_cast(slab), static_cast(CMNotOurs)); } /** * Update the pagemap to reflect a large allocation, of `size` bytes from @@ -160,7 +152,7 @@ namespace snmalloc static void set_large_size(void* p, size_t size) { size_t size_bits = bits::next_pow2_bits(size); - set(p, static_cast(size_bits)); + set(address_cast(p), static_cast(size_bits)); // Set redirect slide auto ss = address_cast(p) + SUPERSLAB_SIZE; for (size_t i = 0; i < size_bits - SUPERSLAB_BITS; i++) @@ -175,7 +167,7 @@ namespace snmalloc * Update the pagemap to remove a large allocation, of `size` bytes from * address `p`. */ - static void clear_large_size(void* vp, size_t size) + static void clear_large_size(CapPtr vp, size_t size) { auto p = address_cast(vp); size_t rounded_size = bits::next_pow2(size); @@ -190,9 +182,9 @@ namespace snmalloc * interface and exists to make it easy to reuse the code in the public * methods in other pagemap adaptors. */ - static void set(void* p, uint8_t x) + static void set(address_t p, uint8_t x) { - PagemapProvider::pagemap().set(address_cast(p), x); + PagemapProvider::pagemap().set(p, x); } }; diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 1f9035d..54de6ac 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -23,7 +23,8 @@ namespace snmalloc * Used to turn a location into a key. This is currently * just the slab address truncated to 16bits and offset by 1. */ - inline static address_t initial_key(void* slab) + template + inline static address_t initial_key(CapPtr slab) { #ifdef CHECK_CLIENT /** @@ -46,12 +47,15 @@ namespace snmalloc return ((p1 ^ p2) >= SLAB_SIZE); } - static inline bool different_slab(address_t p1, void* p2) + template + static inline bool different_slab(address_t p1, CapPtr p2) { return different_slab(p1, address_cast(p2)); } - static inline bool different_slab(void* p1, void* p2) + template + static inline bool + different_slab(CapPtr p1, CapPtr p2) { return different_slab(address_cast(p1), address_cast(p2)); } @@ -60,7 +64,7 @@ namespace snmalloc class EncodeFreeObjectReference { - FreeObject* reference; + CapPtr reference; /** * On architectures which use IntegerPointers, we can obfuscate our free @@ -78,8 +82,8 @@ namespace snmalloc public: #ifdef CHECK_CLIENT template - static std::enable_if_t - encode(uint16_t local_key, T* next_object, LocalEntropy& entropy) + static std::enable_if_t> encode( + uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) { // Simple involutional encoding. The bottom half of each word is // multiplied by a function of both global and local keys (the latter, @@ -92,25 +96,28 @@ namespace snmalloc address_t key = (local_key + 1) * entropy.get_constant_key(); next ^= (((next & MASK) + 1) * key) & ~(bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1); - return reinterpret_cast(next); + return CapPtr(reinterpret_cast(next)); } #endif template - static std::enable_if_t - encode(uint16_t local_key, T* next_object, LocalEntropy& entropy) + static std::enable_if_t> encode( + uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) { UNUSED(local_key); UNUSED(entropy); return next_object; } - void store(FreeObject* value, uint16_t local_key, LocalEntropy& entropy) + void store( + CapPtr value, + uint16_t local_key, + LocalEntropy& entropy) { reference = encode(local_key, value, entropy); } - FreeObject* read(uint16_t local_key, LocalEntropy& entropy) + CapPtr read(uint16_t local_key, LocalEntropy& entropy) { return encode(local_key, reference, entropy); } @@ -129,15 +136,15 @@ namespace snmalloc public: EncodeFreeObjectReference next_object; - static FreeObject* make(void* p) + static CapPtr make(CapPtr p) { - return static_cast(p); + return p.template as_static(); } /** * Read the next pointer handling any required decoding of the pointer */ - FreeObject* read_next(uint16_t key, LocalEntropy& entropy) + CapPtr read_next(uint16_t key, LocalEntropy& entropy) { return next_object.read(key, entropy); } @@ -150,7 +157,7 @@ namespace snmalloc */ class FreeListIter { - FreeObject* curr = nullptr; + CapPtr curr = nullptr; #ifdef CHECK_CLIENT address_t prev = 0; #endif @@ -170,7 +177,7 @@ namespace snmalloc * Currently this is just the value of current before this call. * Other schemes could be used. */ - void update_cursor(FreeObject* next) + void update_cursor(CapPtr next) { #ifdef CHECK_CLIENT # ifndef NDEBUG @@ -187,7 +194,7 @@ namespace snmalloc } public: - FreeListIter(FreeObject* head) + FreeListIter(CapPtr head) : curr(head) #ifdef CHECK_CLIENT , @@ -210,7 +217,7 @@ namespace snmalloc /** * Returns current head without affecting the iterator. */ - void* peek() + CapPtr peek() { return curr; } @@ -218,7 +225,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - void* take(LocalEntropy& entropy) + CapPtr take(LocalEntropy& entropy) { #ifdef CHECK_CLIENT check_client( @@ -309,7 +316,7 @@ namespace snmalloc * Start building a new free list. * Provide pointer to the slab to initialise the system. */ - void open(void* p) + void open(CapPtr p) { SNMALLOC_ASSERT(empty()); for (size_t i = 0; i < LENGTH; i++) @@ -331,17 +338,17 @@ namespace snmalloc { for (size_t i = 0; i < LENGTH; i++) { - if (end[i] != &head[i]) + if (address_cast(end[i]) != address_cast(&head[i])) return false; } return true; } - bool debug_different_slab(void* n) + bool debug_different_slab(CapPtr n) { for (size_t i = 0; i < LENGTH; i++) { - if (!different_slab(end[i], n)) + if (!different_slab(address_cast(end[i]), n)) return false; } return true; @@ -350,18 +357,17 @@ namespace snmalloc /** * Adds an element to the builder */ - void add(void* n, LocalEntropy& entropy) + void add(CapPtr n, LocalEntropy& entropy) { SNMALLOC_ASSERT(!debug_different_slab(n) || empty()); - FreeObject* next = FreeObject::make(n); auto index = RANDOM ? entropy.next_bit() : 0; - end[index]->store(next, get_prev(index), entropy); - end[index] = &(next->next_object); + end[index]->store(n, get_prev(index), entropy); + end[index] = &(n->next_object); #ifdef CHECK_CLIENT prev[index] = curr[index]; - curr[index] = address_cast(next) & 0xffff; + curr[index] = address_cast(n) & 0xffff; #endif } @@ -378,11 +384,11 @@ namespace snmalloc { uint16_t local_prev = HEAD_KEY; EncodeFreeObjectReference* iter = &head[i]; - FreeObject* prev_obj = iter->read(local_prev, entropy); + CapPtr prev_obj = iter->read(local_prev, entropy); uint16_t local_curr = initial_key(prev_obj) & 0xffff; while (end[i] != iter) { - FreeObject* next = iter->read(local_prev, entropy); + CapPtr next = iter->read(local_prev, entropy); check_client(!different_slab(next, prev_obj), "Heap corruption"); local_prev = local_curr; local_curr = address_cast(next) & 0xffff; diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index e381db3..4bec104 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -115,11 +115,11 @@ namespace snmalloc * class specified by `large_class`. Always succeeds. */ SNMALLOC_FAST_PATH void - push_large_stack(Largeslab* slab, size_t large_class) + push_large_stack(CapPtr slab, size_t large_class) { const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; available_large_chunks_in_bytes += rsize; - large_stack[large_class].push(slab); + large_stack[large_class].push(slab.unsafe_capptr); } /** @@ -342,7 +342,7 @@ namespace snmalloc return p; } - void dealloc(void* p, size_t large_class) + void dealloc(CapPtr p, size_t large_class) { if constexpr (decommit_strategy == DecommitSuperLazy) { @@ -360,11 +360,11 @@ namespace snmalloc (large_class != 0 || decommit_strategy == DecommitSuper)) { MemoryProvider::Pal::notify_not_using( - pointer_offset(p, OS_PAGE_SIZE), rsize - OS_PAGE_SIZE); + pointer_offset(p, OS_PAGE_SIZE).unsafe_capptr, rsize - OS_PAGE_SIZE); } stats.superslab_push(); - memory_provider.push_large_stack(static_cast(p), large_class); + memory_provider.push_large_stack(p, large_class); } }; diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index 585a67a..02028b7 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -12,12 +12,12 @@ namespace snmalloc // This is the view of a 16 mb area when it is being used to allocate // medium sized classes: 64 kb to 16 mb, non-inclusive. private: - friend DLList; + friend DLList; // Keep the allocator pointer on a separate cache line. It is read by // other threads, and does not change, so we avoid false sharing. - alignas(CACHELINE_SIZE) Mediumslab* next; - Mediumslab* prev; + alignas(CACHELINE_SIZE) CapPtr next; + CapPtr prev; uint16_t free; uint8_t head; @@ -44,16 +44,19 @@ namespace snmalloc return bits::align_up(sizeof(Mediumslab), min(OS_PAGE_SIZE, SLAB_SIZE)); } - static Mediumslab* get(const void* p) + template + static SNMALLOC_FAST_PATH CapPtr get(CapPtr p) { - return pointer_align_down( - const_cast(p)); + static_assert(B == CBArena || B == CBChunk); + + return pointer_align_down(p); } - // This is pre-factored to take an explicit self parameter so that we can - // eventually annotate that pointer with additional information. - static void - init(Mediumslab* self, RemoteAllocator* alloc, sizeclass_t sc, size_t rsize) + static void init( + CapPtr self, + RemoteAllocator* alloc, + sizeclass_t sc, + size_t rsize) { SNMALLOC_ASSERT(sc >= NUM_SMALL_CLASSES); SNMALLOC_ASSERT((sc - NUM_SMALL_CLASSES) < NUM_MEDIUM_CLASSES); @@ -85,12 +88,13 @@ namespace snmalloc } template - static void* alloc(Mediumslab* self, size_t size) + static CapPtr + alloc(CapPtr self, size_t size) { SNMALLOC_ASSERT(!full(self)); uint16_t index = self->stack[self->head++]; - void* p = pointer_offset(self, (static_cast(index) << 8)); + auto p = pointer_offset(self, (static_cast(index) << 8)); self->free--; if constexpr (zero_mem == YesZero) @@ -101,7 +105,8 @@ namespace snmalloc return p; } - static bool dealloc(Mediumslab* self, void* p) + static bool + dealloc(CapPtr self, CapPtr p) { SNMALLOC_ASSERT(self->head > 0); @@ -113,12 +118,14 @@ namespace snmalloc return was_full; } - static bool full(Mediumslab* self) + template + static bool full(CapPtr self) { return self->free == 0; } - static bool empty(Mediumslab* self) + template + static bool empty(CapPtr self) { return self->head == 0; } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index b815978..fb9d993 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -10,8 +10,8 @@ namespace snmalloc { class Slab; - using SlabList = CDLLNode<>; - using SlabLink = CDLLNode<>; + using SlabList = CDLLNode; + using SlabLink = CDLLNode; static_assert( sizeof(SlabLink) <= MIN_ALLOC_SIZE, @@ -71,7 +71,7 @@ namespace snmalloc return free_queue.s.next; } - void initialise(sizeclass_t sizeclass, Slab* slab) + void initialise(sizeclass_t sizeclass, CapPtr slab) { free_queue.s.sizeclass = static_cast(sizeclass); free_queue.init(); @@ -120,12 +120,12 @@ namespace snmalloc return bits::min(threshold, max); } - SNMALLOC_FAST_PATH void set_full(Slab* slab) + SNMALLOC_FAST_PATH void set_full(CapPtr slab) { SNMALLOC_ASSERT(free_queue.empty()); // Prepare for the next free queue to be built. - free_queue.open(slab); + free_queue.open(slab.as_void()); // Set needed to at least one, possibly more so we only use // a slab when it has a reasonable amount of free elements @@ -133,34 +133,33 @@ namespace snmalloc null_prev(); } - static Slab* get_slab(const void* p) + template + static CapPtr get_slab(CapPtr p) { - return pointer_align_down(const_cast(p)); + return pointer_align_down(p.as_void()); } - static bool is_short(Slab* p) + template + static bool is_short(CapPtr p) { - return pointer_align_down(p) == p; + return pointer_align_down(p.as_void()) == p; } - SNMALLOC_FAST_PATH static bool is_start_of_object(Metaslab* self, void* p) + SNMALLOC_FAST_PATH + static bool is_start_of_object(CapPtr self, address_t p) { return is_multiple_of_sizeclass( - self->sizeclass(), - SLAB_SIZE - pointer_diff(pointer_align_down(p), p)); + self->sizeclass(), SLAB_SIZE - (p - address_align_down(p))); } /** * Takes a free list out of a slabs meta data. * Returns the link as the allocation, and places the free list into the * `fast_free_list` for further allocations. - * - * This is pre-factored to take an explicit self parameter so that we can - * eventually annotate that pointer with additional information. */ template static SNMALLOC_FAST_PATH void* alloc( - Metaslab* self, + CapPtr self, FreeListIter& fast_free_list, size_t rsize, LocalEntropy& entropy) @@ -169,7 +168,7 @@ namespace snmalloc SNMALLOC_ASSERT(!self->is_full()); self->free_queue.close(fast_free_list, entropy); - void* n = fast_free_list.take(entropy); + auto n = fast_free_list.take(entropy); entropy.refresh_bits(); @@ -177,10 +176,11 @@ namespace snmalloc self->remove(); self->set_full(Metaslab::get_slab(n)); - void* p = remove_cache_friendly_offset(n, self->sizeclass()); - SNMALLOC_ASSERT(is_start_of_object(self, p)); + void* p = + remove_cache_friendly_offset(n.unsafe_capptr, self->sizeclass()); + SNMALLOC_ASSERT(is_start_of_object(self, address_cast(p))); - self->debug_slab_invariant(Metaslab::get_slab(p), entropy); + self->debug_slab_invariant(Metaslab::get_slab(n), entropy); if constexpr (zero_mem == YesZero) { @@ -197,7 +197,7 @@ namespace snmalloc return p; } - void debug_slab_invariant(Slab* slab, LocalEntropy& entropy) + void debug_slab_invariant(CapPtr slab, LocalEntropy& entropy) { #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) bool is_short = Metaslab::is_short(slab); diff --git a/src/mem/slab.h b/src/mem/slab.h index c0ac680..b051135 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -17,9 +17,12 @@ namespace snmalloc } public: - static Metaslab& get_meta(Slab* self) + template + static CapPtr get_meta(CapPtr self) { - Superslab* super = Superslab::get(self); + static_assert(B == CBArena || B == CBChunk); + + auto super = Superslab::get(self); return super->get_meta(self); } @@ -30,12 +33,12 @@ namespace snmalloc * page. */ static SNMALLOC_FAST_PATH void alloc_new_list( - void*& bumpptr, + CapPtr& bumpptr, FreeListIter& fast_free_list, size_t rsize, LocalEntropy& entropy) { - void* slab_end = pointer_align_up(pointer_offset(bumpptr, 1)); + auto slab_end = pointer_align_up(pointer_offset(bumpptr, 1)); FreeListBuilder b; SNMALLOC_ASSERT(b.empty()); @@ -46,23 +49,27 @@ namespace snmalloc // Structure to represent the temporary list elements struct PreAllocObject { - PreAllocObject* next; + CapPtr next; }; // The following code implements Sattolo's algorithm for generating // random cyclic permutations. This implementation is in the opposite // direction, so that the original space does not need initialising. This // is described as outside-in without citation on Wikipedia, appears to be // Folklore algorithm. - PreAllocObject* curr = pointer_offset(bumpptr, 0); + auto curr = + pointer_offset(bumpptr, 0).template as_static(); curr->next = curr; uint16_t count = 1; - for (PreAllocObject* p = pointer_offset(bumpptr, rsize); - p < slab_end; - p = pointer_offset(p, rsize)) + for (auto p = pointer_offset(bumpptr, rsize) + .template as_static(); + p.as_void() < slab_end; + p = pointer_offset(p, rsize).template as_static()) { size_t insert_index = entropy.sample(count); p->next = std::exchange( - pointer_offset(bumpptr, insert_index * rsize)->next, + pointer_offset(bumpptr, insert_index * rsize) + .template as_static() + ->next, p); count++; } @@ -70,18 +77,18 @@ namespace snmalloc // Pick entry into space, and then build linked list by traversing cycle // to the start. auto start_index = entropy.sample(count); - auto start_ptr = - pointer_offset(bumpptr, start_index * rsize); + auto start_ptr = pointer_offset(bumpptr, start_index * rsize) + .template as_static(); auto curr_ptr = start_ptr; do { - b.add(curr_ptr, entropy); + b.add(FreeObject::make(curr_ptr.as_void()), entropy); curr_ptr = curr_ptr->next; } while (curr_ptr != start_ptr); #else - for (void* p = bumpptr; p < slab_end; p = pointer_offset(p, rsize)) + for (auto p = bumpptr; p < slab_end; p = pointer_offset(p, rsize)) { - b.add(p, entropy); + b.add(FreeObject::make(p.as_void()), entropy); } #endif // This code consumes everything up to slab_end. @@ -94,20 +101,20 @@ namespace snmalloc // Returns true, if it deallocation can proceed without changing any status // bits. Note that this does remove the use from the meta slab, so it // doesn't need doing on the slow path. - // - // This is pre-factored to take an explicit self parameter so that we can - // eventually annotate that pointer with additional information. - static SNMALLOC_FAST_PATH bool - dealloc_fast(Slab* self, Superslab* super, void* p, LocalEntropy& entropy) + static SNMALLOC_FAST_PATH bool dealloc_fast( + CapPtr self, + CapPtr super, + CapPtr p, + LocalEntropy& entropy) { - Metaslab& meta = super->get_meta(self); - SNMALLOC_ASSERT(!meta.is_unused()); + auto meta = super->get_meta(self); + SNMALLOC_ASSERT(!meta->is_unused()); - if (unlikely(meta.return_object())) + if (unlikely(meta->return_object())) return false; // Update the head and the next pointer in the free list. - meta.free_queue.add(p, entropy); + meta->free_queue.add(p, entropy); return true; } @@ -116,23 +123,20 @@ namespace snmalloc // This does not need to remove the "use" as done by the fast path. // Returns a complex return code for managing the superslab meta data. // i.e. This deallocation could make an entire superslab free. - // - // This is pre-factored to take an explicit self parameter so that we can - // eventually annotate that pointer with additional information. static SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow( - Slab* self, + CapPtr self, SlabList* sl, - Superslab* super, - void* p, + CapPtr super, + CapPtr p, LocalEntropy& entropy) { - Metaslab& meta = super->get_meta(self); - meta.debug_slab_invariant(self, entropy); + auto meta = super->get_meta(self); + meta->debug_slab_invariant(self, entropy); - if (meta.is_full()) + if (meta->is_full()) { auto allocated = get_slab_capacity( - meta.sizeclass(), Metaslab::is_short(Metaslab::get_slab(p))); + meta->sizeclass(), Metaslab::is_short(Metaslab::get_slab(p))); // We are not on the sizeclass list. if (allocated == 1) { @@ -143,15 +147,15 @@ namespace snmalloc return super->dealloc_slab(self); } - meta.free_queue.add(p, entropy); + meta->free_queue.add(p, entropy); // Remove trigger threshold from how many we need before we have fully // freed the slab. - meta.needed() = - allocated - meta.threshold_for_waking_slab(Metaslab::is_short(self)); + meta->needed() = + allocated - meta->threshold_for_waking_slab(Metaslab::is_short(self)); // Push on the list of slabs for this sizeclass. - sl->insert_prev(&meta); - meta.debug_slab_invariant(self, entropy); + sl->insert_prev(meta.template as_static()); + meta->debug_slab_invariant(self, entropy); return Superslab::NoSlabReturn; } @@ -160,7 +164,7 @@ namespace snmalloc // Check free list is well-formed on platforms with // integers as pointers. FreeListIter fl; - meta.free_queue.close(fl, entropy); + meta->free_queue.close(fl, entropy); while (!fl.empty()) { @@ -169,7 +173,7 @@ namespace snmalloc } #endif - meta.remove(); + meta->remove(); if (Metaslab::is_short(self)) return super->dealloc_short_slab(); diff --git a/src/mem/superslab.h b/src/mem/superslab.h index 27fa6aa..b3f805f 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -59,9 +59,10 @@ namespace snmalloc ModArray meta; // Used size_t as results in better code in MSVC - size_t slab_to_index(Slab* slab) + template + size_t slab_to_index(CapPtr slab) { - auto res = (pointer_diff(this, slab) >> SLAB_BITS); + auto res = (pointer_diff(this, slab.unsafe_capptr) >> SLAB_BITS); SNMALLOC_ASSERT(res == static_cast(res)); return static_cast(res); } @@ -82,10 +83,12 @@ namespace snmalloc StatusChange = 2 }; - static Superslab* get(const void* p) + template + static SNMALLOC_FAST_PATH CapPtr get(CapPtr p) { - return pointer_align_down( - const_cast(p)); + static_assert(B == CBArena || B == CBChunk); + + return pointer_align_down(p.as_void()); } static bool is_short_sizeclass(sizeclass_t sizeclass) @@ -180,9 +183,10 @@ namespace snmalloc return Full; } - Metaslab& get_meta(Slab* slab) + template + CapPtr get_meta(CapPtr slab) { - return meta[slab_to_index(slab)]; + return CapPtr(&meta[slab_to_index(slab)]); } // This is pre-factored to take an explicit self parameter so that we can @@ -195,7 +199,7 @@ namespace snmalloc Slab* slab = reinterpret_cast(self); auto& metaz = self->meta[0]; - metaz.initialise(sizeclass, slab); + metaz.initialise(sizeclass, CapPtr(slab)); self->used++; return slab; @@ -212,7 +216,7 @@ namespace snmalloc auto& metah = self->meta[h]; uint8_t n = metah.next(); - metah.initialise(sizeclass, slab); + metah.initialise(sizeclass, CapPtr(slab)); self->head = h + n + 1; self->used += 2; @@ -221,8 +225,11 @@ namespace snmalloc } // Returns true, if this alters the value of get_status - Action dealloc_slab(Slab* slab) + template + Action dealloc_slab(CapPtr slab) { + static_assert(B == CBArena || B == CBChunk); + // This is not the short slab. uint8_t index = static_cast(slab_to_index(slab)); uint8_t n = head - index - 1; diff --git a/src/pal/pal.h b/src/pal/pal.h index c11efec..cceb135 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -70,9 +70,22 @@ namespace snmalloc static constexpr size_t OS_PAGE_SIZE = Pal::page_size; /** - * A centralized, inlinable wrapper around PAL::zero. This will matter more - * when we introduce AuthPtr-s. + * A convenience wrapper that avoids the need to litter unsafe accesses with + * every call to PAL::zero. + * + * We do this here rather than plumb CapPtr further just to minimize + * disruption and avoid code bloat. This wrapper ought to compile down to + * nothing if SROA is doing its job. */ + template + static SNMALLOC_FAST_PATH void pal_zero(CapPtr p, size_t sz) + { + static_assert( + !page_aligned || B == CBArena || B == CBChunkD || B == CBChunk); + PAL::template zero(p.unsafe_capptr, sz); + } + + // TODO: Remove once CapPtr<>s plumbed everywhere they need to be template static SNMALLOC_FAST_PATH void pal_zero(void* p, size_t sz) { diff --git a/src/test/func/sandbox/sandbox.cc b/src/test/func/sandbox/sandbox.cc index 1d2ff2e..99da8f9 100644 --- a/src/test/func/sandbox/sandbox.cc +++ b/src/test/func/sandbox/sandbox.cc @@ -86,7 +86,7 @@ namespace * * This method must be implemented for `LargeAlloc` to work. */ - void push_large_stack(Largeslab* slab, size_t large_class) + void push_large_stack(CapPtr slab, size_t large_class) { real_state->push_large_stack(slab, large_class); }