diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 0a7187f..931798a 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -641,20 +641,25 @@ namespace snmalloc #ifdef CACHE_FRIENDLY_OFFSET size_t remote_offset = 0; - void* apply_cache_friendly_offset(void* p, sizeclass_t sizeclass) + template + CapPtr + apply_cache_friendly_offset(CapPtr p, sizeclass_t sizeclass) { size_t mask = sizeclass_to_cache_friendly_mask(sizeclass); size_t offset = remote_offset & mask; remote_offset += CACHE_FRIENDLY_OFFSET; - return (void*)((uintptr_t)p + offset); + return CapPtr(reinterpret_cast( + reinterpret_cast(p.unsafe_capptr) + offset)); } #else - void* apply_cache_friendly_offset(void* p, sizeclass_t sizeclass) + template + static CapPtr + apply_cache_friendly_offset(CapPtr p, sizeclass_t sizeclass) { UNUSED(sizeclass); - return p; + return p.template as_static(); } #endif @@ -855,10 +860,8 @@ namespace snmalloc p->trunc_target_id() == super->get_allocator()->trunc_id(), "Detected memory corruption. Potential use-after-free"); - auto start_auth = CapPtr( - remove_cache_friendly_offset(p_auth.unsafe_capptr, p->sizeclass())); - dealloc_not_large_local( - super, start_auth, p_auth.as_void(), p->sizeclass()); + auto start_auth = remove_cache_friendly_offset(p_auth, p->sizeclass()); + dealloc_not_large_local(super, start_auth, p_auth, p->sizeclass()); } else { @@ -872,23 +875,23 @@ namespace snmalloc CapPtr p_auth, sizeclass_t sizeclass) { - auto offseted = CapPtr( - apply_cache_friendly_offset(p_auth.unsafe_capptr, sizeclass)); if (likely(target->trunc_id() == get_trunc_id())) { auto super = Superslab::get(p_auth); + auto offseted = apply_cache_friendly_offset(p_auth, sizeclass) + .template as_reinterpret(); dealloc_not_large_local(super, p_auth, offseted, sizeclass); } else { - remote_dealloc_and_post(target, offseted, sizeclass); + remote_dealloc_and_post(target, p_auth, sizeclass); } } SNMALLOC_FAST_PATH void dealloc_not_large_local( CapPtr super, CapPtr p_auth, - CapPtr p_auth_offseted, + CapPtr p_auth_offseted, sizeclass_t sizeclass) { // Guard against remote queues that have colliding IDs @@ -1056,8 +1059,7 @@ namespace snmalloc { stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); - void* p = remove_cache_friendly_offset( - fl.take(entropy).unsafe_capptr, sizeclass); + auto p = remove_cache_friendly_offset(fl.take(entropy), sizeclass); if constexpr (zero_mem == YesZero) { pal_zero( @@ -1173,8 +1175,7 @@ namespace snmalloc SNMALLOC_ASSERT(ffl.empty()); Slab::alloc_new_list(bp, ffl, rsize, entropy); - auto p = remove_cache_friendly_offset( - ffl.take(entropy).unsafe_capptr, sizeclass); + auto p = remove_cache_friendly_offset(ffl.take(entropy), sizeclass); if constexpr (zero_mem == YesZero) { @@ -1258,13 +1259,8 @@ namespace snmalloc if (likely(target == public_state())) { - void* offseted = - apply_cache_friendly_offset(p_auth.unsafe_capptr, sizeclass); - small_dealloc_offseted( - super, - slab, - FreeObject::make(CapPtr(offseted)), - sizeclass); + auto offseted = apply_cache_friendly_offset(p_auth, sizeclass); + small_dealloc_offseted(super, slab, offseted, sizeclass); } else remote_dealloc(target, p_auth, sizeclass); @@ -1613,10 +1609,8 @@ namespace snmalloc // this path. if (remote.capacity > 0) { - void* offseted = - apply_cache_friendly_offset(p.unsafe_capptr, sizeclass); stats().remote_free(sizeclass); - remote.dealloc(target->trunc_id(), offseted, sizeclass); + remote.dealloc(target->trunc_id(), p.unsafe_capptr, sizeclass); return; } @@ -1648,12 +1642,13 @@ namespace snmalloc SNMALLOC_SLOW_PATH void remote_dealloc_and_post( RemoteAllocator* target, - CapPtr offseted, + CapPtr p_auth, sizeclass_t sizeclass) { handle_message_queue(); stats().remote_free(sizeclass); + auto offseted = apply_cache_friendly_offset(p_auth, sizeclass); remote.dealloc(target->trunc_id(), offseted.unsafe_capptr, sizeclass); stats().remote_post(); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 54de6ac..d68d5d5 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -141,6 +141,15 @@ namespace snmalloc return p.template as_static(); } + /** + * Construct a FreeObject for local slabs from a Remote message. + */ + static CapPtr make(CapPtr p) + { + // TODO: Zero the difference between a FreeObject and a Remote + return p.template as_reinterpret(); + } + /** * Read the next pointer handling any required decoding of the pointer */ diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 43404a4..c7f5054 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -176,8 +176,7 @@ namespace snmalloc self->remove(); self->set_full(Metaslab::get_slab(n)); - void* p = - remove_cache_friendly_offset(n.unsafe_capptr, self->sizeclass()); + auto p = remove_cache_friendly_offset(n, self->sizeclass()); SNMALLOC_ASSERT(is_start_of_object(self, address_cast(p))); self->debug_slab_invariant(Metaslab::get_slab(n), entropy); @@ -194,7 +193,7 @@ namespace snmalloc UNUSED(rsize); } - return CapPtr(p); + return p; } void debug_slab_invariant(CapPtr slab, LocalEntropy& entropy) diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 24080e7..a49199d 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -58,29 +58,31 @@ namespace snmalloc bits::ADDRESS_BITS - SUPERSLAB_BITS; #ifdef CACHE_FRIENDLY_OFFSET - SNMALLOC_FAST_PATH static void* - remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) + template + SNMALLOC_FAST_PATH static CapPtr + remove_cache_friendly_offset(CapPtr p, sizeclass_t sizeclass) { size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass); - return p = (void*)((uintptr_t)p & mask); + return CapPtr((void*)((uintptr_t)p.unsafe_capptr & mask)); } - SNMALLOC_FAST_PATH static uintptr_t - remove_cache_friendly_offset(uintptr_t relative, sizeclass_t sizeclass) + SNMALLOC_FAST_PATH static address_t + remove_cache_friendly_offset(address_t relative, sizeclass_t sizeclass) { size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass); return relative & mask; } #else - SNMALLOC_FAST_PATH static void* - remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) + template + SNMALLOC_FAST_PATH static CapPtr + remove_cache_friendly_offset(CapPtr p, sizeclass_t sizeclass) { UNUSED(sizeclass); - return p; + return p.as_void(); } - SNMALLOC_FAST_PATH static uintptr_t - remove_cache_friendly_offset(uintptr_t relative, sizeclass_t sizeclass) + SNMALLOC_FAST_PATH static address_t + remove_cache_friendly_offset(address_t relative, sizeclass_t sizeclass) { UNUSED(sizeclass); return relative;