diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index f5f03f8..9a87d2f 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -402,6 +402,33 @@ namespace snmalloc // TODO: Handle message queue on this path? Metaslab* meta = entry.get_metaslab(); + + if (meta->is_large()) + { + // Handle large deallocation here. + size_t entry_sizeclass = entry.get_sizeclass().as_large(); + size_t size = bits::one_at_bit(entry_sizeclass); + size_t slab_sizeclass = + metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass); + +#ifdef SNMALLOC_TRACING + std::cout << "Large deallocation: " << size + << " chunk sizeclass: " << slab_sizeclass << std::endl; +#else + UNUSED(size); +#endif + + auto slab_record = reinterpret_cast(meta); + + ChunkAllocator::dealloc( + get_backend_local_state(), + chunk_local_state, + slab_record, + slab_sizeclass); + + return; + } + smallsizeclass_t sizeclass = entry.get_sizeclass().as_small(); UNUSED(entropy); @@ -665,8 +692,7 @@ namespace snmalloc SNMALLOC_ASSERT(!meta->is_unused()); snmalloc_check_client( - Metaslab::is_start_of_object( - entry.get_sizeclass().as_small(), address_cast(p)), + is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); auto cp = p.as_static>(); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index e132d06..a1d96dc 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -186,7 +186,7 @@ namespace snmalloc size_to_sizeclass_full(size), large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), - SharedStateHandle::fake_large_remote); + core_alloc->public_state()); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING @@ -194,9 +194,9 @@ namespace snmalloc << bits::next_pow2_bits(size) << std::endl; #endif - // Note that meta data is not currently used for large allocs. - // meta->initialise(size_to_sizeclass(size)); - UNUSED(meta); + // Initialise meta data for a successful large allocation. + if (meta != nullptr) + meta->initialise_large(); if (zero_mem == YesZero) { @@ -662,56 +662,6 @@ namespace snmalloc return; } - // Large deallocation or null. - // also checks for managed by page map. - if (SNMALLOC_LIKELY( - (p_tame != nullptr) && !entry.get_sizeclass().is_default())) - { -# if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) - dealloc_cheri_checks(p_tame.unsafe_ptr()); -# endif - - size_t entry_sizeclass = entry.get_sizeclass().as_large(); - - size_t size = bits::one_at_bit(entry_sizeclass); - size_t slab_sizeclass = - metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass); - - // Check for start of allocation. - snmalloc_check_client( - pointer_align_down(p_tame, size) == p_tame, - "Not start of an allocation."); - -# ifdef SNMALLOC_TRACING - std::cout << "Large deallocation: " << size - << " chunk sizeclass: " << slab_sizeclass << std::endl; -# else - UNUSED(size); -# endif - - auto slab_record = - static_cast(entry.get_metaslab_no_remote()); - - SNMALLOC_ASSERT( - address_cast(slab_record->meta_common.chunk) == address_cast(p_tame)); - - check_init( - []( - CoreAlloc* core_alloc, - ChunkRecord* slab_record, - size_t slab_sizeclass) { - ChunkAllocator::dealloc( - core_alloc->get_backend_local_state(), - core_alloc->chunk_local_state, - slab_record, - slab_sizeclass); - return nullptr; - }, - slab_record, - slab_sizeclass); - return; - } - // If p_tame is not null, then dealloc has been call on something // it shouldn't be called on. // TODO: Should this be tested even in the !CHECK_CLIENT case? diff --git a/src/mem/localcache.h b/src/mem/localcache.h index caa19c6..9b3f174 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -15,7 +15,8 @@ namespace snmalloc inline static SNMALLOC_FAST_PATH capptr::Alloc finish_alloc_no_zero(freelist::HeadPtr p, smallsizeclass_t sizeclass) { - SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); + SNMALLOC_ASSERT(is_start_of_object( + sizeclass_t::from_small_class(sizeclass), address_cast(p))); UNUSED(sizeclass); return p.as_void(); @@ -90,7 +91,8 @@ namespace snmalloc while (!small_fast_free_lists[i].empty()) { auto p = small_fast_free_lists[i].take(key, domesticate); - SNMALLOC_ASSERT(Metaslab::is_start_of_object(i, address_cast(p))); + SNMALLOC_ASSERT(is_start_of_object( + sizeclass_t::from_small_class(i), address_cast(p))); dealloc(p.as_void()); } } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 8ece19a..bad2d3c 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -60,6 +60,12 @@ namespace snmalloc */ bool sleeping_ = false; + /** + * Flag to indicate this is actually a large allocation rather than a slab + * of small allocations. + */ + bool large_ = false; + uint16_t& needed() { return needed_; @@ -82,6 +88,25 @@ namespace snmalloc // allocated from. Hence, the bump allocator slab will never be returned // for use in another size class. set_sleeping(sizeclass, 0); + + large_ = false; + } + + /** + * Make this a chunk represent a large allocation. + * + * Set needed so immediately moves to slow path. + */ + void initialise_large() + { + // We will push to this just to make the fast path clean. + free_queue.init(); + + // Flag to detect that it is a large alloc on the slow path + large_ = true; + + // Jump to slow path on first deallocation. + needed() = 1; } /** @@ -106,6 +131,11 @@ namespace snmalloc return sleeping(); } + bool is_large() + { + return large_; + } + /** * Try to set this metaslab to sleep. If the remaining elements are fewer * than the threshold, then it will actually be set to the sleeping state, @@ -144,14 +174,6 @@ namespace snmalloc sleeping() = false; } - static SNMALLOC_FAST_PATH bool - is_start_of_object(smallsizeclass_t sizeclass, address_t p) - { - return divisible_by_sizeclass( - sizeclass, - p - (bits::align_down(p, sizeclass_to_slab_size(sizeclass)))); - } - /** * Allocates a free list from the meta data. * diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index a697604..e243fc1 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -254,10 +254,12 @@ namespace snmalloc for (size_t sizeclass = 1; sizeclass < bits::BITS; sizeclass++) { auto lsc = sizeclass_t::from_large_class(sizeclass); - fast(lsc).size = bits::one_at_bit(lsc.as_large()); - - // Use slab mask as 0 for power of two sizes. - fast(lsc).slab_mask = 0; + auto& meta = fast(lsc); + meta.size = bits::one_at_bit(lsc.as_large()); + meta.slab_mask = meta.size - 1; + // The slab_mask will do all the necessary work, so + // perform identity multiplication for the test. + meta.mod_zero_mult = 1; } } }; @@ -367,23 +369,24 @@ namespace snmalloc return sizeclass_metadata.fast(sc).size - index_in_object(sc, addr); } - inline static bool divisible_by_sizeclass(smallsizeclass_t sc, size_t offset) + inline static bool is_start_of_object(sizeclass_t sc, address_t addr) { - // Only works up to certain offsets, exhaustively tested by rounding.cc + size_t offset = addr & (sizeclass_full_to_slab_size(sc) - 1); + // Only works up to certain offsets, exhaustively tested by rounding.cc if constexpr (sizeof(offset) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. // This is based on: // https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/ - auto mod_zero_mult = sizeclass_metadata.fast_small(sc).mod_zero_mult; + auto mod_zero_mult = sizeclass_metadata.fast(sc).mod_zero_mult; return (offset * mod_zero_mult) < mod_zero_mult; } else // Use 32-bit division as considerably faster than 64-bit, and // everything fits into 32bits here. - return static_cast(offset % sizeclass_to_size(sc)) == 0; + return static_cast(offset % sizeclass_full_to_size(sc)) == 0; } inline static size_t large_size_to_chunk_size(size_t size) diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index 72fe595..7b1a2d7 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -36,7 +36,7 @@ int main(int argc, char** argv) failed = true; } - bool opt_mod_0 = divisible_by_sizeclass(size_class, offset); + bool opt_mod_0 = is_start_of_object(sc, offset); if (opt_mod_0 != mod_0) { std::cout << "rsize " << rsize << " offset " << offset