From 61314f2260299577ada91b348b9ed9d4fa34eb9c Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 17 Dec 2021 14:08:08 +0000 Subject: [PATCH] Post large deallocations to original thread (#441) * Post large deallocations to original thread This change sets all large allocations to be owned by the originating thread. This means they will be messaged back to the original thread before they can be reused. The following reason for making this change: * This will improve producer/consumer apps involving large allocations. * It enables the implementation of a more complex chunk allocator that reassembles chunks. * It addresses an issue with compartmentalisation where the handling of large allocations can result in meta-data ownership changing. --- src/mem/corealloc.h | 30 ++++++++++- src/mem/localalloc.h | 58 ++-------------------- src/mem/localcache.h | 6 ++- src/mem/metaslab.h | 38 +++++++++++--- src/mem/sizeclasstable.h | 19 ++++--- src/test/func/release-rounding/rounding.cc | 2 +- 6 files changed, 78 insertions(+), 75 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index f5f03f8..9a87d2f 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -402,6 +402,33 @@ namespace snmalloc // TODO: Handle message queue on this path? Metaslab* meta = entry.get_metaslab(); + + if (meta->is_large()) + { + // Handle large deallocation here. + size_t entry_sizeclass = entry.get_sizeclass().as_large(); + size_t size = bits::one_at_bit(entry_sizeclass); + size_t slab_sizeclass = + metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass); + +#ifdef SNMALLOC_TRACING + std::cout << "Large deallocation: " << size + << " chunk sizeclass: " << slab_sizeclass << std::endl; +#else + UNUSED(size); +#endif + + auto slab_record = reinterpret_cast(meta); + + ChunkAllocator::dealloc( + get_backend_local_state(), + chunk_local_state, + slab_record, + slab_sizeclass); + + return; + } + smallsizeclass_t sizeclass = entry.get_sizeclass().as_small(); UNUSED(entropy); @@ -665,8 +692,7 @@ namespace snmalloc SNMALLOC_ASSERT(!meta->is_unused()); snmalloc_check_client( - Metaslab::is_start_of_object( - entry.get_sizeclass().as_small(), address_cast(p)), + is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); auto cp = p.as_static>(); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index e132d06..a1d96dc 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -186,7 +186,7 @@ namespace snmalloc size_to_sizeclass_full(size), large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), - SharedStateHandle::fake_large_remote); + core_alloc->public_state()); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING @@ -194,9 +194,9 @@ namespace snmalloc << bits::next_pow2_bits(size) << std::endl; #endif - // Note that meta data is not currently used for large allocs. - // meta->initialise(size_to_sizeclass(size)); - UNUSED(meta); + // Initialise meta data for a successful large allocation. + if (meta != nullptr) + meta->initialise_large(); if (zero_mem == YesZero) { @@ -662,56 +662,6 @@ namespace snmalloc return; } - // Large deallocation or null. - // also checks for managed by page map. - if (SNMALLOC_LIKELY( - (p_tame != nullptr) && !entry.get_sizeclass().is_default())) - { -# if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) - dealloc_cheri_checks(p_tame.unsafe_ptr()); -# endif - - size_t entry_sizeclass = entry.get_sizeclass().as_large(); - - size_t size = bits::one_at_bit(entry_sizeclass); - size_t slab_sizeclass = - metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass); - - // Check for start of allocation. - snmalloc_check_client( - pointer_align_down(p_tame, size) == p_tame, - "Not start of an allocation."); - -# ifdef SNMALLOC_TRACING - std::cout << "Large deallocation: " << size - << " chunk sizeclass: " << slab_sizeclass << std::endl; -# else - UNUSED(size); -# endif - - auto slab_record = - static_cast(entry.get_metaslab_no_remote()); - - SNMALLOC_ASSERT( - address_cast(slab_record->meta_common.chunk) == address_cast(p_tame)); - - check_init( - []( - CoreAlloc* core_alloc, - ChunkRecord* slab_record, - size_t slab_sizeclass) { - ChunkAllocator::dealloc( - core_alloc->get_backend_local_state(), - core_alloc->chunk_local_state, - slab_record, - slab_sizeclass); - return nullptr; - }, - slab_record, - slab_sizeclass); - return; - } - // If p_tame is not null, then dealloc has been call on something // it shouldn't be called on. // TODO: Should this be tested even in the !CHECK_CLIENT case? diff --git a/src/mem/localcache.h b/src/mem/localcache.h index caa19c6..9b3f174 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -15,7 +15,8 @@ namespace snmalloc inline static SNMALLOC_FAST_PATH capptr::Alloc finish_alloc_no_zero(freelist::HeadPtr p, smallsizeclass_t sizeclass) { - SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); + SNMALLOC_ASSERT(is_start_of_object( + sizeclass_t::from_small_class(sizeclass), address_cast(p))); UNUSED(sizeclass); return p.as_void(); @@ -90,7 +91,8 @@ namespace snmalloc while (!small_fast_free_lists[i].empty()) { auto p = small_fast_free_lists[i].take(key, domesticate); - SNMALLOC_ASSERT(Metaslab::is_start_of_object(i, address_cast(p))); + SNMALLOC_ASSERT(is_start_of_object( + sizeclass_t::from_small_class(i), address_cast(p))); dealloc(p.as_void()); } } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 8ece19a..bad2d3c 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -60,6 +60,12 @@ namespace snmalloc */ bool sleeping_ = false; + /** + * Flag to indicate this is actually a large allocation rather than a slab + * of small allocations. + */ + bool large_ = false; + uint16_t& needed() { return needed_; @@ -82,6 +88,25 @@ namespace snmalloc // allocated from. Hence, the bump allocator slab will never be returned // for use in another size class. set_sleeping(sizeclass, 0); + + large_ = false; + } + + /** + * Make this a chunk represent a large allocation. + * + * Set needed so immediately moves to slow path. + */ + void initialise_large() + { + // We will push to this just to make the fast path clean. + free_queue.init(); + + // Flag to detect that it is a large alloc on the slow path + large_ = true; + + // Jump to slow path on first deallocation. + needed() = 1; } /** @@ -106,6 +131,11 @@ namespace snmalloc return sleeping(); } + bool is_large() + { + return large_; + } + /** * Try to set this metaslab to sleep. If the remaining elements are fewer * than the threshold, then it will actually be set to the sleeping state, @@ -144,14 +174,6 @@ namespace snmalloc sleeping() = false; } - static SNMALLOC_FAST_PATH bool - is_start_of_object(smallsizeclass_t sizeclass, address_t p) - { - return divisible_by_sizeclass( - sizeclass, - p - (bits::align_down(p, sizeclass_to_slab_size(sizeclass)))); - } - /** * Allocates a free list from the meta data. * diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index a697604..e243fc1 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -254,10 +254,12 @@ namespace snmalloc for (size_t sizeclass = 1; sizeclass < bits::BITS; sizeclass++) { auto lsc = sizeclass_t::from_large_class(sizeclass); - fast(lsc).size = bits::one_at_bit(lsc.as_large()); - - // Use slab mask as 0 for power of two sizes. - fast(lsc).slab_mask = 0; + auto& meta = fast(lsc); + meta.size = bits::one_at_bit(lsc.as_large()); + meta.slab_mask = meta.size - 1; + // The slab_mask will do all the necessary work, so + // perform identity multiplication for the test. + meta.mod_zero_mult = 1; } } }; @@ -367,23 +369,24 @@ namespace snmalloc return sizeclass_metadata.fast(sc).size - index_in_object(sc, addr); } - inline static bool divisible_by_sizeclass(smallsizeclass_t sc, size_t offset) + inline static bool is_start_of_object(sizeclass_t sc, address_t addr) { - // Only works up to certain offsets, exhaustively tested by rounding.cc + size_t offset = addr & (sizeclass_full_to_slab_size(sc) - 1); + // Only works up to certain offsets, exhaustively tested by rounding.cc if constexpr (sizeof(offset) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. // This is based on: // https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/ - auto mod_zero_mult = sizeclass_metadata.fast_small(sc).mod_zero_mult; + auto mod_zero_mult = sizeclass_metadata.fast(sc).mod_zero_mult; return (offset * mod_zero_mult) < mod_zero_mult; } else // Use 32-bit division as considerably faster than 64-bit, and // everything fits into 32bits here. - return static_cast(offset % sizeclass_to_size(sc)) == 0; + return static_cast(offset % sizeclass_full_to_size(sc)) == 0; } inline static size_t large_size_to_chunk_size(size_t size) diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index 72fe595..7b1a2d7 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -36,7 +36,7 @@ int main(int argc, char** argv) failed = true; } - bool opt_mod_0 = divisible_by_sizeclass(size_class, offset); + bool opt_mod_0 = is_start_of_object(sc, offset); if (opt_mod_0 != mod_0) { std::cout << "rsize " << rsize << " offset " << offset