From 77506765989bd4ce456e7d60844e2471b6569efd Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 13 Sep 2021 20:56:34 +0100 Subject: [PATCH] NFC: FreeListIter domestication plumbing Just an intermediate syntactic step to chase dependencies. All these introduced "domestication" callbacks are just the identity function, but they will let us thread the LocalAlloc's handle to the Backend state down to where it's needed. --- src/mem/corealloc.h | 26 ++++++++++++++++++-------- src/mem/freelist.h | 7 +++---- src/mem/localalloc.h | 5 ++++- src/mem/localcache.h | 16 +++++++++++----- src/mem/metaslab.h | 4 +++- src/mem/remotecache.h | 4 +++- 6 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 06858f8..f66f504 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -180,8 +180,11 @@ namespace snmalloc void* small_alloc_one(size_t size) { SNMALLOC_ASSERT(attached_cache != nullptr); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( + domesticate, size, [&]( sizeclass_t sizeclass, @@ -278,7 +281,9 @@ namespace snmalloc FreeListIter fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); - void* p = finish_alloc_no_zero(fl.take(key), sizeclass); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; + void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); #ifdef SNMALLOC_CHECK_CLIENT // Check free list is well-formed on platforms with @@ -286,7 +291,7 @@ namespace snmalloc size_t count = 1; // Already taken one above. while (!fl.empty()) { - fl.take(key); + fl.take(key, domesticate); count++; } // Check the list contains all the elements @@ -301,7 +306,7 @@ namespace snmalloc while (!fl.empty()) { - fl.take(key); + fl.take(key, domesticate); count++; } } @@ -634,8 +639,10 @@ namespace snmalloc if (meta->needed() == 0) alloc_classes[sizeclass].unused--; - auto [p, still_active] = - Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto [p, still_active] = Metaslab::alloc_free_list( + domesticate, meta, fast_free_list, entropy, sizeclass); if (still_active) { @@ -701,8 +708,10 @@ namespace snmalloc // Build a free list for the slab alloc_new_list(slab, meta, rsize, slab_size, entropy); - auto [p, still_active] = - Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto [p, still_active] = Metaslab::alloc_free_list( + domesticate, meta, fast_free_list, entropy, sizeclass); if (still_active) { @@ -750,7 +759,8 @@ namespace snmalloc auto posted = attached_cache->flush( - backend_state_ptr(), [&](auto p) { dealloc_local_object(p); }); + backend_state_ptr(), + [&](capptr::Alloc p) { dealloc_local_object(p); }); // We may now have unused slabs, return to the global allocator. for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) diff --git a/src/mem/freelist.h b/src/mem/freelist.h index fadf81e..8d6ad07 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -416,12 +416,11 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - FreeObject::HeadPtr take(const FreeListKey& key) + template + FreeObject::HeadPtr + take(const FreeListKey& key, Domesticator domesticate) { auto c = curr; - // TODO: Placeholder - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; auto next = curr->read_next(key, domesticate); Aal::prefetch(next.unsafe_ptr()); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 0aa7062..9d1c5f2 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,6 +211,9 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); + + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; auto slowpath = [&]( sizeclass_t sizeclass, @@ -239,7 +242,7 @@ namespace snmalloc }; return local_cache.template alloc( - size, slowpath); + domesticate, size, slowpath); } /** diff --git a/src/mem/localcache.h b/src/mem/localcache.h index d72648d..e146208 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -82,6 +82,8 @@ namespace snmalloc typename SharedStateHandle::LocalState* local_state, DeallocFun dealloc) { auto& key = entropy.get_free_list_key(); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; for (size_t i = 0; i < NUM_SIZECLASSES; i++) { @@ -89,7 +91,7 @@ namespace snmalloc // call. while (!small_fast_free_lists[i].empty()) { - auto p = small_fast_free_lists[i].take(key); + auto p = small_fast_free_lists[i].take(key, domesticate); SNMALLOC_ASSERT(Metaslab::is_start_of_object(i, address_cast(p))); dealloc(p.as_void()); } @@ -99,18 +101,22 @@ namespace snmalloc local_state, remote_allocator->trunc_id(), key_global); } - template - SNMALLOC_FAST_PATH void* alloc(size_t size, Slowpath slowpath) + template< + ZeroMem zero_mem, + typename SharedStateHandle, + typename Slowpath, + typename Domesticator> + SNMALLOC_FAST_PATH void* + alloc(Domesticator domesticate, size_t size, Slowpath slowpath) { auto& key = entropy.get_free_list_key(); - sizeclass_t sizeclass = size_to_sizeclass(size); stats.alloc_request(size); stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; if (likely(!fl.empty())) { - auto p = fl.take(key); + auto p = fl.take(key, domesticate); return finish_alloc(p, sizeclass); } return slowpath(sizeclass, &fl); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index dca9dca..015dff1 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -154,9 +154,11 @@ namespace snmalloc * component, but with randomisation, it may only return part of the * available objects for this metaslab. */ + template static SNMALLOC_FAST_PATH std::pair, bool> alloc_free_list( + Domesticator domesticate, Metaslab* meta, FreeListIter& fast_free_list, @@ -167,7 +169,7 @@ namespace snmalloc std::remove_reference_t tmp_fl; auto remaining = meta->free_queue.close(tmp_fl, key); - auto p = tmp_fl.take(key); + auto p = tmp_fl.take(key, domesticate); fast_free_list = tmp_fl; #ifdef SNMALLOC_CHECK_CLIENT diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 4f38ab9..039d83d 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -92,6 +92,8 @@ namespace snmalloc SNMALLOC_ASSERT(initialised); size_t post_round = 0; bool sent_something = false; + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; while (true) { @@ -127,7 +129,7 @@ namespace snmalloc { // Use the next N bits to spread out remote deallocs in our own // slot. - auto r = resend.take(key); + auto r = resend.take(key, domesticate); MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( local_state, address_cast(r)); auto i = entry.get_remote()->trunc_id();