diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 06858f8..f66f504 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -180,8 +180,11 @@ namespace snmalloc void* small_alloc_one(size_t size) { SNMALLOC_ASSERT(attached_cache != nullptr); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( + domesticate, size, [&]( sizeclass_t sizeclass, @@ -278,7 +281,9 @@ namespace snmalloc FreeListIter fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); - void* p = finish_alloc_no_zero(fl.take(key), sizeclass); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; + void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); #ifdef SNMALLOC_CHECK_CLIENT // Check free list is well-formed on platforms with @@ -286,7 +291,7 @@ namespace snmalloc size_t count = 1; // Already taken one above. while (!fl.empty()) { - fl.take(key); + fl.take(key, domesticate); count++; } // Check the list contains all the elements @@ -301,7 +306,7 @@ namespace snmalloc while (!fl.empty()) { - fl.take(key); + fl.take(key, domesticate); count++; } } @@ -634,8 +639,10 @@ namespace snmalloc if (meta->needed() == 0) alloc_classes[sizeclass].unused--; - auto [p, still_active] = - Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto [p, still_active] = Metaslab::alloc_free_list( + domesticate, meta, fast_free_list, entropy, sizeclass); if (still_active) { @@ -701,8 +708,10 @@ namespace snmalloc // Build a free list for the slab alloc_new_list(slab, meta, rsize, slab_size, entropy); - auto [p, still_active] = - Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto [p, still_active] = Metaslab::alloc_free_list( + domesticate, meta, fast_free_list, entropy, sizeclass); if (still_active) { @@ -750,7 +759,8 @@ namespace snmalloc auto posted = attached_cache->flush( - backend_state_ptr(), [&](auto p) { dealloc_local_object(p); }); + backend_state_ptr(), + [&](capptr::Alloc p) { dealloc_local_object(p); }); // We may now have unused slabs, return to the global allocator. for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) diff --git a/src/mem/freelist.h b/src/mem/freelist.h index fadf81e..8d6ad07 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -416,12 +416,11 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - FreeObject::HeadPtr take(const FreeListKey& key) + template + FreeObject::HeadPtr + take(const FreeListKey& key, Domesticator domesticate) { auto c = curr; - // TODO: Placeholder - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; auto next = curr->read_next(key, domesticate); Aal::prefetch(next.unsafe_ptr()); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 0aa7062..9d1c5f2 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,6 +211,9 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); + + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; auto slowpath = [&]( sizeclass_t sizeclass, @@ -239,7 +242,7 @@ namespace snmalloc }; return local_cache.template alloc( - size, slowpath); + domesticate, size, slowpath); } /** diff --git a/src/mem/localcache.h b/src/mem/localcache.h index d72648d..e146208 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -82,6 +82,8 @@ namespace snmalloc typename SharedStateHandle::LocalState* local_state, DeallocFun dealloc) { auto& key = entropy.get_free_list_key(); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; for (size_t i = 0; i < NUM_SIZECLASSES; i++) { @@ -89,7 +91,7 @@ namespace snmalloc // call. while (!small_fast_free_lists[i].empty()) { - auto p = small_fast_free_lists[i].take(key); + auto p = small_fast_free_lists[i].take(key, domesticate); SNMALLOC_ASSERT(Metaslab::is_start_of_object(i, address_cast(p))); dealloc(p.as_void()); } @@ -99,18 +101,22 @@ namespace snmalloc local_state, remote_allocator->trunc_id(), key_global); } - template - SNMALLOC_FAST_PATH void* alloc(size_t size, Slowpath slowpath) + template< + ZeroMem zero_mem, + typename SharedStateHandle, + typename Slowpath, + typename Domesticator> + SNMALLOC_FAST_PATH void* + alloc(Domesticator domesticate, size_t size, Slowpath slowpath) { auto& key = entropy.get_free_list_key(); - sizeclass_t sizeclass = size_to_sizeclass(size); stats.alloc_request(size); stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; if (likely(!fl.empty())) { - auto p = fl.take(key); + auto p = fl.take(key, domesticate); return finish_alloc(p, sizeclass); } return slowpath(sizeclass, &fl); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index dca9dca..015dff1 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -154,9 +154,11 @@ namespace snmalloc * component, but with randomisation, it may only return part of the * available objects for this metaslab. */ + template static SNMALLOC_FAST_PATH std::pair, bool> alloc_free_list( + Domesticator domesticate, Metaslab* meta, FreeListIter& fast_free_list, @@ -167,7 +169,7 @@ namespace snmalloc std::remove_reference_t tmp_fl; auto remaining = meta->free_queue.close(tmp_fl, key); - auto p = tmp_fl.take(key); + auto p = tmp_fl.take(key, domesticate); fast_free_list = tmp_fl; #ifdef SNMALLOC_CHECK_CLIENT diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 4f38ab9..039d83d 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -92,6 +92,8 @@ namespace snmalloc SNMALLOC_ASSERT(initialised); size_t post_round = 0; bool sent_something = false; + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; while (true) { @@ -127,7 +129,7 @@ namespace snmalloc { // Use the next N bits to spread out remote deallocs in our own // slot. - auto r = resend.take(key); + auto r = resend.take(key, domesticate); MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( local_state, address_cast(r)); auto i = entry.get_remote()->trunc_id();