From d4c120dfe5af7cbfc9b1c05de244f3238e3a0b32 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 15 Sep 2021 00:19:39 +0100 Subject: [PATCH] Free queues hold Wild pointers --- src/mem/corealloc.h | 51 ++++++++++++++++-------- src/mem/localalloc.h | 95 +++++++++++++++++++++++--------------------- src/mem/localcache.h | 21 ++++++---- src/mem/metaslab.h | 23 ++++++----- 4 files changed, 109 insertions(+), 81 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index f66f504..c882912 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -180,15 +180,19 @@ namespace snmalloc void* small_alloc_one(size_t size) { SNMALLOC_ASSERT(attached_cache != nullptr); - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [this](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate( + backend_state_ptr(), p); + }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( domesticate, size, [&]( sizeclass_t sizeclass, - FreeListIter* fl) { + FreeListIter* fl) { return small_alloc(sizeclass, *fl); }); } @@ -252,7 +256,8 @@ namespace snmalloc do { b.add( - FreeObject::make( + // Here begins our treatment of the heap as containing Wild pointers + FreeObject::make( capptr_to_user_address_control(curr_ptr.as_void())), key, entropy); @@ -263,7 +268,8 @@ namespace snmalloc do { b.add( - FreeObject::make( + // Here begins our treatment of the heap as containing Wild pointers + FreeObject::make( capptr_to_user_address_control( Aal::capptr_bound( p.as_void(), rsize))), @@ -278,11 +284,15 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter fl; + FreeListIter fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto local_state = backend_state_ptr(); + auto domesticate = + [local_state](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); #ifdef SNMALLOC_CHECK_CLIENT @@ -601,7 +611,7 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = p.as_static>(); + auto cp = p.as_static>(); auto& key = entropy.get_free_list_key(); @@ -614,7 +624,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc( sizeclass_t sizeclass, - FreeListIter& + FreeListIter& fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -639,8 +649,12 @@ namespace snmalloc if (meta->needed() == 0) alloc_classes[sizeclass].unused--; - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [this](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate( + backend_state_ptr(), p); + }; auto [p, still_active] = Metaslab::alloc_free_list( domesticate, meta, fast_free_list, entropy, sizeclass); @@ -676,7 +690,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc_slow( sizeclass_t sizeclass, - FreeListIter& + FreeListIter& fast_free_list, size_t rsize) { @@ -708,8 +722,12 @@ namespace snmalloc // Build a free list for the slab alloc_new_list(slab, meta, rsize, slab_size, entropy); - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [this](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate( + backend_state_ptr(), p); + }; auto [p, still_active] = Metaslab::alloc_free_list( domesticate, meta, fast_free_list, entropy, sizeclass); @@ -745,7 +763,8 @@ namespace snmalloc auto& entry = SharedStateHandle::Pagemap::get_metaentry( backend_state_ptr(), snmalloc::address_cast(p)); handle_dealloc_remote(entry, p.as_void(), need_post); - // XXX n is not known to be domesticated + // TODO p = SharedStateHandle::capptr_domesticate(backend_state_ptr(), + // n); p = n; } } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 9d1c5f2..83c78c6 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,35 +211,38 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); - - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [this](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate( + core_alloc->backend_state_ptr(), p); + }; auto slowpath = [&]( sizeclass_t sizeclass, - FreeListIter* fl) - SNMALLOC_FAST_PATH_LAMBDA { - if (likely(core_alloc != nullptr)) - { - return core_alloc->handle_message_queue( - []( - CoreAlloc* core_alloc, - sizeclass_t sizeclass, - FreeListIter* - fl) { - return core_alloc->template small_alloc( - sizeclass, *fl); - }, - core_alloc, - sizeclass, - fl); - } - return lazy_init( - [&](CoreAlloc*, sizeclass_t sizeclass) { - return small_alloc(sizeclass_to_size(sizeclass)); + FreeListIter* + fl) SNMALLOC_FAST_PATH_LAMBDA { + if (likely(core_alloc != nullptr)) + { + return core_alloc->handle_message_queue( + []( + CoreAlloc* core_alloc, + sizeclass_t sizeclass, + FreeListIter* + fl) { + return core_alloc->template small_alloc( + sizeclass, *fl); }, - sizeclass); - }; + core_alloc, + sizeclass, + fl); + } + return lazy_init( + [&](CoreAlloc*, sizeclass_t sizeclass) { + return small_alloc(sizeclass_to_size(sizeclass)); + }, + sizeclass); + }; return local_cache.template alloc( domesticate, size, slowpath); @@ -278,8 +281,8 @@ namespace snmalloc return; } - // Recheck what kind of dealloc we should do incase, the allocator we get - // from lazy_init is the originating allocator. + // Recheck what kind of dealloc we should do incase, the allocator we + // get from lazy_init is the originating allocator. lazy_init( [&](CoreAlloc*, CapPtr p) { dealloc(p.unsafe_ptr()); // TODO don't double count statistics @@ -327,8 +330,8 @@ namespace snmalloc {} /** - * Call `SharedStateHandle::ensure_init()` if it is implemented, do nothing - * otherwise. + * Call `SharedStateHandle::ensure_init()` if it is implemented, do + * nothing otherwise. */ SNMALLOC_FAST_PATH void ensure_init() @@ -458,17 +461,17 @@ namespace snmalloc // TODO: // Care is needed so that dealloc(nullptr) works before init // The backend allocator must ensure that a minimal page map exists - // before init, that maps null to a remote_deallocator that will never be - // in thread local state. + // before init, that maps null to a remote_deallocator that will never + // be in thread local state. capptr::AllocWild p_wild = capptr_from_client(p_raw); /* - * p_tame may be nullptr, even if p_raw/p_wild are not, in the case where - * domestication fails. We exclusively use p_tame below so that such - * failures become no ops; in the nullptr path, which should be well off - * the fast path, we could be slightly more aggressive and test that p_raw - * is also nullptr and Pal::error() if not. (TODO) + * p_tame may be nullptr, even if p_raw/p_wild are not, in the case + * where domestication fails. We exclusively use p_tame below so that + * such failures become no ops; in the nullptr path, which should be + * well off the fast path, we could be slightly more aggressive and test + * that p_raw is also nullptr and Pal::error() if not. (TODO) * * We do not rely on the bounds-checking ability of domestication here, * and just check the address (and, on other architectures, perhaps @@ -589,17 +592,17 @@ namespace snmalloc #ifdef SNMALLOC_PASS_THROUGH return external_alloc::malloc_usable_size(const_cast(p_raw)); #else - // TODO What's the domestication policy here? At the moment we just probe - // the pagemap with the raw address, without checks. There could be - // implicit domestication through the `SharedStateHandle::Pagemap` or we - // could just leave well enough alone. + // TODO What's the domestication policy here? At the moment we just + // probe the pagemap with the raw address, without checks. There could + // be implicit domestication through the `SharedStateHandle::Pagemap` or + // we could just leave well enough alone. // Note that this should return 0 for nullptr. // Other than nullptr, we know the system will be initialised as it must // be called with something we have already allocated. // To handle this case we require the uninitialised pagemap contain an - // entry for the first chunk of memory, that states it represents a large - // object, so we can pull the check for null off the fast path. + // entry for the first chunk of memory, that states it represents a + // large object, so we can pull the check for null off the fast path. MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p_raw)); @@ -625,10 +628,10 @@ namespace snmalloc void* external_pointer(void* p_raw) { #ifndef SNMALLOC_PASS_THROUGH - // TODO What's the domestication policy here? At the moment we just probe - // the pagemap with the raw address, without checks. There could be - // implicit domestication through the `SharedStateHandle::Pagemap` or we - // could just leave well enough alone. + // TODO What's the domestication policy here? At the moment we just + // probe the pagemap with the raw address, without checks. There could + // be implicit domestication through the `SharedStateHandle::Pagemap` or + // we could just leave well enough alone. // TODO bring back the CHERI bits. Wes to review if required. MetaEntry entry = diff --git a/src/mem/localcache.h b/src/mem/localcache.h index e146208..0f54dc7 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -12,9 +12,9 @@ namespace snmalloc { using Stats = AllocStats; + template inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero( - FreeObject::HeadPtr p, - sizeclass_t sizeclass) + FreeObject::HeadPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -24,10 +24,12 @@ namespace snmalloc return r; } - template + template< + ZeroMem zero_mem, + typename SharedStateHandle, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> inline static SNMALLOC_FAST_PATH void* finish_alloc( - FreeObject::HeadPtr p, - sizeclass_t sizeclass) + FreeObject::HeadPtr p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); @@ -48,7 +50,7 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - FreeListIter + FreeListIter small_fast_free_lists[NUM_SIZECLASSES]; // This is the entropy for a particular thread. @@ -82,8 +84,11 @@ namespace snmalloc typename SharedStateHandle::LocalState* local_state, DeallocFun dealloc) { auto& key = entropy.get_free_list_key(); - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [local_state](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; for (size_t i = 0; i < NUM_SIZECLASSES; i++) { diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 015dff1..4be0499 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -25,10 +25,10 @@ namespace snmalloc * Data-structure for building the free list for this slab. */ #ifdef SNMALLOC_CHECK_CLIENT - FreeListBuilder + FreeListBuilder free_queue; #else - FreeListBuilder + FreeListBuilder free_queue; #endif @@ -155,15 +155,16 @@ namespace snmalloc * available objects for this metaslab. */ template - static SNMALLOC_FAST_PATH - std::pair, bool> - alloc_free_list( - Domesticator domesticate, - Metaslab* meta, - FreeListIter& - fast_free_list, - LocalEntropy& entropy, - sizeclass_t sizeclass) + static SNMALLOC_FAST_PATH std::pair< + FreeObject::HeadPtr, + bool> + alloc_free_list( + Domesticator domesticate, + Metaslab* meta, + FreeListIter& + fast_free_list, + LocalEntropy& entropy, + sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key();