diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 9b3909b..cb3869d 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -188,7 +188,7 @@ namespace snmalloc static SNMALLOC_FAST_PATH void alloc_new_list( CapPtr& bumpptr, - FreeListIter& fast_free_list, + Metaslab* meta, size_t rsize, size_t slab_size, LocalEntropy& entropy) @@ -197,8 +197,7 @@ namespace snmalloc auto& key = entropy.get_free_list_key(); - FreeListBuilder b; - SNMALLOC_ASSERT(b.empty()); + auto& b = meta->free_queue; #ifdef SNMALLOC_CHECK_CLIENT // Structure to represent the temporary list elements @@ -243,7 +242,7 @@ namespace snmalloc auto curr_ptr = start_ptr; do { - b.add(FreeObject::make(curr_ptr.as_void()), key); + b.add(FreeObject::make(curr_ptr.as_void()), key, entropy); curr_ptr = curr_ptr->next; } while (curr_ptr != start_ptr); #else @@ -256,16 +255,14 @@ namespace snmalloc #endif // This code consumes everything up to slab_end. bumpptr = slab_end; - - SNMALLOC_ASSERT(!b.empty()); - b.close(fast_free_list, key); } ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); FreeListIter fl; - meta->free_queue.close(fl, key); + auto more = meta->free_queue.close(fl, key); + UNUSED(more); void* p = finish_alloc_no_zero(fl.take(key), sizeclass); #ifdef SNMALLOC_CHECK_CLIENT @@ -278,6 +275,21 @@ namespace snmalloc count++; } // Check the list contains all the elements + SNMALLOC_ASSERT( + (count + more) == snmalloc::sizeclass_to_slab_object_count(sizeclass)); + + if (more > 0) + { + auto no_more = meta->free_queue.close(fl, key); + SNMALLOC_ASSERT(no_more == 0); + UNUSED(no_more); + + while (!fl.empty()) + { + fl.take(key); + count++; + } + } SNMALLOC_ASSERT( count == snmalloc::sizeclass_to_slab_object_count(sizeclass)); #endif @@ -582,15 +594,32 @@ namespace snmalloc // Look to see if we can grab a free list. auto& sl = alloc_classes[sizeclass].queue; - if (likely(!(sl.is_empty()))) + if (likely(alloc_classes[sizeclass].length > 0)) { +#ifdef SNMALLOC_CHECK_CLIENT + // Occassionally don't use the last list. + if ( + unlikely(alloc_classes[sizeclass].length == 1) && + (entropy.next_bit() == 0)) + { + return small_alloc_slow(sizeclass, fast_free_list, rsize); + } +#endif + auto meta = sl.pop(); // Drop length of sl, and empty count if it was empty. alloc_classes[sizeclass].length--; if (meta->needed() == 0) alloc_classes[sizeclass].unused--; - auto p = Metaslab::alloc(meta, fast_free_list, entropy, sizeclass); + auto [p, still_active] = + Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass); + + if (still_active) + { + alloc_classes[sizeclass].length++; + sl.insert(meta); + } return finish_alloc(p, sizeclass); } @@ -641,16 +670,20 @@ namespace snmalloc return nullptr; } - // Build a free list for the slab - alloc_new_list(slab, fast_free_list, rsize, slab_size, entropy); - // Set meta slab to empty. meta->initialise(sizeclass); - auto& key = entropy.get_free_list_key(); + // Build a free list for the slab + alloc_new_list(slab, meta, rsize, slab_size, entropy); - // take an allocation from the free list - auto p = fast_free_list.take(key); + auto [p, still_active] = + Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass); + + if (still_active) + { + alloc_classes[sizeclass].length++; + alloc_classes[sizeclass].queue.insert(meta); + } return finish_alloc(p, sizeclass); } diff --git a/src/mem/freelist.h b/src/mem/freelist.h index a8bc4df..9cfa49b 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -301,6 +301,8 @@ namespace snmalloc // This enables branch free enqueuing. std::array*, LENGTH> end{nullptr}; + std::array length{}; + public: constexpr FreeListBuilder() { @@ -336,6 +338,10 @@ namespace snmalloc index = 0; end[index] = FreeObject::store_next(end[index], n, key); + if constexpr (RANDOM) + { + length[index]++; + } } /** @@ -388,41 +394,42 @@ namespace snmalloc /** * Close a free list, and set the iterator parameter * to iterate it. + * + * In the RANDOM case, it may return only part of the freelist. + * + * The return value is how many entries are still contained in the builder. */ - SNMALLOC_FAST_PATH void close(FreeListIter& fl, const FreeListKey& key) + SNMALLOC_FAST_PATH uint16_t close(FreeListIter& fl, const FreeListKey& key) { + uint32_t i; if constexpr (RANDOM) { SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); - // If second list is non-empty, perform append. - if (end[1] != &head[1]) - { - // The start token has been corrupted. - // TOCTTOU issue, but small window here. - read_head(1, key)->check_prev(get_fake_signed_prev(1, key)); - - terminate_list(1, key); - - // Append 1 to 0 - FreeObject::store_next(end[0], read_head(1, key), key); - - SNMALLOC_ASSERT(end[1] != &head[0]); - SNMALLOC_ASSERT(end[0] != &head[1]); - } - else - { - terminate_list(0, key); - } + // Select longest list. + i = length[0] > length[1] ? 0 : 1; } else { - terminate_list(0, key); + i = 0; } - fl = {read_head(0, key), get_fake_signed_prev(0, key)}; - init(); + terminate_list(i, key); + + fl = {read_head(i, key), get_fake_signed_prev(i, key)}; + + end[i] = &head[i]; + + if constexpr (RANDOM) + { + length[i] = 0; + return length[1 - i]; + } + else + { + return 0; + } } /** @@ -433,6 +440,10 @@ namespace snmalloc for (size_t i = 0; i < LENGTH; i++) { end[i] = &head[i]; + if (RANDOM) + { + length[i] = 0; + } } } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 88f3721..66dbecb 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -71,7 +71,7 @@ namespace snmalloc // returned all the elements, but this is a slab that is still being bump // allocated from. Hence, the bump allocator slab will never be returned // for use in another size class. - set_sleeping(sizeclass); + set_sleeping(sizeclass, 0); } /** @@ -96,14 +96,28 @@ namespace snmalloc return sleeping(); } - SNMALLOC_FAST_PATH void set_sleeping(sizeclass_t sizeclass) + /** + * Try to set this metaslab to sleep. If the remaining elements are fewer + * than the threshold, then it will actually be set to the sleeping state, + * and will return true, otherwise it will return false. + */ + SNMALLOC_FAST_PATH bool + set_sleeping(sizeclass_t sizeclass, uint16_t remaining) { - SNMALLOC_ASSERT(free_queue.empty()); + auto threshold = threshold_for_waking_slab(sizeclass); + if (remaining >= threshold) + { + // Set needed to at least one, possibly more so we only use + // a slab when it has a reasonable amount of free elements + auto allocated = sizeclass_to_slab_object_count(sizeclass); + needed() = allocated - remaining; + sleeping() = false; + return false; + } - // Set needed to at least one, possibly more so we only use - // a slab when it has a reasonable amount of free elements - needed() = threshold_for_waking_slab(sizeclass); sleeping() = true; + needed() = threshold - remaining; + return true; } SNMALLOC_FAST_PATH void set_not_sleeping(sizeclass_t sizeclass) @@ -129,9 +143,18 @@ namespace snmalloc } /** - * TODO + * Allocates a free list from the meta data. + * + * Returns a freshly allocated object of the correct size, and a bool that + * specifies if the metaslab should be placed in the queue for that + * sizeclass. + * + * If Randomisation is not used, it will always return false for the second + * component, but with randomisation, it may only return part of the + * available objects for this metaslab. */ - static SNMALLOC_FAST_PATH CapPtr alloc( + static SNMALLOC_FAST_PATH std::pair, bool> + alloc_free_list( Metaslab* meta, FreeListIter& fast_free_list, LocalEntropy& entropy, @@ -140,7 +163,7 @@ namespace snmalloc auto& key = entropy.get_free_list_key(); FreeListIter tmp_fl; - meta->free_queue.close(tmp_fl, key); + auto remaining = meta->free_queue.close(tmp_fl, key); auto p = tmp_fl.take(key); fast_free_list = tmp_fl; @@ -150,12 +173,13 @@ namespace snmalloc UNUSED(entropy); #endif - // Treat stealing the free list as allocating it all. // This marks the slab as sleeping, and sets a wakeup // when sufficient deallocations have occurred to this slab. - meta->set_sleeping(sizeclass); + // Takes how many deallocations were not grabbed on this call + // This will be zero if there is no randomisation. + auto sleeping = meta->set_sleeping(sizeclass, remaining); - return p; + return {p, !sleeping}; } };