From 90a0274743effbe7ecd42d5e4e08b01c22cc3b03 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 26 Jun 2019 16:41:49 +0100 Subject: [PATCH 01/28] Made TLS initial Exec. This massively improves the TLS access at the expense of not being dynamically loadable. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a5eb4b..79e9bd4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,7 +105,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi") set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG") else() - add_compile_options(-march=native -fno-exceptions -fno-rtti -g) + add_compile_options(-march=native -fno-exceptions -fno-rtti -g -ftls-model=initial-exec) endif() macro(subdirlist result curdir) From 830b06a616a36271c5097cd81f552cbda678fd5b Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 26 Jun 2019 16:53:39 +0100 Subject: [PATCH 02/28] Add a couple of likely annotations. --- src/ds/helpers.h | 2 +- src/mem/threadalloc.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ds/helpers.h b/src/ds/helpers.h index e94227d..fdae63f 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -27,7 +27,7 @@ namespace snmalloc // If defined should be initially false; assert(first == nullptr || *first == false); - if (!initialised.load(std::memory_order_acquire)) + if (unlikely(!initialised.load(std::memory_order_acquire))) { FlagLock lock(flag); if (!initialised) diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 59a05e3..98f8096 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -284,7 +284,7 @@ namespace snmalloc { Alloc*& per_thread = inner_get(); - if (per_thread != nullptr) + if (likely(per_thread != nullptr)) return per_thread; // Slow path that performs initialization From 7a8eaec2ccea711406c44cc37b1630c2df6a1033 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 26 Jun 2019 17:40:26 +0100 Subject: [PATCH 03/28] Made a sizecass_t to wrap the sizeclass This is useful as codegen is nicer if we use size_t, but the semantics is uint8_t, and is stored as that in many places in the metadata. Ultimately should introduce a wrapper to check this invariant. --- src/mem/alloc.h | 53 ++++++++++++++-------------- src/mem/allocstats.h | 15 ++++---- src/mem/mediumslab.h | 4 +-- src/mem/sizeclass.h | 36 +++++++++++-------- src/mem/sizeclasstable.h | 19 +++++----- src/mem/superslab.h | 13 +++---- src/override/malloc.cc | 2 +- src/test/func/malloc/malloc.cc | 4 +-- src/test/func/sizeclass/sizeclass.cc | 4 +-- 9 files changed, 81 insertions(+), 69 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 703c163..bf104c8 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -277,7 +277,7 @@ namespace snmalloc else return calloc(1, size); #else - constexpr uint8_t sizeclass = size_to_sizeclass_const(size); + constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size); stats().alloc_request(size); @@ -317,7 +317,7 @@ namespace snmalloc handle_message_queue(); - uint8_t sizeclass = size_to_sizeclass(size); + sizeclass_t sizeclass = size_to_sizeclass(size); // Allocate memory of a dynamically known size. if (sizeclass < NUM_SMALL_CLASSES) @@ -346,7 +346,7 @@ namespace snmalloc return free(p); #else - constexpr uint8_t sizeclass = size_to_sizeclass_const(size); + constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size); handle_message_queue(); @@ -389,7 +389,7 @@ namespace snmalloc // Free memory of a dynamically known size. Must be called with an // external pointer. - uint8_t sizeclass = size_to_sizeclass(size); + sizeclass_t sizeclass = size_to_sizeclass(size); if (sizeclass < NUM_SMALL_CLASSES) { @@ -445,7 +445,7 @@ namespace snmalloc // Reading a remote sizeclass won't fail, since the other allocator // can't reuse the slab, as we have not yet deallocated this // pointer. - uint8_t sizeclass = meta.sizeclass; + sizeclass_t sizeclass = meta.sizeclass; if (super->get_allocator() == public_state()) small_dealloc(super, p, sizeclass); @@ -460,7 +460,7 @@ namespace snmalloc // Reading a remote sizeclass won't fail, since the other allocator // can't reuse the slab, as we have no yet deallocated this pointer. - uint8_t sizeclass = slab->get_sizeclass(); + sizeclass_t sizeclass = slab->get_sizeclass(); if (target == public_state()) medium_dealloc(slab, p, sizeclass); @@ -494,7 +494,7 @@ namespace snmalloc Slab* slab = Slab::get(p); Metaslab& meta = super->get_meta(slab); - uint8_t sc = meta.sizeclass; + sizeclass_t sc = meta.sizeclass; size_t slab_end = static_cast(address_cast(slab) + SLAB_SIZE); return external_pointer(p, sc, slab_end); @@ -503,7 +503,7 @@ namespace snmalloc { Mediumslab* slab = Mediumslab::get(p); - uint8_t sc = slab->get_sizeclass(); + sizeclass_t sc = slab->get_sizeclass(); size_t slab_end = static_cast(address_cast(slab) + SUPERSLAB_SIZE); @@ -622,7 +622,7 @@ namespace snmalloc return (id >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK; } - void dealloc(alloc_id_t target_id, void* p, uint8_t sizeclass) + void dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass) { this->size += sizeclass_to_size(sizeclass); @@ -706,7 +706,7 @@ namespace snmalloc #ifdef CACHE_FRIENDLY_OFFSET size_t remote_offset = 0; - void* apply_cache_friendly_offset(void* p, uint8_t sizeclass) + void* apply_cache_friendly_offset(void* p, sizeclass_t sizeclass) { size_t mask = sizeclass_to_cache_friendly_mask(sizeclass); @@ -716,7 +716,7 @@ namespace snmalloc return (void*)((uintptr_t)p + offset); } #else - void* apply_cache_friendly_offset(void* p, uint8_t sizeclass) + void* apply_cache_friendly_offset(void* p, sizeclass_t sizeclass) { UNUSED(sizeclass); return p; @@ -770,11 +770,11 @@ namespace snmalloc message_queue().invariant(); #ifndef NDEBUG - for (uint8_t i = 0; i < NUM_SIZECLASSES; i++) + for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) { size_t size = sizeclass_to_size(i); - uint8_t sc1 = size_to_sizeclass(size); - uint8_t sc2 = size_to_sizeclass_const(size); + sizeclass_t sc1 = size_to_sizeclass(size); + sizeclass_t sc2 = size_to_sizeclass_const(size); size_t size1 = sizeclass_to_size(sc1); size_t size2 = sizeclass_to_size(sc2); @@ -794,7 +794,7 @@ namespace snmalloc template static uintptr_t - external_pointer(void* p, uint8_t sizeclass, size_t end_point) + external_pointer(void* p, sizeclass_t sizeclass, size_t end_point) { size_t rsize = sizeclass_to_size(sizeclass); size_t end_point_correction = location == End ? @@ -840,7 +840,7 @@ namespace snmalloc Mediumslab* slab = Mediumslab::get(p); if (p->target_id() == id()) { - uint8_t sizeclass = slab->get_sizeclass(); + sizeclass_t sizeclass = slab->get_sizeclass(); void* start = remove_cache_friendly_offset(p, sizeclass); medium_dealloc(slab, start, sizeclass); } @@ -938,7 +938,7 @@ namespace snmalloc } template - Slab* alloc_slab(uint8_t sizeclass) + Slab* alloc_slab(sizeclass_t sizeclass) { stats().sizeclass_alloc_slab(sizeclass); if (Superslab::is_short_sizeclass(sizeclass)) @@ -978,7 +978,7 @@ namespace snmalloc } template - void* small_alloc(uint8_t sizeclass, size_t rsize) + void* small_alloc(sizeclass_t sizeclass, size_t rsize) { MEASURE_TIME_MARKERS( small_alloc, @@ -1011,7 +1011,7 @@ namespace snmalloc return slab->alloc(sc, rsize, large_allocator.memory_provider); } - void small_dealloc(Superslab* super, void* p, uint8_t sizeclass) + void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) { #ifdef CHECK_CLIENT Slab* slab = Slab::get(p); @@ -1025,7 +1025,8 @@ namespace snmalloc small_dealloc_offseted(super, offseted, sizeclass); } - void small_dealloc_offseted(Superslab* super, void* p, uint8_t sizeclass) + void + small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) { MEASURE_TIME(small_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); @@ -1100,7 +1101,7 @@ namespace snmalloc } template - void* medium_alloc(uint8_t sizeclass, size_t rsize, size_t size) + void* medium_alloc(sizeclass_t sizeclass, size_t rsize, size_t size) { MEASURE_TIME_MARKERS( medium_alloc, @@ -1110,7 +1111,7 @@ namespace snmalloc zero_mem == YesZero ? "zeromem" : "nozeromem", allow_reserve == NoReserve ? "noreserve" : "reserve")); - uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES; + sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; DLList* sc = &medium_classes[medium_class]; Mediumslab* slab = sc->get_head(); @@ -1144,7 +1145,7 @@ namespace snmalloc return p; } - void medium_dealloc(Mediumslab* slab, void* p, uint8_t sizeclass) + void medium_dealloc(Mediumslab* slab, void* p, sizeclass_t sizeclass) { MEASURE_TIME(medium_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); @@ -1163,7 +1164,7 @@ namespace snmalloc { if (!was_full) { - uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES; + sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; DLList* sc = &medium_classes[medium_class]; sc->remove(slab); } @@ -1180,7 +1181,7 @@ namespace snmalloc } else if (was_full) { - uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES; + sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; DLList* sc = &medium_classes[medium_class]; sc->insert(slab); } @@ -1233,7 +1234,7 @@ namespace snmalloc large_allocator.dealloc(slab, large_class); } - void remote_dealloc(RemoteAllocator* target, void* p, uint8_t sizeclass) + void remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) { MEASURE_TIME(remote_dealloc, 4, 16); diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h index b047a3a..83e0b2e 100644 --- a/src/mem/allocstats.h +++ b/src/mem/allocstats.h @@ -1,6 +1,7 @@ #pragma once #include "../ds/bits.h" +#include "../mem/sizeclass.h" #include @@ -165,7 +166,7 @@ namespace snmalloc #endif } - void sizeclass_alloc(uint8_t sc) + void sizeclass_alloc(sizeclass_t sc) { UNUSED(sc); @@ -175,7 +176,7 @@ namespace snmalloc #endif } - void sizeclass_dealloc(uint8_t sc) + void sizeclass_dealloc(sizeclass_t sc) { UNUSED(sc); @@ -194,7 +195,7 @@ namespace snmalloc #endif } - void sizeclass_alloc_slab(uint8_t sc) + void sizeclass_alloc_slab(sizeclass_t sc) { UNUSED(sc); @@ -204,7 +205,7 @@ namespace snmalloc #endif } - void sizeclass_dealloc_slab(uint8_t sc) + void sizeclass_dealloc_slab(sizeclass_t sc) { UNUSED(sc); @@ -251,7 +252,7 @@ namespace snmalloc #endif } - void remote_free(uint8_t sc) + void remote_free(sizeclass_t sc) { UNUSED(sc); @@ -267,7 +268,7 @@ namespace snmalloc #endif } - void remote_receive(uint8_t sc) + void remote_receive(sizeclass_t sc) { UNUSED(sc); @@ -348,7 +349,7 @@ namespace snmalloc << "Count" << csv.endl; } - for (uint8_t i = 0; i < N; i++) + for (sizeclass_t i = 0; i < N; i++) { if (sizeclass[i].count.is_unused()) continue; diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index ce533d8..38042ad 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -44,7 +44,7 @@ namespace snmalloc return pointer_cast(address_cast(p) & SUPERSLAB_MASK); } - void init(RemoteAllocator* alloc, uint8_t sc, size_t rsize) + void init(RemoteAllocator* alloc, sizeclass_t sc, size_t rsize) { assert(sc >= NUM_SMALL_CLASSES); assert((sc - NUM_SMALL_CLASSES) < NUM_MEDIUM_CLASSES); @@ -56,7 +56,7 @@ namespace snmalloc // initialise the allocation stack. if ((kind != Medium) || (sizeclass != sc)) { - sizeclass = sc; + sizeclass = static_cast(sc); uint16_t ssize = static_cast(rsize >> 8); kind = Medium; free = medium_slab_free(sc); diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 724ac28..0f2d28a 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -4,28 +4,34 @@ namespace snmalloc { - constexpr static uint16_t get_initial_bumpptr(uint8_t sc, bool is_short); - constexpr static uint16_t get_initial_link(uint8_t sc, bool is_short); - constexpr static size_t sizeclass_to_size(uint8_t sizeclass); - constexpr static size_t sizeclass_to_cache_friendly_mask(uint8_t sizeclass); - constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(uint8_t sc); - constexpr static uint16_t medium_slab_free(uint8_t sizeclass); + // Both usings should compile + using sizeclass_t = size_t; + // using sizeclass_t = uint8_t; - static inline uint8_t size_to_sizeclass(size_t size) + constexpr static uint16_t get_initial_bumpptr(sizeclass_t sc, bool is_short); + constexpr static uint16_t get_initial_link(sizeclass_t sc, bool is_short); + constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); + constexpr static size_t + sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); + constexpr static size_t + sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); + constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass); + + static inline sizeclass_t size_to_sizeclass(size_t size) { // Don't use sizeclasses that are not a multiple of the alignment. // For example, 24 byte allocations can be // problematic for some data due to alignment issues. - return static_cast( + return static_cast( bits::to_exp_mant(size)); } - constexpr static inline uint8_t size_to_sizeclass_const(size_t size) + constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) { // Don't use sizeclasses that are not a multiple of the alignment. // For example, 24 byte allocations can be // problematic for some data due to alignment issues. - return static_cast( + return static_cast( bits::to_exp_mant_const(size)); } @@ -143,26 +149,28 @@ namespace snmalloc } #ifdef CACHE_FRIENDLY_OFFSET - inline static void* remove_cache_friendly_offset(void* p, uint8_t sizeclass) + inline static void* + remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) { size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass); return p = (void*)((uintptr_t)p & mask); } inline static uint16_t - remove_cache_friendly_offset(uint16_t relative, uint8_t sizeclass) + remove_cache_friendly_offset(uint16_t relative, sizeclass_t sizeclass) { size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass); return relative & mask; } #else - inline static void* remove_cache_friendly_offset(void* p, uint8_t sizeclass) + inline static void* + remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) { UNUSED(sizeclass); return p; } inline static uint16_t - remove_cache_friendly_offset(uint16_t relative, uint8_t sizeclass) + remove_cache_friendly_offset(uint16_t relative, sizeclass_t sizeclass) { UNUSED(sizeclass); return relative; diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 06c80aa..4f4958d 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -26,7 +26,7 @@ namespace snmalloc short_initial_link_ptr(), medium_slab_slots() { - for (uint8_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) + for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { size[sizeclass] = bits::from_exp_mant(sizeclass); @@ -40,7 +40,7 @@ namespace snmalloc size_t header_size = sizeof(Superslab); size_t short_slab_size = SLAB_SIZE - header_size; - for (uint8_t i = 0; i < NUM_SMALL_CLASSES; i++) + for (sizeclass_t i = 0; i < NUM_SMALL_CLASSES; i++) { // We align to the end of the block to remove special cases for the // short block. Calculate remainders @@ -63,7 +63,7 @@ namespace snmalloc bump_ptr_start[i] = static_cast((after_link + 1) % SLAB_SIZE); } - for (uint8_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) + for (sizeclass_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) { medium_slab_slots[i - NUM_SMALL_CLASSES] = static_cast( (SUPERSLAB_SIZE - Mediumslab::header_size()) / size[i]); @@ -74,7 +74,7 @@ namespace snmalloc static constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); static inline constexpr uint16_t - get_initial_bumpptr(uint8_t sc, bool is_short) + get_initial_bumpptr(sizeclass_t sc, bool is_short) { if (is_short) return sizeclass_metadata.short_bump_ptr_start[sc]; @@ -82,7 +82,8 @@ namespace snmalloc return sizeclass_metadata.bump_ptr_start[sc]; } - static inline constexpr uint16_t get_initial_link(uint8_t sc, bool is_short) + static inline constexpr uint16_t + get_initial_link(sizeclass_t sc, bool is_short) { if (is_short) return sizeclass_metadata.short_initial_link_ptr[sc]; @@ -90,24 +91,24 @@ namespace snmalloc return sizeclass_metadata.initial_link_ptr[sc]; } - constexpr static inline size_t sizeclass_to_size(uint8_t sizeclass) + constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) { return sizeclass_metadata.size[sizeclass]; } constexpr static inline size_t - sizeclass_to_cache_friendly_mask(uint8_t sizeclass) + sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass) { return sizeclass_metadata.cache_friendly_mask[sizeclass]; } constexpr static inline size_t - sizeclass_to_inverse_cache_friendly_mask(uint8_t sizeclass) + sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sizeclass) { return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass]; } - constexpr static inline uint16_t medium_slab_free(uint8_t sizeclass) + constexpr static inline uint16_t medium_slab_free(sizeclass_t sizeclass) { return sizeclass_metadata .medium_slab_slots[(sizeclass - NUM_SMALL_CLASSES)]; diff --git a/src/mem/superslab.h b/src/mem/superslab.h index c131d78..2c8d8b0 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -70,9 +70,9 @@ namespace snmalloc return pointer_cast(address_cast(p) & SUPERSLAB_MASK); } - static bool is_short_sizeclass(uint8_t sizeclass) + static bool is_short_sizeclass(sizeclass_t sizeclass) { - constexpr uint8_t h = size_to_sizeclass_const(sizeof(Superslab)); + constexpr sizeclass_t h = size_to_sizeclass_const(sizeof(Superslab)); return sizeclass <= h; } @@ -154,13 +154,14 @@ namespace snmalloc } template - Slab* alloc_short_slab(uint8_t sizeclass, MemoryProvider& memory_provider) + Slab* + alloc_short_slab(sizeclass_t sizeclass, MemoryProvider& memory_provider) { if ((used & 1) == 1) return alloc_slab(sizeclass, memory_provider); meta[0].head = get_initial_bumpptr(sizeclass, true); - meta[0].sizeclass = sizeclass; + meta[0].sizeclass = static_cast(sizeclass); meta[0].link = get_initial_link(sizeclass, true); if constexpr (decommit_strategy == DecommitAll) @@ -174,7 +175,7 @@ namespace snmalloc } template - Slab* alloc_slab(uint8_t sizeclass, MemoryProvider& memory_provider) + Slab* alloc_slab(sizeclass_t sizeclass, MemoryProvider& memory_provider) { uint8_t h = head; Slab* slab = pointer_cast( @@ -183,7 +184,7 @@ namespace snmalloc uint8_t n = meta[h].next; meta[h].head = get_initial_bumpptr(sizeclass, false); - meta[h].sizeclass = sizeclass; + meta[h].sizeclass = static_cast(sizeclass); meta[h].link = get_initial_link(sizeclass, false); head = h + n + 1; diff --git a/src/override/malloc.cc b/src/override/malloc.cc index db1b1bb..aba9ac3 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -137,7 +137,7 @@ extern "C" } size = bits::max(size, alignment); - uint8_t sc = size_to_sizeclass(size); + snmalloc::sizeclass_t sc = size_to_sizeclass(size); if (sc >= NUM_SIZECLASSES) { // large allocs are 16M aligned. diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 508f3c8..e692f3c 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -77,7 +77,7 @@ int main(int argc, char** argv) test_calloc(0, 0, SUCCESS, false); - for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (snmalloc::sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); @@ -103,7 +103,7 @@ int main(int argc, char** argv) for (size_t align = sizeof(size_t); align <= SUPERSLAB_SIZE; align <<= 1) { - for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (snmalloc::sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); test_posix_memalign(size, align, SUCCESS, false); diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index a98c201..6ca6c8d 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -2,7 +2,7 @@ #include NOINLINE -uint8_t size_to_sizeclass(size_t size) +snmalloc::sizeclass_t size_to_sizeclass(size_t size) { return snmalloc::size_to_sizeclass(size); } @@ -17,7 +17,7 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; - for (uint8_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++) + for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++) { // Separate printing for small and medium sizeclasses if (sz == snmalloc::NUM_SMALL_CLASSES) From 7f7704b6fc58e42d2755d6b21325e5d8c430f4d4 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 26 Jun 2019 17:08:27 +0100 Subject: [PATCH 04/28] Made a size to sizeclass table. --- CMakeLists.txt | 1 + src/mem/sizeclass.h | 17 ++++------------- src/mem/sizeclasstable.h | 40 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79e9bd4..69193b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) warnings_high() if(MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /constexpr:steps 10000000") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi") set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG") else() diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 0f2d28a..cb5d3b8 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -11,20 +11,11 @@ namespace snmalloc constexpr static uint16_t get_initial_bumpptr(sizeclass_t sc, bool is_short); constexpr static uint16_t get_initial_link(sizeclass_t sc, bool is_short); constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); - constexpr static size_t - sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); - constexpr static size_t - sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); + constexpr static size_t sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); + constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass); + static sizeclass_t size_to_sizeclass(size_t size); - static inline sizeclass_t size_to_sizeclass(size_t size) - { - // Don't use sizeclasses that are not a multiple of the alignment. - // For example, 24 byte allocations can be - // problematic for some data due to alignment issues. - return static_cast( - bits::to_exp_mant(size)); - } constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) { @@ -38,7 +29,7 @@ namespace snmalloc constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class) { return bits::one_at_bit(large_class + SUPERSLAB_BITS); - } + } // Small classes range from [MIN, SLAB], i.e. inclusive. static constexpr size_t NUM_SMALL_CLASSES = diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 4f4958d..cab6595 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -5,8 +5,21 @@ namespace snmalloc { + constexpr size_t PTR_BITS = bits::next_pow2_bits_const(sizeof(void*)); + + constexpr static size_t sizeclass_lookup_index(size_t s) + { + // We shift by PTR_BITS as makes code-gen for the table lookup nicer. + // We could shift by MIN_ALLOC_BITS, but then there is a slightly more + // complex sequence. + return (s - 1) >> PTR_BITS; + } + + constexpr static size_t sizeclass_lookup_size = sizeclass_lookup_index(SLAB_SIZE + 1); + struct SizeClassTable { + sizeclass_t sizeclass_lookup[sizeclass_lookup_size]; ModArray size; ModArray cache_friendly_mask; ModArray inverse_cache_friendly_mask; @@ -16,8 +29,10 @@ namespace snmalloc ModArray short_initial_link_ptr; ModArray medium_slab_slots; + constexpr SizeClassTable() - : size(), + : sizeclass_lookup(), + size(), cache_friendly_mask(), inverse_cache_friendly_mask(), bump_ptr_start(), @@ -26,10 +41,18 @@ namespace snmalloc short_initial_link_ptr(), medium_slab_slots() { + size_t curr = 1; for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { size[sizeclass] = bits::from_exp_mant(sizeclass); + if (sizeclass < NUM_SMALL_CLASSES) + { + for ( ; curr <= size[sizeclass]; curr+= 1 << PTR_BITS) + { + sizeclass_lookup[sizeclass_lookup_index(curr)] = sizeclass; + } + } size_t alignment = bits::min( bits::one_at_bit(bits::ctz_const(size[sizeclass])), OS_PAGE_SIZE); @@ -108,6 +131,21 @@ namespace snmalloc return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass]; } + static inline sizeclass_t size_to_sizeclass(size_t size) + { + if ((size-1) <= (SLAB_SIZE-1)) + { + return sizeclass_metadata.sizeclass_lookup[sizeclass_lookup_index(size)]; + } + + // Don't use sizeclasses that are not a multiple of the alignment. + // For example, 24 byte allocations can be + // problematic for some data due to alignment issues. + return static_cast( + bits::to_exp_mant(size)); + } + + constexpr static inline uint16_t medium_slab_free(sizeclass_t sizeclass) { return sizeclass_metadata From b0c15312218aae4762a02a17b1f8dd8c16b068b0 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 28 Jun 2019 08:09:42 +0100 Subject: [PATCH 05/28] Improved fast path for pagemap. --- src/mem/pagemap.h | 57 ++++++++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index 89bfd4f..e6dce02 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -112,33 +112,48 @@ namespace snmalloc // to see that correctly. PagemapEntry* value = e->load(std::memory_order_relaxed); + if (likely((value != nullptr) && (value != LOCKED_ENTRY))) + { + result = true; + return value; + } + if constexpr (create_addr) + { + return get_node_slow(e, result); + } + else + { + result = false; + return nullptr; + } + } + + NOINLINE PagemapEntry* get_node_slow(std::atomic* e, bool& result) + { + // The page map nodes are all allocated directly from the OS zero + // initialised with a system call. We don't need any ordered to guarantee + // to see that correctly. + PagemapEntry* value = e->load(std::memory_order_relaxed); + if ((value == nullptr) || (value == LOCKED_ENTRY)) { - if constexpr (create_addr) - { - value = nullptr; + value = nullptr; - if (e->compare_exchange_strong( - value, LOCKED_ENTRY, std::memory_order_relaxed)) - { - auto& v = default_memory_provider; - value = v.alloc_chunk(); - e->store(value, std::memory_order_release); - } - else - { - while (address_cast(e->load(std::memory_order_relaxed)) == - LOCKED_ENTRY) - { - bits::pause(); - } - value = e->load(std::memory_order_acquire); - } + if (e->compare_exchange_strong( + value, LOCKED_ENTRY, std::memory_order_relaxed)) + { + auto& v = default_memory_provider; + value = v.alloc_chunk(); + e->store(value, std::memory_order_release); } else { - result = false; - return nullptr; + while (address_cast(e->load(std::memory_order_relaxed)) == + LOCKED_ENTRY) + { + bits::pause(); + } + value = e->load(std::memory_order_acquire); } } result = true; From 7a198cbda5e823370fd24b7b145c76ecb87e31b4 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 28 Jun 2019 11:50:17 +0100 Subject: [PATCH 06/28] Aggressively optimise fast path for allocation This change introduces a per small sizeclass free list. That can be used to access the free objects for that sizeclass with minimal calculations being required. It changes to a partial bump ptr. We bump allocate a whole OS page worth of objects at a go, so we don't switch as frequently between bump and free list allocation. The code for the fast paths has been restructured to minimise the work required on the common case, and also it is all inlined for the common case. Allocating a zero sized object is moved off the fast path. Ask for 1 byte if you want to be fast. --- src/mem/alloc.h | 140 ++++++++++++++++++++++-------- src/mem/metaslab.h | 108 ++++++++++++----------- src/mem/sizeclass.h | 3 +- src/mem/sizeclasstable.h | 41 ++------- src/mem/slab.h | 182 ++++++++++++++++++++++++++------------- src/mem/superslab.h | 11 +-- src/override/malloc.cc | 5 -- 7 files changed, 299 insertions(+), 191 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index bf104c8..dbccb96 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -226,6 +226,17 @@ namespace snmalloc # define SNMALLOC_DEFAULT_PAGEMAP snmalloc::SuperslabMap<> #endif + // This class is just used so that the free lists are the first entry + // in the allocator and hence has better code gen. + // It contains a free list per small size class. These are used for allocation + // on the fast path. + // This part of the code is inspired by mimalloc. + class FastFreeLists + { + protected: + FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES]; + }; + /** * Allocator. This class is parameterised on three template parameters. The * `MemoryProvider` defines the source of memory for this allocator. @@ -247,7 +258,7 @@ namespace snmalloc class PageMap = SNMALLOC_DEFAULT_PAGEMAP, bool IsQueueInline = true> class Allocator - : public Pooled> + : public FastFreeLists, public Pooled> { LargeAlloc large_allocator; PageMap page_map; @@ -281,28 +292,27 @@ namespace snmalloc stats().alloc_request(size); - handle_message_queue(); - // Allocate memory of a statically known size. if constexpr (sizeclass < NUM_SMALL_CLASSES) { - constexpr size_t rsize = sizeclass_to_size(sizeclass); - return small_alloc(sizeclass, rsize); + return small_alloc(size); } else if constexpr (sizeclass < NUM_SIZECLASSES) { + handle_message_queue(); constexpr size_t rsize = sizeclass_to_size(sizeclass); return medium_alloc(sizeclass, rsize, size); } else - { + { + handle_message_queue(); return large_alloc(size); } #endif } template - ALLOCATOR void* alloc(size_t size) + inline ALLOCATOR void* alloc(size_t size) { #ifdef USE_MALLOC static_assert( @@ -315,18 +325,30 @@ namespace snmalloc #else stats().alloc_request(size); - handle_message_queue(); - - sizeclass_t sizeclass = size_to_sizeclass(size); - // Allocate memory of a dynamically known size. - if (sizeclass < NUM_SMALL_CLASSES) + // Perform the - 1 on size, so that zero wraps around and ends up on + // slow path. + if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1))) { // Allocations smaller than the slab size are more likely. Improve // branch prediction by placing this case first. - size_t rsize = sizeclass_to_size(sizeclass); - return small_alloc(sizeclass, rsize); + return small_alloc(size); } + + return alloc_not_small(size); + } + + template + NOINLINE ALLOCATOR void* alloc_not_small(size_t size) + { + handle_message_queue(); + + if (size == 0) + { + return small_alloc(1); + } + + sizeclass_t sizeclass = size_to_sizeclass(size); if (sizeclass < NUM_SIZECLASSES) { size_t rsize = sizeclass_to_size(sizeclass); @@ -418,7 +440,7 @@ namespace snmalloc #endif } - void dealloc(void* p) + ALWAYSINLINE void dealloc(void* p) { #ifdef USE_MALLOC return free(p); @@ -429,14 +451,10 @@ namespace snmalloc // pointer. uint8_t size = pagemap().get(address_cast(p)); - if (size == 0) - { - error("Not allocated by this allocator"); - } Superslab* super = Superslab::get(p); - if (size == PMSuperslab) + if (likely(size == PMSuperslab)) { RemoteAllocator* target = super->get_allocator(); Slab* slab = Slab::get(p); @@ -447,12 +465,17 @@ namespace snmalloc // pointer. sizeclass_t sizeclass = meta.sizeclass; - if (super->get_allocator() == public_state()) + if (likely(super->get_allocator() == public_state())) small_dealloc(super, p, sizeclass); else remote_dealloc(target, p, sizeclass); return; } + dealloc_not_small(p, size); + } + + NOINLINE void dealloc_not_small(void* p, uint8_t size) + { if (size == PMMediumslab) { Mediumslab* slab = Mediumslab::get(p); @@ -469,7 +492,13 @@ namespace snmalloc return; } + if (size == 0) + { + error("Not allocated by this allocator"); + } + # ifdef CHECK_CLIENT + Superslab* super = Superslab::get(p); if (size > 64 || address_cast(super) != address_cast(p)) { error("Not deallocating start of an object"); @@ -821,11 +850,11 @@ namespace snmalloc if (p->target_id() != super->get_allocator()->id()) error("Detected memory corruption. Potential use-after-free"); #endif - if (super->get_kind() == Super) + if (likely(super->get_kind() == Super)) { Slab* slab = Slab::get(p); Metaslab& meta = super->get_meta(slab); - if (p->target_id() == id()) + if (likely(p->target_id() == id())) { small_dealloc_offseted(super, p, meta.sizeclass); } @@ -978,7 +1007,7 @@ namespace snmalloc } template - void* small_alloc(sizeclass_t sizeclass, size_t rsize) + inline void* small_alloc(size_t size) { MEASURE_TIME_MARKERS( small_alloc, @@ -988,14 +1017,42 @@ namespace snmalloc zero_mem == YesZero ? "zeromem" : "nozeromem", allow_reserve == NoReserve ? "noreserve" : "reserve")); + sizeclass_t sizeclass = size_to_sizeclass(size); stats().sizeclass_alloc(sizeclass); - SlabList* sc = &small_classes[sizeclass]; + + auto& fl = small_fast_free_lists[sizeclass]; + auto head = fl.value; + if (likely((reinterpret_cast(head) & 1) == 0)) + { + void * p = head; + // Read the next slot from the memory that's about to be allocated. + fl.value = Metaslab::follow_next(p); + + if constexpr (zero_mem == YesZero) + { + large_allocator.memory_provider.zero(p, size); + } + return p; + } + + return small_alloc_slow(size); + } + + template + NOINLINE + void* small_alloc_slow(size_t size) + { + handle_message_queue(); + sizeclass_t sizeclass = size_to_sizeclass(size); + size_t rsize = sizeclass_to_size(sizeclass); + auto& sl = small_classes[sizeclass]; + Slab* slab; - if (!sc->is_empty()) + if (!sl.is_empty()) { - SlabLink* link = sc->get_head(); + SlabLink* link = sl.get_head(); slab = link->get_slab(); } else @@ -1005,13 +1062,13 @@ namespace snmalloc if ((allow_reserve == NoReserve) && (slab == nullptr)) return nullptr; - sc->insert(slab->get_link()); + sl.insert(slab->get_link()); } - - return slab->alloc(sc, rsize, large_allocator.memory_provider); + auto& ffl = small_fast_free_lists[sizeclass]; + return slab->alloc(sl, ffl, rsize, large_allocator.memory_provider); } - void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) + ALWAYSINLINE void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) { #ifdef CHECK_CLIENT Slab* slab = Slab::get(p); @@ -1025,20 +1082,27 @@ namespace snmalloc small_dealloc_offseted(super, offseted, sizeclass); } - void - small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) + ALWAYSINLINE void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) { MEASURE_TIME(small_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); - bool was_full = super->is_full(); - SlabList* sc = &small_classes[sizeclass]; Slab* slab = Slab::get(p); - Superslab::Action a = - slab->dealloc(sc, super, p, large_allocator.memory_provider); - if (a == Superslab::NoSlabReturn) + if (likely(slab->dealloc_fast(super, p))) return; + small_dealloc_offseted_slow(super, p, sizeclass); + } + + NOINLINE void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass) + { + bool was_full = super->is_full(); + SlabList* sl = &small_classes[sizeclass]; + Slab* slab = Slab::get(p); + Superslab::Action a = + slab->dealloc_slow(sl, super, p, large_allocator.memory_provider); + if (likely(a == Superslab::NoSlabReturn)) + return; stats().sizeclass_dealloc_slab(sizeclass); if (a == Superslab::NoStatusChange) diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 7bb2a32..df03708 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -29,22 +29,24 @@ namespace snmalloc // This can be either a short or a standard slab. class Metaslab { - private: - // How many entries are used in this slab. + public: + // How many entries are not in the free list of slab. uint16_t used = 0; - public: - // Bump free list of unused entries in this sizeclass. - // If the bottom bit is 1, then this represents a bump_ptr - // of where we have allocated up to in this slab. Otherwise, - // it represents the location of the first block in the free - // list. The free list is chained through deallocated blocks. - // It is terminated with a bump ptr. - // - // Note that, the first entry in a slab is never bump allocated - // but is used for the link. This means that 1 represents the fully - // bump allocated slab. + // How many entries have been allocated from this slab. + uint16_t allocated; + + // Index of first entry in this slab that forms the free + // list. The list entries are stored as the first pointer + // in each unused object. The terminator is a pointer or + // offset into the block with the bottom bit set. This means + // I.e. + // * an empty list has a head of 1. + // * a one element list has an head contains an offset to this + // this block, and then contains a pointer with the bottom + // bit set. Mod head; + // When a slab has free space it will be on the has space list for // that size class. We use an empty block in this slab to be the // doubly linked node into that size class's free list. @@ -93,35 +95,38 @@ namespace snmalloc /// Value used to check for corruptions in a block static constexpr size_t POISON = - static_cast(bits::is64() ? 0xDEADBEEFDEAD0000 : 0xDEAD0000); + static_cast(bits::is64() ? 0xDEADBEEFDEADBEEF : 0xDEADBEEF); /// Store next pointer in a block. In Debug using magic value to detect some /// simple corruptions. - static void store_next(void* p, uint16_t head) + static void store_next(void* p, void* head) { #ifndef CHECK_CLIENT - *static_cast(p) = head; + *static_cast(p) = head; #else - *static_cast(p) = - head ^ POISON ^ (static_cast(head) << (bits::BITS - 16)); + *static_cast(p) = head; + *(static_cast(p) + 1) = + address_cast(head) ^ POISON; #endif } /// Accessor function for the next pointer in a block. /// In Debug checks for simple corruptions. - static uint16_t follow_next(void* node) + static void* follow_next(void* node) { - size_t next = *static_cast(node); #ifdef CHECK_CLIENT - if (((next ^ POISON) ^ (next << (bits::BITS - 16))) > 0xFFFF) + uintptr_t next = *static_cast(node); + uintptr_t chk = *(static_cast(node) + 1); + if ((next ^ chk) != POISON) error("Detected memory corruption. Use-after-free."); #endif - return static_cast(next); + return *static_cast(node); } + bool valid_head(bool is_short) { size_t size = sizeclass_to_size(sizeclass); - size_t slab_start = get_initial_link(sizeclass, is_short); + size_t slab_start = get_initial_offset(sizeclass, is_short); size_t all_high_bits = ~static_cast(1); size_t head_start = @@ -138,18 +143,19 @@ namespace snmalloc * We don't expect a cycle, so worst case is only followed by a crash, so * slow doesn't mater. **/ - void debug_slab_acyclic_free_list(Slab* slab) + size_t debug_slab_acyclic_free_list(Slab* slab) { #ifndef NDEBUG - uint16_t curr = head; - uint16_t curr_slow = head; + size_t length = 0; + void* curr = pointer_offset(slab, head); + void* curr_slow = pointer_offset(slab, head); bool both = false; - while ((curr & 1) != 1) + while ((reinterpret_cast(curr) & 1) == 0) { - curr = follow_next(pointer_offset(slab, curr)); + curr = follow_next(curr); if (both) { - curr_slow = follow_next(pointer_offset(slab, curr_slow)); + curr_slow = follow_next(curr_slow); } if (curr == curr_slow) @@ -158,9 +164,12 @@ namespace snmalloc } both = !both; + length ++; } + return length; #else UNUSED(slab); + return 0; #endif } @@ -168,7 +177,10 @@ namespace snmalloc { #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) size_t size = sizeclass_to_size(sizeclass); - size_t offset = get_initial_link(sizeclass, is_short); + size_t offset = get_initial_offset(sizeclass, is_short); + + if (is_unused()) + return; size_t accounted_for = used * size + offset; @@ -184,40 +196,40 @@ namespace snmalloc // Block is not full assert(SLAB_SIZE > accounted_for); - debug_slab_acyclic_free_list(slab); - + size_t length = debug_slab_acyclic_free_list(slab); + UNUSED(length); // Walk bump-free-list-segment accounting for unused space - uint16_t curr = head; - while ((curr & 1) != 1) + void* curr = pointer_offset(slab, head); + while ((reinterpret_cast(curr) & 1) == 0) { // Check we are looking at a correctly aligned block - uint16_t start = remove_cache_friendly_offset(curr, sizeclass); - assert((start - offset) % size == 0); + void* start = curr; + assert(((address_cast(start) - address_cast(slab) - offset) % size) == 0); // Account for free elements in free list accounted_for += size; assert(SLAB_SIZE >= accounted_for); // We should never reach the link node in the free list. - assert(curr != link); + assert(curr != pointer_offset(slab, link)); // Iterate bump/free list segment - curr = follow_next(pointer_offset(slab, curr)); + curr = follow_next(curr); } - if (curr != 1) + auto bumpptr = (allocated * size) + offset; + // Check we haven't allocaated more than gits in a slab + assert(bumpptr <= SLAB_SIZE); + + // Account for to be bump allocated space + accounted_for += SLAB_SIZE - bumpptr; + + if (bumpptr != SLAB_SIZE) { - // Check we terminated traversal on a correctly aligned block - uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass); - assert((start - offset) % size == 0); - - // Account for to be bump allocated space - accounted_for += SLAB_SIZE - (curr - 1); - // The link should be the first allocation as we // haven't completely filled this block at any point. - assert(link == get_initial_link(sizeclass, is_short)); + assert(link == get_initial_offset(sizeclass, is_short)); } - + assert(!is_full()); // Add the link node. accounted_for += size; diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index cb5d3b8..735adab 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -8,8 +8,7 @@ namespace snmalloc using sizeclass_t = size_t; // using sizeclass_t = uint8_t; - constexpr static uint16_t get_initial_bumpptr(sizeclass_t sc, bool is_short); - constexpr static uint16_t get_initial_link(sizeclass_t sc, bool is_short); + constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short); constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); constexpr static size_t sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index cab6595..6ecab99 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -23,10 +23,8 @@ namespace snmalloc ModArray size; ModArray cache_friendly_mask; ModArray inverse_cache_friendly_mask; - ModArray bump_ptr_start; - ModArray short_bump_ptr_start; - ModArray initial_link_ptr; - ModArray short_initial_link_ptr; + ModArray initial_offset_ptr; + ModArray short_initial_offset_ptr; ModArray medium_slab_slots; @@ -35,10 +33,8 @@ namespace snmalloc size(), cache_friendly_mask(), inverse_cache_friendly_mask(), - bump_ptr_start(), - short_bump_ptr_start(), - initial_link_ptr(), - short_initial_link_ptr(), + initial_offset_ptr(), + short_initial_offset_ptr(), medium_slab_slots() { size_t curr = 1; @@ -71,19 +67,9 @@ namespace snmalloc size_t correction = SLAB_SIZE % size[i]; // First element in the block is the link - initial_link_ptr[i] = static_cast(correction); - short_initial_link_ptr[i] = + initial_offset_ptr[i] = static_cast(correction); + short_initial_offset_ptr[i] = static_cast(header_size + short_correction); - - // Move to object after link. - auto short_after_link = short_initial_link_ptr[i] + size[i]; - size_t after_link = initial_link_ptr[i] + size[i]; - - // Bump ptr has bottom bit set. - // In case we only have one object on this slab check for wrap around. - short_bump_ptr_start[i] = - static_cast((short_after_link + 1) % SLAB_SIZE); - bump_ptr_start[i] = static_cast((after_link + 1) % SLAB_SIZE); } for (sizeclass_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) @@ -97,21 +83,12 @@ namespace snmalloc static constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); static inline constexpr uint16_t - get_initial_bumpptr(sizeclass_t sc, bool is_short) + get_initial_offset(sizeclass_t sc, bool is_short) { if (is_short) - return sizeclass_metadata.short_bump_ptr_start[sc]; + return sizeclass_metadata.short_initial_offset_ptr[sc]; - return sizeclass_metadata.bump_ptr_start[sc]; - } - - static inline constexpr uint16_t - get_initial_link(sizeclass_t sc, bool is_short) - { - if (is_short) - return sizeclass_metadata.short_initial_link_ptr[sc]; - - return sizeclass_metadata.initial_link_ptr[sc]; + return sizeclass_metadata.initial_offset_ptr[sc]; } constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) diff --git a/src/mem/slab.h b/src/mem/slab.h index 0442740..141b6e8 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -4,6 +4,12 @@ namespace snmalloc { + struct FreeListHead + { + // Use a value with bottom bit set for empty list. + void* value = pointer_offset(nullptr, 1); + }; + class Slab { private: @@ -31,7 +37,7 @@ namespace snmalloc } template - void* alloc(SlabList* sc, size_t rsize, MemoryProvider& memory_provider) + inline void* alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize, MemoryProvider& memory_provider) { // Read the head from the metadata stored in the superslab. Metaslab& meta = get_meta(); @@ -39,38 +45,81 @@ namespace snmalloc assert(rsize == sizeclass_to_size(meta.sizeclass)); meta.debug_slab_invariant(is_short(), this); - assert(sc->get_head() == (SlabLink*)((size_t)this + meta.link)); + assert(sl.get_head() == (SlabLink*)((size_t)this + meta.link)); assert(!meta.is_full()); - meta.add_use(); - void* p; - if ((head & 1) == 0) + if (head == 1) { - void* node = pointer_offset(this, head); - - // Read the next slot from the memory that's about to be allocated. - meta.head = Metaslab::follow_next(node); - - p = remove_cache_friendly_offset(node, meta.sizeclass); - } - else - { - if (meta.head == 1) + size_t bumpptr = get_initial_offset(meta.sizeclass, is_short()); + bumpptr += meta.allocated * rsize; + if (bumpptr == SLAB_SIZE) { + meta.add_use(); + assert(meta.used == meta.allocated); + p = pointer_offset(this, meta.link); - sc->pop(); meta.set_full(); + sl.pop(); + goto finish1; } else { - // This slab is being bump allocated. - p = pointer_offset(this, head - 1); - meta.head = (head + static_cast(rsize)) & (SLAB_SIZE - 1); + + void* curr = nullptr; + bool commit = false; + while (true) + { + size_t newbumpptr = bumpptr + rsize; + auto alignedbumpptr = + bits::align_up(bumpptr - 1, OS_PAGE_SIZE); + auto alignednewbumpptr = bits::align_up(newbumpptr, OS_PAGE_SIZE); + + if (alignedbumpptr != alignednewbumpptr) + { + // We have committed once already. + if (commit) break; + + memory_provider.template notify_using(pointer_offset(this, alignedbumpptr), alignednewbumpptr - alignedbumpptr); + + commit = true; + } + + if (curr == nullptr) + { + meta.head = static_cast(bumpptr); + } + else + { + Metaslab::store_next(curr, pointer_offset(this, bumpptr)); + } + curr = pointer_offset(this, bumpptr); + bumpptr = newbumpptr; + meta.allocated = meta.allocated + 1; + } + + assert(curr != nullptr); + Metaslab::store_next(curr, pointer_offset(nullptr, 1)); } } + { + p = pointer_offset(this, meta.head); + + // Read the next slot from the memory that's about to be allocated. + void* next = Metaslab::follow_next(p); + // Put everything in allocators small_class free list. + meta.head = 1; + fast_free_list.value = next; + // Treat stealing the free list as allocating it all. + // Link is not in use, i.e. - 1 is required. + meta.used = meta.allocated - 1; + } + + assert (is_start_of_object(Superslab::get(p), p)); + + finish1: meta.debug_slab_invariant(is_short(), this); if constexpr (zero_mem == YesZero) @@ -92,38 +141,56 @@ namespace snmalloc address_cast(this) + SLAB_SIZE - address_cast(p)); } - // Returns true, if it alters get_status. - template - inline typename Superslab::Action dealloc( - SlabList* sc, Superslab* super, void* p, MemoryProvider& memory_provider) + // Returns true, if it deallocation can proceed without changing any status bits. + // Note that this does remove the use from the meta slab, so it doesn't need doing + // on the slow path. + ALWAYSINLINE bool dealloc_fast(Superslab* super, void* p) { Metaslab& meta = super->get_meta(this); - - bool was_full = meta.is_full(); - #ifdef CHECK_CLIENT if (meta.is_unused()) error("Detected potential double free."); #endif - meta.debug_slab_invariant(is_short(), this); meta.sub_use(); + bool was_full = meta.is_full(); + if (unlikely(was_full)) return false; + + bool is_unused = meta.is_unused(); + if (unlikely(is_unused)) return false; + + // Update the head and the next pointer in the free list. + uint16_t head = meta.head; + uint16_t current = pointer_to_index(p); + + // Set the head to the memory being deallocated. + meta.head = current; + assert(meta.valid_head(is_short())); + + // Set the next pointer to the previous head. + Metaslab::store_next(p, pointer_offset(this, head)); + meta.debug_slab_invariant(is_short(), this); + return true; + } + + // If dealloc fast returns false, then call this. + // This does not need to remove the "use" as done by the fast path. + // Returns a complex return code for managing the superslab meta data. + // i.e. This deallocation could make an entire superslab free. + template + NOINLINE typename Superslab::Action dealloc_slow( + SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider) + { + Metaslab& meta = super->get_meta(this); + + bool was_full = meta.is_full(); + bool is_unused = meta.is_unused(); + if (was_full) { // We are not on the sizeclass list. - if (!meta.is_unused()) - { - // Update the head and the sizeclass link. - uint16_t index = pointer_to_index(p); - assert(meta.head == 1); - meta.link = index; - - // Push on the list of slabs for this sizeclass. - sc->insert(meta.get_link(this)); - meta.debug_slab_invariant(is_short(), this); - } - else + if (is_unused) { // Dealloc on the superslab. if (is_short()) @@ -131,37 +198,30 @@ namespace snmalloc return super->dealloc_slab(this, memory_provider); } + // Update the head and the sizeclass link. + uint16_t index = pointer_to_index(p); + assert(meta.head == 1); +// assert(meta.fully_allocated(is_short())); + meta.link = index; + + // Push on the list of slabs for this sizeclass. + sl->insert(meta.get_link(this)); + meta.debug_slab_invariant(is_short(), this); + return Superslab::NoSlabReturn; } - else if (meta.is_unused()) + + if (is_unused) { // Remove from the sizeclass list and dealloc on the superslab. - sc->remove(meta.get_link(this)); + sl->remove(meta.get_link(this)); if (is_short()) return super->dealloc_short_slab(memory_provider); return super->dealloc_slab(this, memory_provider); } - else - { -#ifndef NDEBUG - sc->debug_check_contains(meta.get_link(this)); -#endif - - // Update the head and the next pointer in the free list. - uint16_t head = meta.head; - uint16_t current = pointer_to_index(p); - - // Set the head to the memory being deallocated. - meta.head = current; - assert(meta.valid_head(is_short())); - - // Set the next pointer to the previous head. - Metaslab::store_next(p, head); - - meta.debug_slab_invariant(is_short(), this); - } - return Superslab::NoSlabReturn; + + abort(); } bool is_short() diff --git a/src/mem/superslab.h b/src/mem/superslab.h index 2c8d8b0..66d65a5 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -160,11 +160,11 @@ namespace snmalloc if ((used & 1) == 1) return alloc_slab(sizeclass, memory_provider); - meta[0].head = get_initial_bumpptr(sizeclass, true); + meta[0].allocated = 1; + meta[0].head = 1; meta[0].sizeclass = static_cast(sizeclass); - meta[0].link = get_initial_link(sizeclass, true); + meta[0].link = get_initial_offset(sizeclass, true); - if constexpr (decommit_strategy == DecommitAll) { memory_provider.template notify_using( pointer_offset(this, OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE); @@ -183,9 +183,10 @@ namespace snmalloc uint8_t n = meta[h].next; - meta[h].head = get_initial_bumpptr(sizeclass, false); + meta[h].head = 1; + meta[h].allocated = 1; meta[h].sizeclass = static_cast(sizeclass); - meta[h].link = get_initial_link(sizeclass, false); + meta[h].link = get_initial_offset(sizeclass, false); head = h + n + 1; used += 2; diff --git a/src/override/malloc.cc b/src/override/malloc.cc index aba9ac3..1e55b84 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -23,9 +23,6 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) { - // Include size 0 in the first sizeclass. - size = ((size - 1) >> (bits::BITS - 1)) + size; - return ThreadAlloc::get()->alloc(size); } @@ -46,8 +43,6 @@ extern "C" errno = ENOMEM; return nullptr; } - // Include size 0 in the first sizeclass. - sz = ((sz - 1) >> (bits::BITS - 1)) + sz; return ThreadAlloc::get()->alloc(sz); } From fb825dde098bbc5c7eaf592eb29ce33a76b0c1ce Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 28 Jun 2019 16:56:35 +0100 Subject: [PATCH 07/28] Add an assert. --- src/mem/alloc.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index dbccb96..8d4f181 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1021,6 +1021,7 @@ namespace snmalloc stats().sizeclass_alloc(sizeclass); + assert(sizeclass < NUM_SMALL_CLASSES); auto& fl = small_fast_free_lists[sizeclass]; auto head = fl.value; if (likely((reinterpret_cast(head) & 1) == 0)) From c2780f99edc4af89ff2f90e28b131d432347e696 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Mon, 1 Jul 2019 10:46:52 +0100 Subject: [PATCH 08/28] [FreeBSD] Fix a warning with GCC. --- src/pal/pal_freebsd.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index e399786..9f3725b 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -38,7 +38,14 @@ namespace snmalloc assert( bits::is_aligned_block(p, size) || (zero_mem == NoZero)); if constexpr (zero_mem == YesZero) + { zero(p, size); + } + else + { + UNUSED(size); + UNUSED(p); + } } /// OS specific function for zeroing memory From 3c7d122dea0b405bf2254c7046b0767c60d32d19 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 1 Jul 2019 11:49:35 +0100 Subject: [PATCH 09/28] Add macro for ASSUME and FAST_PATH/SLOW_PATH Fixes GCC warning that was incorrect using an ASSUME. Made fast path and slow path Macros so we can add additional attributes. --- src/ds/bits.h | 14 ++++++++++++++ src/mem/alloc.h | 19 ++++++++++--------- src/mem/largealloc.h | 4 ++-- src/mem/pagemap.h | 8 ++++---- src/mem/slab.h | 4 ++-- src/mem/threadalloc.h | 18 +++++++++--------- 6 files changed, 41 insertions(+), 26 deletions(-) diff --git a/src/ds/bits.h b/src/ds/bits.h index 9ba07d3..cf6f79c 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -11,6 +11,13 @@ # define HEADER_GLOBAL __declspec(selectany) # define likely(x) !!(x) # define unlikely(x) !!(x) +# define SLOW_PATH NOINLINE +# define FAST_PATH ALWAYSINLINE +# ifdef NDEBUG +# define ASSUME(x) +# else +# define ASSUME(x) assert(x); +# endif #else # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) @@ -18,6 +25,13 @@ # include # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) +# define SLOW_PATH NOINLINE +# define FAST_PATH ALWAYSINLINE +# ifdef NDEBUG +# define ASSUME(x) if (!(x)) __builtin_unreachable(); +# else +# define ASSUME(x) assert(x); +# endif # ifdef __clang__ # define HEADER_GLOBAL __attribute__((selectany)) # else diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 8d4f181..7e8d402 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -339,7 +339,7 @@ namespace snmalloc } template - NOINLINE ALLOCATOR void* alloc_not_small(size_t size) + SLOW_PATH ALLOCATOR void* alloc_not_small(size_t size) { handle_message_queue(); @@ -440,7 +440,7 @@ namespace snmalloc #endif } - ALWAYSINLINE void dealloc(void* p) + FAST_PATH void dealloc(void* p) { #ifdef USE_MALLOC return free(p); @@ -474,7 +474,7 @@ namespace snmalloc dealloc_not_small(p, size); } - NOINLINE void dealloc_not_small(void* p, uint8_t size) + SLOW_PATH void dealloc_not_small(void* p, uint8_t size) { if (size == PMMediumslab) { @@ -882,7 +882,7 @@ namespace snmalloc } } - NOINLINE void handle_message_queue_inner() + SLOW_PATH void handle_message_queue_inner() { for (size_t i = 0; i < REMOTE_BATCH; i++) { @@ -902,7 +902,7 @@ namespace snmalloc remote.post(id()); } - ALWAYSINLINE void handle_message_queue() + FAST_PATH void handle_message_queue() { // Inline the empty check, but not necessarily the full queue handling. if (likely(message_queue().is_empty())) @@ -1017,6 +1017,7 @@ namespace snmalloc zero_mem == YesZero ? "zeromem" : "nozeromem", allow_reserve == NoReserve ? "noreserve" : "reserve")); + ASSUME(size <= SLAB_SIZE); sizeclass_t sizeclass = size_to_sizeclass(size); stats().sizeclass_alloc(sizeclass); @@ -1041,7 +1042,7 @@ namespace snmalloc } template - NOINLINE + SLOW_PATH void* small_alloc_slow(size_t size) { handle_message_queue(); @@ -1069,7 +1070,7 @@ namespace snmalloc return slab->alloc(sl, ffl, rsize, large_allocator.memory_provider); } - ALWAYSINLINE void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) + FAST_PATH void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) { #ifdef CHECK_CLIENT Slab* slab = Slab::get(p); @@ -1083,7 +1084,7 @@ namespace snmalloc small_dealloc_offseted(super, offseted, sizeclass); } - ALWAYSINLINE void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) + FAST_PATH void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) { MEASURE_TIME(small_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); @@ -1095,7 +1096,7 @@ namespace snmalloc small_dealloc_offseted_slow(super, p, sizeclass); } - NOINLINE void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass) + SLOW_PATH void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass) { bool was_full = super->is_full(); SlabList* sl = &small_classes[sizeclass]; diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 929cbd7..4b427c4 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -189,7 +189,7 @@ namespace snmalloc * (over the lifetime of this process). If the underlying system does not * support low memory notifications, this will return 0. */ - ALWAYSINLINE + FAST_PATH uint64_t low_memory_epoch() { if constexpr (pal_supports()) @@ -235,7 +235,7 @@ namespace snmalloc } } - ALWAYSINLINE void lazy_decommit_if_needed() + FAST_PATH void lazy_decommit_if_needed() { #ifdef TEST_LAZY_DECOMMIT static_assert( diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index e6dce02..55176cb 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -105,7 +105,7 @@ namespace snmalloc std::atomic top[TOPLEVEL_ENTRIES]; // = {nullptr}; template - inline PagemapEntry* get_node(std::atomic* e, bool& result) + FAST_PATH PagemapEntry* get_node(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -128,7 +128,7 @@ namespace snmalloc } } - NOINLINE PagemapEntry* get_node_slow(std::atomic* e, bool& result) + SLOW_PATH PagemapEntry* get_node_slow(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -161,7 +161,7 @@ namespace snmalloc } template - inline std::pair get_leaf_index(uintptr_t addr, bool& result) + FAST_PATH std::pair get_leaf_index(uintptr_t addr, bool& result) { #ifdef FreeBSD_KERNEL // Zero the top 16 bits - kernel addresses all have them set, but the @@ -204,7 +204,7 @@ namespace snmalloc } template - inline std::atomic* get_addr(uintptr_t p, bool& success) + FAST_PATH std::atomic* get_addr(uintptr_t p, bool& success) { auto leaf_ix = get_leaf_index(p, success); return &(leaf_ix.first->values[leaf_ix.second]); diff --git a/src/mem/slab.h b/src/mem/slab.h index 141b6e8..3b70680 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -144,7 +144,7 @@ namespace snmalloc // Returns true, if it deallocation can proceed without changing any status bits. // Note that this does remove the use from the meta slab, so it doesn't need doing // on the slow path. - ALWAYSINLINE bool dealloc_fast(Superslab* super, void* p) + FAST_PATH bool dealloc_fast(Superslab* super, void* p) { Metaslab& meta = super->get_meta(this); #ifdef CHECK_CLIENT @@ -179,7 +179,7 @@ namespace snmalloc // Returns a complex return code for managing the superslab meta data. // i.e. This deallocation could make an entire superslab free. template - NOINLINE typename Superslab::Action dealloc_slow( + SLOW_PATH typename Superslab::Action dealloc_slow( SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider) { Metaslab& meta = super->get_meta(this); diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 98f8096..cc17005 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -1,4 +1,4 @@ -#pragma once + #pragma once #include "../ds/helpers.h" #include "globalalloc.h" @@ -28,7 +28,7 @@ namespace snmalloc class ThreadAllocUntypedWrapper { public: - static inline Alloc*& get() + static FAST_PATH Alloc*& get() { return (Alloc*&)ThreadAllocUntyped::get(); } @@ -76,7 +76,7 @@ namespace snmalloc * The non-create case exists so that the `per_thread` variable can be a * local static and not a global, allowing ODR to deduplicate it. */ - static inline Alloc*& get(bool create = true) + static FAST_PATH Alloc*& get(bool create = true) { static thread_local Alloc* per_thread; if (!per_thread && create) @@ -224,7 +224,7 @@ namespace snmalloc * Private accessor to the per thread allocator * Provides no checking for initialization */ - static ALWAYSINLINE Alloc*& inner_get() + static FAST_PATH Alloc*& inner_get() { static thread_local Alloc* per_thread; return per_thread; @@ -242,7 +242,7 @@ namespace snmalloc /** * Private initialiser for the per thread allocator */ - static NOINLINE Alloc*& inner_init() + static SLOW_PATH Alloc*& inner_init() { Alloc*& per_thread = inner_get(); @@ -262,15 +262,15 @@ namespace snmalloc // Associate the new allocator with the destructor. tls_set_value(key, &per_thread); -# ifdef USE_SNMALLOC_STATS + # ifdef USE_SNMALLOC_STATS // Allocator is up and running now, safe to call atexit. if (first) { atexit(print_stats); } -# else + # else UNUSED(first); -# endif + # endif } return per_thread; } @@ -280,7 +280,7 @@ namespace snmalloc * Public interface, returns the allocator for the current thread, * constructing it if necessary. */ - static ALWAYSINLINE Alloc*& get() + static FAST_PATH Alloc*& get() { Alloc*& per_thread = inner_get(); From 187a016c39930e6b02e859b1849b1f33c31486eb Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 1 Jul 2019 13:10:57 +0100 Subject: [PATCH 10/28] Clang Tidy and Warnings --- src/mem/alloc.h | 2 ++ src/mem/sizeclasstable.h | 4 +++- src/mem/slab.h | 7 ++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 7e8d402..acf6277 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -235,6 +235,8 @@ namespace snmalloc { protected: FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES]; + public: + FastFreeLists () : small_fast_free_lists() {} }; /** diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 6ecab99..d4fc1ee 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -112,7 +112,9 @@ namespace snmalloc { if ((size-1) <= (SLAB_SIZE-1)) { - return sizeclass_metadata.sizeclass_lookup[sizeclass_lookup_index(size)]; + auto index = sizeclass_lookup_index(size); + ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE)); + return sizeclass_metadata.sizeclass_lookup[index]; } // Don't use sizeclasses that are not a multiple of the alignment. diff --git a/src/mem/slab.h b/src/mem/slab.h index 3b70680..850b595 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -48,7 +48,8 @@ namespace snmalloc assert(sl.get_head() == (SlabLink*)((size_t)this + meta.link)); assert(!meta.is_full()); - void* p; + void* p = nullptr; + bool p_has_value = false; if (head == 1) { @@ -62,7 +63,7 @@ namespace snmalloc p = pointer_offset(this, meta.link); meta.set_full(); sl.pop(); - goto finish1; + p_has_value = true; } else { @@ -104,6 +105,7 @@ namespace snmalloc } } + if (!p_has_value) { p = pointer_offset(this, meta.head); @@ -119,7 +121,6 @@ namespace snmalloc assert (is_start_of_object(Superslab::get(p), p)); - finish1: meta.debug_slab_invariant(is_short(), this); if constexpr (zero_mem == YesZero) From 9098b1c84f30a80d5836026e6441b64d53a4feba Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 1 Jul 2019 13:28:15 +0100 Subject: [PATCH 11/28] Only add Stats if passed to CMake. --- src/mem/alloc.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index acf6277..0805c40 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1,10 +1,5 @@ #pragma once -#if !defined(NDEBUG) && !defined(OPEN_ENCLAVE) && !defined(FreeBSD_KERNEL) && \ - !defined(USE_SNMALLOC_STATS) -# define USE_SNMALLOC_STATS -#endif - #ifdef _MSC_VER # define ALLOCATOR __declspec(allocator) #else From 84c0117e54d134432b3b9d6dd8d0857796ebd7c7 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 1 Jul 2019 13:42:34 +0100 Subject: [PATCH 12/28] Clang tidy. --- src/mem/sizeclasstable.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index d4fc1ee..ebf8116 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -19,7 +19,7 @@ namespace snmalloc struct SizeClassTable { - sizeclass_t sizeclass_lookup[sizeclass_lookup_size]; + sizeclass_t sizeclass_lookup[sizeclass_lookup_size] = {{}}; ModArray size; ModArray cache_friendly_mask; ModArray inverse_cache_friendly_mask; @@ -29,8 +29,7 @@ namespace snmalloc constexpr SizeClassTable() - : sizeclass_lookup(), - size(), + : size(), cache_friendly_mask(), inverse_cache_friendly_mask(), initial_offset_ptr(), From 1b26c6d1636d15324e69cdc3f5d6ad7d4b3315b7 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 1 Jul 2019 15:11:39 +0100 Subject: [PATCH 13/28] Updated differences --- difference.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/difference.md b/difference.md index 7cc6237..77b4f5c 100644 --- a/difference.md +++ b/difference.md @@ -18,3 +18,21 @@ This document outlines the changes that have diverged from The value 1, is never a valid bump allocation value, as we initially allocate the first entry as the link, so we can use 1 as the no more bump space value. + + 2. Separate Bump/Free list. We have separate bump ptr and free list. This + is required to have a "fast free list" in each allocator for each + sizeclass. We bump allocate a whole os page (4KiB) worth of allocations + in one go, so that we generally predicate the free list path. + + 3. Per allocator per sizeclass fast free list. Each allocator has an array + for each small size class that contains a free list of some elements for + that sizeclass. This enables a very compressed path for the common + allocation case. + + 4. We now store a direct pointer to the next element in each slabs free list + rather than a relative offset into the slab. This enables list + calculation on the fast path. + + +[2-4] Are changes that are directly inspired by +(mimalloc)[http://github.com/microsoft/mimalloc]. \ No newline at end of file From 0453e43cc5710ff002eafcd693af3be9978317eb Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 1 Jul 2019 15:23:01 +0100 Subject: [PATCH 14/28] Typo --- difference.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/difference.md b/difference.md index 77b4f5c..aa8bb1c 100644 --- a/difference.md +++ b/difference.md @@ -22,7 +22,8 @@ This document outlines the changes that have diverged from 2. Separate Bump/Free list. We have separate bump ptr and free list. This is required to have a "fast free list" in each allocator for each sizeclass. We bump allocate a whole os page (4KiB) worth of allocations - in one go, so that we generally predicate the free list path. + in one go, so that the CPU predicts the free list path for the fast + path. 3. Per allocator per sizeclass fast free list. Each allocator has an array for each small size class that contains a free list of some elements for From 86fde1005271515a83e4e09a0e9291fdb6573fc5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 1 Jul 2019 17:05:24 +0100 Subject: [PATCH 15/28] Minor changes to pagemap codegen --- src/mem/pagemap.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index 55176cb..4b944c2 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -112,7 +112,7 @@ namespace snmalloc // to see that correctly. PagemapEntry* value = e->load(std::memory_order_relaxed); - if (likely((value != nullptr) && (value != LOCKED_ENTRY))) + if (likely(value > LOCKED_ENTRY)) { result = true; return value; @@ -175,7 +175,7 @@ namespace snmalloc for (size_t i = 0; i < INDEX_LEVELS; i++) { PagemapEntry* value = get_node(e, result); - if (!result) + if (unlikely(!result)) return std::pair(nullptr, 0); shift -= BITS_PER_INDEX_LEVEL; @@ -195,7 +195,7 @@ namespace snmalloc Leaf* leaf = reinterpret_cast(get_node(e, result)); - if (!result) + if (unlikely(!result)) return std::pair(nullptr, 0); shift -= BITS_FOR_LEAF; From 0dbd10bd747063f785b374b3f91be9ff04584fa9 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 1 Jul 2019 17:06:04 +0100 Subject: [PATCH 16/28] Minor alterations to slow path, and when to handle messages. --- src/mem/alloc.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 0805c40..c03de63 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -442,7 +442,6 @@ namespace snmalloc #ifdef USE_MALLOC return free(p); #else - handle_message_queue(); // Free memory of an unknown size. Must be called with an external // pointer. @@ -473,6 +472,8 @@ namespace snmalloc SLOW_PATH void dealloc_not_small(void* p, uint8_t size) { + handle_message_queue(); + if (size == PMMediumslab) { Mediumslab* slab = Mediumslab::get(p); @@ -1035,15 +1036,14 @@ namespace snmalloc return p; } - return small_alloc_slow(size); + return small_alloc_slow(sizeclass); } template SLOW_PATH - void* small_alloc_slow(size_t size) + void* small_alloc_slow(sizeclass_t sizeclass) { handle_message_queue(); - sizeclass_t sizeclass = size_to_sizeclass(size); size_t rsize = sizeclass_to_size(sizeclass); auto& sl = small_classes[sizeclass]; @@ -1301,6 +1301,8 @@ namespace snmalloc { MEASURE_TIME(remote_dealloc, 4, 16); + handle_message_queue(); + void* offseted = apply_cache_friendly_offset(p, sizeclass); stats().remote_free(sizeclass); From b14735ff06b50b0157770018b0578fb1b5aad63e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 10:46:37 +0100 Subject: [PATCH 17/28] Fix clang format check. --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e136fd0..7a22327 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -172,8 +172,8 @@ phases: cmakeArgs: '..' - script: | - if [ "$(git diff $(Build.SourceVersion))" != "" ]; then exit 1; fi make clangformat + if [ "$(git diff $(Build.SourceVersion))" != "" ]; then exit 1; fi workingDirectory: build displayName: 'Clang-Format' From 621b7e6b9a34e5d10ed8caa46d5cd0713eaefa51 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 10:51:18 +0100 Subject: [PATCH 18/28] Clang format. --- src/ds/bits.h | 4 +++- src/mem/alloc.h | 36 +++++++++++++++++---------------- src/mem/metaslab.h | 14 ++++++------- src/mem/pagemap.h | 13 +++++++----- src/mem/sizeclass.h | 9 +++++---- src/mem/sizeclasstable.h | 11 +++++----- src/mem/slab.h | 43 +++++++++++++++++++++++----------------- src/mem/threadalloc.h | 8 ++++---- 8 files changed, 76 insertions(+), 62 deletions(-) diff --git a/src/ds/bits.h b/src/ds/bits.h index cf6f79c..f7c3d77 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -28,7 +28,9 @@ # define SLOW_PATH NOINLINE # define FAST_PATH ALWAYSINLINE # ifdef NDEBUG -# define ASSUME(x) if (!(x)) __builtin_unreachable(); +# define ASSUME(x) \ + if (!(x)) \ + __builtin_unreachable(); # else # define ASSUME(x) assert(x); # endif diff --git a/src/mem/alloc.h b/src/mem/alloc.h index c03de63..ffd4425 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -223,15 +223,15 @@ namespace snmalloc // This class is just used so that the free lists are the first entry // in the allocator and hence has better code gen. - // It contains a free list per small size class. These are used for allocation - // on the fast path. - // This part of the code is inspired by mimalloc. + // It contains a free list per small size class. These are used for + // allocation on the fast path. This part of the code is inspired by mimalloc. class FastFreeLists { protected: FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES]; + public: - FastFreeLists () : small_fast_free_lists() {} + FastFreeLists() : small_fast_free_lists() {} }; /** @@ -255,7 +255,8 @@ namespace snmalloc class PageMap = SNMALLOC_DEFAULT_PAGEMAP, bool IsQueueInline = true> class Allocator - : public FastFreeLists, public Pooled> + : public FastFreeLists, + public Pooled> { LargeAlloc large_allocator; PageMap page_map; @@ -301,7 +302,7 @@ namespace snmalloc return medium_alloc(sizeclass, rsize, size); } else - { + { handle_message_queue(); return large_alloc(size); } @@ -323,7 +324,7 @@ namespace snmalloc stats().alloc_request(size); // Allocate memory of a dynamically known size. - // Perform the - 1 on size, so that zero wraps around and ends up on + // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1))) { @@ -447,7 +448,6 @@ namespace snmalloc // pointer. uint8_t size = pagemap().get(address_cast(p)); - Superslab* super = Superslab::get(p); if (likely(size == PMSuperslab)) @@ -1019,14 +1019,13 @@ namespace snmalloc sizeclass_t sizeclass = size_to_sizeclass(size); stats().sizeclass_alloc(sizeclass); - assert(sizeclass < NUM_SMALL_CLASSES); auto& fl = small_fast_free_lists[sizeclass]; auto head = fl.value; if (likely((reinterpret_cast(head) & 1) == 0)) { - void * p = head; - // Read the next slot from the memory that's about to be allocated. + void* p = head; + // Read the next slot from the memory that's about to be allocated. fl.value = Metaslab::follow_next(p); if constexpr (zero_mem == YesZero) @@ -1040,8 +1039,7 @@ namespace snmalloc } template - SLOW_PATH - void* small_alloc_slow(sizeclass_t sizeclass) + SLOW_PATH void* small_alloc_slow(sizeclass_t sizeclass) { handle_message_queue(); size_t rsize = sizeclass_to_size(sizeclass); @@ -1064,10 +1062,12 @@ namespace snmalloc sl.insert(slab->get_link()); } auto& ffl = small_fast_free_lists[sizeclass]; - return slab->alloc(sl, ffl, rsize, large_allocator.memory_provider); + return slab->alloc( + sl, ffl, rsize, large_allocator.memory_provider); } - FAST_PATH void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) + FAST_PATH void + small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) { #ifdef CHECK_CLIENT Slab* slab = Slab::get(p); @@ -1081,7 +1081,8 @@ namespace snmalloc small_dealloc_offseted(super, offseted, sizeclass); } - FAST_PATH void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) + FAST_PATH void + small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) { MEASURE_TIME(small_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); @@ -1093,7 +1094,8 @@ namespace snmalloc small_dealloc_offseted_slow(super, p, sizeclass); } - SLOW_PATH void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass) + SLOW_PATH void small_dealloc_offseted_slow( + Superslab* super, void* p, sizeclass_t sizeclass) { bool was_full = super->is_full(); SlabList* sl = &small_classes[sizeclass]; diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index df03708..f737e71 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -40,10 +40,10 @@ namespace snmalloc // list. The list entries are stored as the first pointer // in each unused object. The terminator is a pointer or // offset into the block with the bottom bit set. This means - // I.e. + // I.e. // * an empty list has a head of 1. // * a one element list has an head contains an offset to this - // this block, and then contains a pointer with the bottom + // this block, and then contains a pointer with the bottom // bit set. Mod head; @@ -105,8 +105,7 @@ namespace snmalloc *static_cast(p) = head; #else *static_cast(p) = head; - *(static_cast(p) + 1) = - address_cast(head) ^ POISON; + *(static_cast(p) + 1) = address_cast(head) ^ POISON; #endif } @@ -164,7 +163,7 @@ namespace snmalloc } both = !both; - length ++; + length++; } return length; #else @@ -204,7 +203,8 @@ namespace snmalloc { // Check we are looking at a correctly aligned block void* start = curr; - assert(((address_cast(start) - address_cast(slab) - offset) % size) == 0); + assert( + ((address_cast(start) - address_cast(slab) - offset) % size) == 0); // Account for free elements in free list accounted_for += size; @@ -229,7 +229,7 @@ namespace snmalloc // haven't completely filled this block at any point. assert(link == get_initial_offset(sizeclass, is_short)); } - + assert(!is_full()); // Add the link node. accounted_for += size; diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index 4b944c2..e870084 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -105,7 +105,8 @@ namespace snmalloc std::atomic top[TOPLEVEL_ENTRIES]; // = {nullptr}; template - FAST_PATH PagemapEntry* get_node(std::atomic* e, bool& result) + FAST_PATH PagemapEntry* + get_node(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -127,8 +128,9 @@ namespace snmalloc return nullptr; } } - - SLOW_PATH PagemapEntry* get_node_slow(std::atomic* e, bool& result) + + SLOW_PATH PagemapEntry* + get_node_slow(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -149,7 +151,7 @@ namespace snmalloc else { while (address_cast(e->load(std::memory_order_relaxed)) == - LOCKED_ENTRY) + LOCKED_ENTRY) { bits::pause(); } @@ -161,7 +163,8 @@ namespace snmalloc } template - FAST_PATH std::pair get_leaf_index(uintptr_t addr, bool& result) + FAST_PATH std::pair + get_leaf_index(uintptr_t addr, bool& result) { #ifdef FreeBSD_KERNEL // Zero the top 16 bits - kernel addresses all have them set, but the diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 735adab..0665fed 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -10,12 +10,13 @@ namespace snmalloc constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short); constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); - constexpr static size_t sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); - constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); + constexpr static size_t + sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); + constexpr static size_t + sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass); static sizeclass_t size_to_sizeclass(size_t size); - constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) { // Don't use sizeclasses that are not a multiple of the alignment. @@ -28,7 +29,7 @@ namespace snmalloc constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class) { return bits::one_at_bit(large_class + SUPERSLAB_BITS); - } + } // Small classes range from [MIN, SLAB], i.e. inclusive. static constexpr size_t NUM_SMALL_CLASSES = diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index ebf8116..d0e3bb1 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -15,7 +15,8 @@ namespace snmalloc return (s - 1) >> PTR_BITS; } - constexpr static size_t sizeclass_lookup_size = sizeclass_lookup_index(SLAB_SIZE + 1); + constexpr static size_t sizeclass_lookup_size = + sizeclass_lookup_index(SLAB_SIZE + 1); struct SizeClassTable { @@ -27,7 +28,6 @@ namespace snmalloc ModArray short_initial_offset_ptr; ModArray medium_slab_slots; - constexpr SizeClassTable() : size(), cache_friendly_mask(), @@ -42,8 +42,8 @@ namespace snmalloc size[sizeclass] = bits::from_exp_mant(sizeclass); if (sizeclass < NUM_SMALL_CLASSES) - { - for ( ; curr <= size[sizeclass]; curr+= 1 << PTR_BITS) + { + for (; curr <= size[sizeclass]; curr += 1 << PTR_BITS) { sizeclass_lookup[sizeclass_lookup_index(curr)] = sizeclass; } @@ -109,7 +109,7 @@ namespace snmalloc static inline sizeclass_t size_to_sizeclass(size_t size) { - if ((size-1) <= (SLAB_SIZE-1)) + if ((size - 1) <= (SLAB_SIZE - 1)) { auto index = sizeclass_lookup_index(size); ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE)); @@ -123,7 +123,6 @@ namespace snmalloc bits::to_exp_mant(size)); } - constexpr static inline uint16_t medium_slab_free(sizeclass_t sizeclass) { return sizeclass_metadata diff --git a/src/mem/slab.h b/src/mem/slab.h index 850b595..387bce6 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -37,7 +37,11 @@ namespace snmalloc } template - inline void* alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize, MemoryProvider& memory_provider) + inline void* alloc( + SlabList& sl, + FreeListHead& fast_free_list, + size_t rsize, + MemoryProvider& memory_provider) { // Read the head from the metadata stored in the superslab. Metaslab& meta = get_meta(); @@ -67,22 +71,23 @@ namespace snmalloc } else { - void* curr = nullptr; bool commit = false; while (true) { size_t newbumpptr = bumpptr + rsize; - auto alignedbumpptr = - bits::align_up(bumpptr - 1, OS_PAGE_SIZE); + auto alignedbumpptr = bits::align_up(bumpptr - 1, OS_PAGE_SIZE); auto alignednewbumpptr = bits::align_up(newbumpptr, OS_PAGE_SIZE); if (alignedbumpptr != alignednewbumpptr) { // We have committed once already. - if (commit) break; + if (commit) + break; - memory_provider.template notify_using(pointer_offset(this, alignedbumpptr), alignednewbumpptr - alignedbumpptr); + memory_provider.template notify_using( + pointer_offset(this, alignedbumpptr), + alignednewbumpptr - alignedbumpptr); commit = true; } @@ -109,7 +114,7 @@ namespace snmalloc { p = pointer_offset(this, meta.head); - // Read the next slot from the memory that's about to be allocated. + // Read the next slot from the memory that's about to be allocated. void* next = Metaslab::follow_next(p); // Put everything in allocators small_class free list. meta.head = 1; @@ -119,7 +124,7 @@ namespace snmalloc meta.used = meta.allocated - 1; } - assert (is_start_of_object(Superslab::get(p), p)); + assert(is_start_of_object(Superslab::get(p), p)); meta.debug_slab_invariant(is_short(), this); @@ -142,9 +147,9 @@ namespace snmalloc address_cast(this) + SLAB_SIZE - address_cast(p)); } - // Returns true, if it deallocation can proceed without changing any status bits. - // Note that this does remove the use from the meta slab, so it doesn't need doing - // on the slow path. + // Returns true, if it deallocation can proceed without changing any status + // bits. Note that this does remove the use from the meta slab, so it + // doesn't need doing on the slow path. FAST_PATH bool dealloc_fast(Superslab* super, void* p) { Metaslab& meta = super->get_meta(this); @@ -156,10 +161,12 @@ namespace snmalloc meta.sub_use(); bool was_full = meta.is_full(); - if (unlikely(was_full)) return false; + if (unlikely(was_full)) + return false; bool is_unused = meta.is_unused(); - if (unlikely(is_unused)) return false; + if (unlikely(is_unused)) + return false; // Update the head and the next pointer in the free list. uint16_t head = meta.head; @@ -175,7 +182,7 @@ namespace snmalloc return true; } - // If dealloc fast returns false, then call this. + // If dealloc fast returns false, then call this. // This does not need to remove the "use" as done by the fast path. // Returns a complex return code for managing the superslab meta data. // i.e. This deallocation could make an entire superslab free. @@ -184,7 +191,7 @@ namespace snmalloc SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider) { Metaslab& meta = super->get_meta(this); - + bool was_full = meta.is_full(); bool is_unused = meta.is_unused(); @@ -202,7 +209,7 @@ namespace snmalloc // Update the head and the sizeclass link. uint16_t index = pointer_to_index(p); assert(meta.head == 1); -// assert(meta.fully_allocated(is_short())); + // assert(meta.fully_allocated(is_short())); meta.link = index; // Push on the list of slabs for this sizeclass. @@ -210,7 +217,7 @@ namespace snmalloc meta.debug_slab_invariant(is_short(), this); return Superslab::NoSlabReturn; } - + if (is_unused) { // Remove from the sizeclass list and dealloc on the superslab. @@ -221,7 +228,7 @@ namespace snmalloc return super->dealloc_slab(this, memory_provider); } - + abort(); } diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index cc17005..ef3fd8a 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -1,4 +1,4 @@ - #pragma once +#pragma once #include "../ds/helpers.h" #include "globalalloc.h" @@ -262,15 +262,15 @@ namespace snmalloc // Associate the new allocator with the destructor. tls_set_value(key, &per_thread); - # ifdef USE_SNMALLOC_STATS +# ifdef USE_SNMALLOC_STATS // Allocator is up and running now, safe to call atexit. if (first) { atexit(print_stats); } - # else +# else UNUSED(first); - # endif +# endif } return per_thread; } From d4e94d9c49719cc9a5349c1b51376fd11fd73408 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 11:17:16 +0100 Subject: [PATCH 19/28] Minor changes to mpscq fast path. --- CMakeLists.txt | 2 +- src/ds/mpscq.h | 9 +++++---- src/mem/alloc.h | 15 ++++++++------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 69193b1..8348758 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,7 +121,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) endmacro() macro(add_shim name) - add_library(${name} SHARED src/override/malloc.cc) + add_library(${name} SHARED src/override/malloc.cc src/override/new.cc) target_link_libraries(${name} snmalloc_lib) target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))") set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden) diff --git a/src/ds/mpscq.h b/src/ds/mpscq.h index f554473..39ee2f9 100644 --- a/src/ds/mpscq.h +++ b/src/ds/mpscq.h @@ -3,6 +3,8 @@ #include "bits.h" #include "helpers.h" +#include + namespace snmalloc { template @@ -59,7 +61,7 @@ namespace snmalloc prev->next.store(first, std::memory_order_relaxed); } - T* dequeue() + std::pair dequeue() { // Returns the front message, or null if not possible to return a message. invariant(); @@ -69,14 +71,13 @@ namespace snmalloc if (next != nullptr) { front = next; - assert(front); std::atomic_thread_fence(std::memory_order_acquire); invariant(); - return first; + return std::pair(first, true); } - return nullptr; + return std::pair(nullptr, false); } }; } // namespace snmalloc diff --git a/src/mem/alloc.h b/src/mem/alloc.h index ffd4425..25b5aca 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -649,7 +649,8 @@ namespace snmalloc return (id >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK; } - void dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass) + FAST_PATH void + dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass) { this->size += sizeclass_to_size(sizeclass); @@ -838,9 +839,9 @@ namespace snmalloc message_queue().init(&stub); } - void handle_dealloc_remote(Remote* p) + FAST_PATH void handle_dealloc_remote(Remote* p) { - if (p != &stub) + if (likely(p != &stub)) { Superslab* super = Superslab::get(p); @@ -884,16 +885,16 @@ namespace snmalloc { for (size_t i = 0; i < REMOTE_BATCH; i++) { - Remote* r = message_queue().dequeue(); + auto r = message_queue().dequeue(); - if (r == nullptr) + if (unlikely(!r.second)) break; - handle_dealloc_remote(r); + handle_dealloc_remote(r.first); } // Our remote queues may be larger due to forwarding remote frees. - if (remote.size < REMOTE_CACHE) + if (likely(remote.size < REMOTE_CACHE)) return; stats().remote_post(); From daebe3f5e5d07727b754a2c4511375724577ef4a Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 11:19:19 +0100 Subject: [PATCH 20/28] Remove constexpr steps. --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8348758..e77a7d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,7 +102,6 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) warnings_high() if(MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /constexpr:steps 10000000") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi") set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG") else() From fdcbcf701618612dd6509a84464e333744091ded Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 14:06:24 +0100 Subject: [PATCH 21/28] Add prefetch to mpscq. --- src/ds/bits.h | 9 +++++++++ src/ds/mpscq.h | 1 + 2 files changed, 10 insertions(+) diff --git a/src/ds/bits.h b/src/ds/bits.h index f7c3d77..62e6288 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -136,6 +136,15 @@ namespace snmalloc #endif } + inline void prefetch(void* ptr) + { +#if defined(PLATFORM_IS_X86) + _mm_prefetch(reinterpret_cast(ptr), _MM_HINT_T0); +#else +# warning "Missing pause intrinsic" +#endif + } + inline uint64_t tick() { #if defined(PLATFORM_IS_X86) diff --git a/src/ds/mpscq.h b/src/ds/mpscq.h index 39ee2f9..06b7265 100644 --- a/src/ds/mpscq.h +++ b/src/ds/mpscq.h @@ -71,6 +71,7 @@ namespace snmalloc if (next != nullptr) { front = next; + bits::prefetch(&(next->next)); assert(front); std::atomic_thread_fence(std::memory_order_acquire); invariant(); From eb4e28e8d0c9ee37096771728b90c5e364548e8b Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 14:01:16 +0100 Subject: [PATCH 22/28] CR Feedback Removed stub from message queue, and use an actual allocation. --- src/ds/bits.h | 38 +++++++--------- src/ds/mpscq.h | 5 +- src/mem/alloc.h | 98 ++++++++++++++++++++++------------------ src/mem/largealloc.h | 4 +- src/mem/metaslab.h | 8 ++-- src/mem/pagemap.h | 11 +++-- src/mem/sizeclass.h | 16 +++++-- src/mem/sizeclasstable.h | 13 ++++-- src/mem/slab.h | 4 +- src/mem/threadalloc.h | 10 ++-- 10 files changed, 111 insertions(+), 96 deletions(-) diff --git a/src/ds/bits.h b/src/ds/bits.h index 62e6288..eedd0f0 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -11,13 +11,9 @@ # define HEADER_GLOBAL __declspec(selectany) # define likely(x) !!(x) # define unlikely(x) !!(x) -# define SLOW_PATH NOINLINE -# define FAST_PATH ALWAYSINLINE -# ifdef NDEBUG -# define ASSUME(x) -# else -# define ASSUME(x) assert(x); -# endif +# define SNMALLOC_SLOW_PATH NOINLINE +# define SNMALLOC_FAST_PATH ALWAYSINLINE +# define SNMALLOC_PURE #else # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) @@ -25,15 +21,9 @@ # include # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) -# define SLOW_PATH NOINLINE -# define FAST_PATH ALWAYSINLINE -# ifdef NDEBUG -# define ASSUME(x) \ - if (!(x)) \ - __builtin_unreachable(); -# else -# define ASSUME(x) assert(x); -# endif +# define SNMALLOC_SLOW_PATH NOINLINE +# define SNMALLOC_FAST_PATH ALWAYSINLINE +# define SNMALLOC_PURE __attribute__((const)) # ifdef __clang__ # define HEADER_GLOBAL __attribute__((selectany)) # else @@ -68,13 +58,17 @@ #define UNUSED(x) ((void)(x)) -#if __has_builtin(__builtin_assume) -# define SNMALLOC_ASSUME(x) __builtin_assume(x) +#ifndef NDEBUG +# define SNMALLOC_ASSUME(x) assert(x) #else -# define SNMALLOC_ASSUME(x) \ - do \ - { \ - } while (0) +# if __has_builtin(__builtin_assume) +# define SNMALLOC_ASSUME(x) __builtin_assume((x)) +# else +# define SNMALLOC_ASSUME(x) \ + do \ + { \ + } while (0) +# endif #endif // #define USE_LZCNT diff --git a/src/ds/mpscq.h b/src/ds/mpscq.h index 06b7265..1b34a91 100644 --- a/src/ds/mpscq.h +++ b/src/ds/mpscq.h @@ -4,7 +4,6 @@ #include "helpers.h" #include - namespace snmalloc { template @@ -15,8 +14,8 @@ namespace snmalloc std::is_same>::value, "T->next must be a std::atomic"); - std::atomic back; - T* front; + std::atomic back = nullptr; + T* front = nullptr; public: void invariant() diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 25b5aca..17c8619 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -337,7 +337,7 @@ namespace snmalloc } template - SLOW_PATH ALLOCATOR void* alloc_not_small(size_t size) + SNMALLOC_SLOW_PATH ALLOCATOR void* alloc_not_small(size_t size) { handle_message_queue(); @@ -438,7 +438,7 @@ namespace snmalloc #endif } - FAST_PATH void dealloc(void* p) + SNMALLOC_FAST_PATH void dealloc(void* p) { #ifdef USE_MALLOC return free(p); @@ -470,7 +470,7 @@ namespace snmalloc dealloc_not_small(p, size); } - SLOW_PATH void dealloc_not_small(void* p, uint8_t size) + SNMALLOC_SLOW_PATH void dealloc_not_small(void* p, uint8_t size) { handle_message_queue(); @@ -649,7 +649,7 @@ namespace snmalloc return (id >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK; } - FAST_PATH void + SNMALLOC_FAST_PATH void dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass) { this->size += sizeclass_to_size(sizeclass); @@ -726,7 +726,6 @@ namespace snmalloc DLList super_only_short_available; RemoteCache remote; - Remote stub; std::conditional_t remote_alloc; @@ -836,52 +835,62 @@ namespace snmalloc void init_message_queue() { - message_queue().init(&stub); + // Manufacture an allocation to prime the queue + // Using an actual allocation removes a conditional of a critical path. + message_queue().init( + reinterpret_cast(alloc(MIN_ALLOC_SIZE))); } - FAST_PATH void handle_dealloc_remote(Remote* p) + SNMALLOC_FAST_PATH void handle_dealloc_remote(Remote* p) { - if (likely(p != &stub)) - { - Superslab* super = Superslab::get(p); + Superslab* super = Superslab::get(p); #ifdef CHECK_CLIENT - if (p->target_id() != super->get_allocator()->id()) - error("Detected memory corruption. Potential use-after-free"); + if (p->target_id() != super->get_allocator()->id()) + error("Detected memory corruption. Potential use-after-free"); #endif - if (likely(super->get_kind() == Super)) + if (likely(super->get_kind() == Super)) + { + Slab* slab = Slab::get(p); + Metaslab& meta = super->get_meta(slab); + if (likely(p->target_id() == id())) { - Slab* slab = Slab::get(p); - Metaslab& meta = super->get_meta(slab); - if (likely(p->target_id() == id())) - { - small_dealloc_offseted(super, p, meta.sizeclass); - } - else - { - // Queue for remote dealloc elsewhere. - remote.dealloc(p->target_id(), p, meta.sizeclass); - } + small_dealloc_offseted(super, p, meta.sizeclass); + return; + } + } + handle_dealloc_remote_slow(p); + } + + SNMALLOC_SLOW_PATH void handle_dealloc_remote_slow(Remote* p) + { + Superslab* super = Superslab::get(p); + if (likely(super->get_kind() == Medium)) + { + Mediumslab* slab = Mediumslab::get(p); + if (p->target_id() == id()) + { + sizeclass_t sizeclass = slab->get_sizeclass(); + void* start = remove_cache_friendly_offset(p, sizeclass); + medium_dealloc(slab, start, sizeclass); } else { - Mediumslab* slab = Mediumslab::get(p); - if (p->target_id() == id()) - { - sizeclass_t sizeclass = slab->get_sizeclass(); - void* start = remove_cache_friendly_offset(p, sizeclass); - medium_dealloc(slab, start, sizeclass); - } - else - { - // Queue for remote dealloc elsewhere. - remote.dealloc(p->target_id(), p, slab->get_sizeclass()); - } + // Queue for remote dealloc elsewhere. + remote.dealloc(p->target_id(), p, slab->get_sizeclass()); } } + else + { + assert(likely(p->target_id() != id())); + Slab* slab = Slab::get(p); + Metaslab& meta = super->get_meta(slab); + // Queue for remote dealloc elsewhere. + remote.dealloc(p->target_id(), p, meta.sizeclass); + } } - SLOW_PATH void handle_message_queue_inner() + SNMALLOC_SLOW_PATH void handle_message_queue_inner() { for (size_t i = 0; i < REMOTE_BATCH; i++) { @@ -901,7 +910,7 @@ namespace snmalloc remote.post(id()); } - FAST_PATH void handle_message_queue() + SNMALLOC_FAST_PATH void handle_message_queue() { // Inline the empty check, but not necessarily the full queue handling. if (likely(message_queue().is_empty())) @@ -1016,7 +1025,7 @@ namespace snmalloc zero_mem == YesZero ? "zeromem" : "nozeromem", allow_reserve == NoReserve ? "noreserve" : "reserve")); - ASSUME(size <= SLAB_SIZE); + SNMALLOC_ASSUME(size <= SLAB_SIZE); sizeclass_t sizeclass = size_to_sizeclass(size); stats().sizeclass_alloc(sizeclass); @@ -1040,7 +1049,7 @@ namespace snmalloc } template - SLOW_PATH void* small_alloc_slow(sizeclass_t sizeclass) + SNMALLOC_SLOW_PATH void* small_alloc_slow(sizeclass_t sizeclass) { handle_message_queue(); size_t rsize = sizeclass_to_size(sizeclass); @@ -1067,7 +1076,7 @@ namespace snmalloc sl, ffl, rsize, large_allocator.memory_provider); } - FAST_PATH void + SNMALLOC_FAST_PATH void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) { #ifdef CHECK_CLIENT @@ -1082,7 +1091,7 @@ namespace snmalloc small_dealloc_offseted(super, offseted, sizeclass); } - FAST_PATH void + SNMALLOC_FAST_PATH void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) { MEASURE_TIME(small_dealloc, 4, 16); @@ -1095,7 +1104,7 @@ namespace snmalloc small_dealloc_offseted_slow(super, p, sizeclass); } - SLOW_PATH void small_dealloc_offseted_slow( + SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow( Superslab* super, void* p, sizeclass_t sizeclass) { bool was_full = super->is_full(); @@ -1300,7 +1309,8 @@ namespace snmalloc large_allocator.dealloc(slab, large_class); } - void remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) + SNMALLOC_FAST_PATH void + remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) { MEASURE_TIME(remote_dealloc, 4, 16); diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 4b427c4..6d5ee5c 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -189,7 +189,7 @@ namespace snmalloc * (over the lifetime of this process). If the underlying system does not * support low memory notifications, this will return 0. */ - FAST_PATH + SNMALLOC_FAST_PATH uint64_t low_memory_epoch() { if constexpr (pal_supports()) @@ -235,7 +235,7 @@ namespace snmalloc } } - FAST_PATH void lazy_decommit_if_needed() + SNMALLOC_FAST_PATH void lazy_decommit_if_needed() { #ifdef TEST_LAZY_DECOMMIT static_assert( diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index f737e71..a7a60b9 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -99,7 +99,7 @@ namespace snmalloc /// Store next pointer in a block. In Debug using magic value to detect some /// simple corruptions. - static void store_next(void* p, void* head) + static SNMALLOC_FAST_PATH void store_next(void* p, void* head) { #ifndef CHECK_CLIENT *static_cast(p) = head; @@ -111,7 +111,7 @@ namespace snmalloc /// Accessor function for the next pointer in a block. /// In Debug checks for simple corruptions. - static void* follow_next(void* node) + static SNMALLOC_FAST_PATH void* follow_next(void* node) { #ifdef CHECK_CLIENT uintptr_t next = *static_cast(node); @@ -195,11 +195,13 @@ namespace snmalloc // Block is not full assert(SLAB_SIZE > accounted_for); + // Keep variable so it appears in debugger. size_t length = debug_slab_acyclic_free_list(slab); UNUSED(length); + // Walk bump-free-list-segment accounting for unused space void* curr = pointer_offset(slab, head); - while ((reinterpret_cast(curr) & 1) == 0) + while ((address_cast(curr) & 1) == 0) { // Check we are looking at a correctly aligned block void* start = curr; diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index e870084..0c264c4 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -105,12 +105,13 @@ namespace snmalloc std::atomic top[TOPLEVEL_ENTRIES]; // = {nullptr}; template - FAST_PATH PagemapEntry* + SNMALLOC_FAST_PATH PagemapEntry* get_node(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee - // to see that correctly. + // to see that correctly. The only transistions are monotone and handled + // by the slow path. PagemapEntry* value = e->load(std::memory_order_relaxed); if (likely(value > LOCKED_ENTRY)) @@ -129,7 +130,7 @@ namespace snmalloc } } - SLOW_PATH PagemapEntry* + SNMALLOC_SLOW_PATH PagemapEntry* get_node_slow(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero @@ -163,7 +164,7 @@ namespace snmalloc } template - FAST_PATH std::pair + SNMALLOC_FAST_PATH std::pair get_leaf_index(uintptr_t addr, bool& result) { #ifdef FreeBSD_KERNEL @@ -207,7 +208,7 @@ namespace snmalloc } template - FAST_PATH std::atomic* get_addr(uintptr_t p, bool& success) + SNMALLOC_FAST_PATH std::atomic* get_addr(uintptr_t p, bool& success) { auto leaf_ix = get_leaf_index(p, success); return &(leaf_ix.first->values[leaf_ix.second]); diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 0665fed..eb2ef61 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -5,6 +5,7 @@ namespace snmalloc { // Both usings should compile + // We use size_t as it generates better code. using sizeclass_t = size_t; // using sizeclass_t = uint8_t; @@ -22,8 +23,12 @@ namespace snmalloc // Don't use sizeclasses that are not a multiple of the alignment. // For example, 24 byte allocations can be // problematic for some data due to alignment issues. - return static_cast( + auto sc = static_cast( bits::to_exp_mant_const(size)); + + assert(sc == static_cast(sc)); + + return sc; } constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class) @@ -140,27 +145,28 @@ namespace snmalloc } #ifdef CACHE_FRIENDLY_OFFSET - inline static void* + SNMALLOC_FAST_PATH static void* remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) { size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass); return p = (void*)((uintptr_t)p & mask); } - inline static uint16_t + SNMALLOC_FAST_PATH static uint16_t remove_cache_friendly_offset(uint16_t relative, sizeclass_t sizeclass) { size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass); return relative & mask; } #else - inline static void* + SNMALLOC_FAST_PATH static void* remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) { UNUSED(sizeclass); return p; } - inline static uint16_t + + SNMALLOC_FAST_PATH static uint16_t remove_cache_friendly_offset(uint16_t relative, sizeclass_t sizeclass) { UNUSED(sizeclass); diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index d0e3bb1..e138abd 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -7,11 +7,14 @@ namespace snmalloc { constexpr size_t PTR_BITS = bits::next_pow2_bits_const(sizeof(void*)); - constexpr static size_t sizeclass_lookup_index(size_t s) + constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s) { - // We shift by PTR_BITS as makes code-gen for the table lookup nicer. - // We could shift by MIN_ALLOC_BITS, but then there is a slightly more - // complex sequence. + // We subtract and shirt to reduce the size of the table, i.e. we don't have + // to store a value for every size class. + // We could shift by MIN_ALLOC_BITS, as this would give us the most + // compressed table, but by shifting by PTR_BITS the code-gen is better + // as the most important path using this subsequently shifts left by + // PTR_BITS, hence they can be fused into a single mask. return (s - 1) >> PTR_BITS; } @@ -112,7 +115,7 @@ namespace snmalloc if ((size - 1) <= (SLAB_SIZE - 1)) { auto index = sizeclass_lookup_index(size); - ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE)); + SNMALLOC_ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE)); return sizeclass_metadata.sizeclass_lookup[index]; } diff --git a/src/mem/slab.h b/src/mem/slab.h index 387bce6..24c4fe2 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -150,7 +150,7 @@ namespace snmalloc // Returns true, if it deallocation can proceed without changing any status // bits. Note that this does remove the use from the meta slab, so it // doesn't need doing on the slow path. - FAST_PATH bool dealloc_fast(Superslab* super, void* p) + SNMALLOC_FAST_PATH bool dealloc_fast(Superslab* super, void* p) { Metaslab& meta = super->get_meta(this); #ifdef CHECK_CLIENT @@ -187,7 +187,7 @@ namespace snmalloc // Returns a complex return code for managing the superslab meta data. // i.e. This deallocation could make an entire superslab free. template - SLOW_PATH typename Superslab::Action dealloc_slow( + SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow( SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider) { Metaslab& meta = super->get_meta(this); diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index ef3fd8a..d19cd16 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -28,7 +28,7 @@ namespace snmalloc class ThreadAllocUntypedWrapper { public: - static FAST_PATH Alloc*& get() + static SNMALLOC_FAST_PATH Alloc*& get() { return (Alloc*&)ThreadAllocUntyped::get(); } @@ -76,7 +76,7 @@ namespace snmalloc * The non-create case exists so that the `per_thread` variable can be a * local static and not a global, allowing ODR to deduplicate it. */ - static FAST_PATH Alloc*& get(bool create = true) + static SNMALLOC_FAST_PATH Alloc*& get(bool create = true) { static thread_local Alloc* per_thread; if (!per_thread && create) @@ -224,7 +224,7 @@ namespace snmalloc * Private accessor to the per thread allocator * Provides no checking for initialization */ - static FAST_PATH Alloc*& inner_get() + static SNMALLOC_FAST_PATH Alloc*& inner_get() { static thread_local Alloc* per_thread; return per_thread; @@ -242,7 +242,7 @@ namespace snmalloc /** * Private initialiser for the per thread allocator */ - static SLOW_PATH Alloc*& inner_init() + static SNMALLOC_SLOW_PATH Alloc*& inner_init() { Alloc*& per_thread = inner_get(); @@ -280,7 +280,7 @@ namespace snmalloc * Public interface, returns the allocator for the current thread, * constructing it if necessary. */ - static FAST_PATH Alloc*& get() + static SNMALLOC_FAST_PATH Alloc*& get() { Alloc*& per_thread = inner_get(); From b74116e209164b476d1745fb14b3d88ff93c3a19 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 14:58:13 +0100 Subject: [PATCH 23/28] Fixes --- src/ds/bits.h | 4 ++-- src/mem/alloc.h | 6 ++++-- src/mem/sizeclasstable.h | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ds/bits.h b/src/ds/bits.h index eedd0f0..5211f70 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -133,9 +133,9 @@ namespace snmalloc inline void prefetch(void* ptr) { #if defined(PLATFORM_IS_X86) - _mm_prefetch(reinterpret_cast(ptr), _MM_HINT_T0); + _mm_prefetch(reinterpret_cast(ptr), _MM_HINT_T0); #else -# warning "Missing pause intrinsic" +# warning "Missing prefetch intrinsic" #endif } diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 17c8619..aa210a0 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -837,8 +837,10 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional of a critical path. - message_queue().init( - reinterpret_cast(alloc(MIN_ALLOC_SIZE))); + Remote* remote = + reinterpret_cast(alloc(MIN_ALLOC_SIZE)); + remote->set_target_id(id()); + message_queue().init(remote); } SNMALLOC_FAST_PATH void handle_dealloc_remote(Remote* p) diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index e138abd..5e067db 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -104,7 +104,7 @@ namespace snmalloc return sizeclass_metadata.cache_friendly_mask[sizeclass]; } - constexpr static inline size_t + constexpr static SNMALLOC_FAST_PATH size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sizeclass) { return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass]; From 54879fbb4aefb9593f2cd55c769afb52a373c105 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 15:38:28 +0100 Subject: [PATCH 24/28] Make GCC happy with inline --- src/ds/bits.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ds/bits.h b/src/ds/bits.h index 5211f70..e6721cd 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -22,7 +22,7 @@ # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) # define SNMALLOC_SLOW_PATH NOINLINE -# define SNMALLOC_FAST_PATH ALWAYSINLINE +# define SNMALLOC_FAST_PATH inline ALWAYSINLINE # define SNMALLOC_PURE __attribute__((const)) # ifdef __clang__ # define HEADER_GLOBAL __attribute__((selectany)) From 57e94b5824d14a43905853398bd08e91a005c199 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 15:38:48 +0100 Subject: [PATCH 25/28] Ensure id set on dummy deallocation. --- src/mem/alloc.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index aa210a0..42e58e9 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -837,10 +837,10 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional of a critical path. - Remote* remote = + Remote* dummy = reinterpret_cast(alloc(MIN_ALLOC_SIZE)); - remote->set_target_id(id()); - message_queue().init(remote); + dummy->set_target_id(id()); + message_queue().init(dummy); } SNMALLOC_FAST_PATH void handle_dealloc_remote(Remote* p) From 8970e708060916246a8cddcac148deb5494327d8 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 15:44:10 +0100 Subject: [PATCH 26/28] Clangformat --- src/mem/alloc.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 42e58e9..5e09d70 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -837,8 +837,7 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional of a critical path. - Remote* dummy = - reinterpret_cast(alloc(MIN_ALLOC_SIZE)); + Remote* dummy = reinterpret_cast(alloc(MIN_ALLOC_SIZE)); dummy->set_target_id(id()); message_queue().init(dummy); } From ea399660ce686547111099f8d1e922f74ce2bdcb Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 15:58:19 +0100 Subject: [PATCH 27/28] Fix for zero size allocations. --- src/mem/alloc.h | 2 ++ src/override/malloc.cc | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 5e09d70..e63a4ec 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -473,6 +473,8 @@ namespace snmalloc SNMALLOC_SLOW_PATH void dealloc_not_small(void* p, uint8_t size) { handle_message_queue(); + + if (p == nullptr) return; if (size == PMMediumslab) { diff --git a/src/override/malloc.cc b/src/override/malloc.cc index 1e55b84..42c1fd2 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -28,9 +28,6 @@ extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free)(void* ptr) { - if (ptr == nullptr) - return; - ThreadAlloc::get()->dealloc(ptr); } From 4cf19f3a4c81308d0aa22869b49abcfc62821647 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 16:18:03 +0100 Subject: [PATCH 28/28] Clangformat. --- src/mem/alloc.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index e63a4ec..7e5e569 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -473,8 +473,9 @@ namespace snmalloc SNMALLOC_SLOW_PATH void dealloc_not_small(void* p, uint8_t size) { handle_message_queue(); - - if (p == nullptr) return; + + if (p == nullptr) + return; if (size == PMMediumslab) {