diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a5eb4b..e77a7d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,7 +105,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi") set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG") else() - add_compile_options(-march=native -fno-exceptions -fno-rtti -g) + add_compile_options(-march=native -fno-exceptions -fno-rtti -g -ftls-model=initial-exec) endif() macro(subdirlist result curdir) @@ -120,7 +120,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) endmacro() macro(add_shim name) - add_library(${name} SHARED src/override/malloc.cc) + add_library(${name} SHARED src/override/malloc.cc src/override/new.cc) target_link_libraries(${name} snmalloc_lib) target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))") set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e136fd0..7a22327 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -172,8 +172,8 @@ phases: cmakeArgs: '..' - script: | - if [ "$(git diff $(Build.SourceVersion))" != "" ]; then exit 1; fi make clangformat + if [ "$(git diff $(Build.SourceVersion))" != "" ]; then exit 1; fi workingDirectory: build displayName: 'Clang-Format' diff --git a/difference.md b/difference.md index 7cc6237..aa8bb1c 100644 --- a/difference.md +++ b/difference.md @@ -18,3 +18,22 @@ This document outlines the changes that have diverged from The value 1, is never a valid bump allocation value, as we initially allocate the first entry as the link, so we can use 1 as the no more bump space value. + + 2. Separate Bump/Free list. We have separate bump ptr and free list. This + is required to have a "fast free list" in each allocator for each + sizeclass. We bump allocate a whole os page (4KiB) worth of allocations + in one go, so that the CPU predicts the free list path for the fast + path. + + 3. Per allocator per sizeclass fast free list. Each allocator has an array + for each small size class that contains a free list of some elements for + that sizeclass. This enables a very compressed path for the common + allocation case. + + 4. We now store a direct pointer to the next element in each slabs free list + rather than a relative offset into the slab. This enables list + calculation on the fast path. + + +[2-4] Are changes that are directly inspired by +(mimalloc)[http://github.com/microsoft/mimalloc]. \ No newline at end of file diff --git a/src/ds/bits.h b/src/ds/bits.h index 9ba07d3..e6721cd 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -11,6 +11,9 @@ # define HEADER_GLOBAL __declspec(selectany) # define likely(x) !!(x) # define unlikely(x) !!(x) +# define SNMALLOC_SLOW_PATH NOINLINE +# define SNMALLOC_FAST_PATH ALWAYSINLINE +# define SNMALLOC_PURE #else # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) @@ -18,6 +21,9 @@ # include # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) +# define SNMALLOC_SLOW_PATH NOINLINE +# define SNMALLOC_FAST_PATH inline ALWAYSINLINE +# define SNMALLOC_PURE __attribute__((const)) # ifdef __clang__ # define HEADER_GLOBAL __attribute__((selectany)) # else @@ -52,13 +58,17 @@ #define UNUSED(x) ((void)(x)) -#if __has_builtin(__builtin_assume) -# define SNMALLOC_ASSUME(x) __builtin_assume(x) +#ifndef NDEBUG +# define SNMALLOC_ASSUME(x) assert(x) #else -# define SNMALLOC_ASSUME(x) \ - do \ - { \ - } while (0) +# if __has_builtin(__builtin_assume) +# define SNMALLOC_ASSUME(x) __builtin_assume((x)) +# else +# define SNMALLOC_ASSUME(x) \ + do \ + { \ + } while (0) +# endif #endif // #define USE_LZCNT @@ -120,6 +130,15 @@ namespace snmalloc #endif } + inline void prefetch(void* ptr) + { +#if defined(PLATFORM_IS_X86) + _mm_prefetch(reinterpret_cast(ptr), _MM_HINT_T0); +#else +# warning "Missing prefetch intrinsic" +#endif + } + inline uint64_t tick() { #if defined(PLATFORM_IS_X86) diff --git a/src/ds/helpers.h b/src/ds/helpers.h index e94227d..fdae63f 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -27,7 +27,7 @@ namespace snmalloc // If defined should be initially false; assert(first == nullptr || *first == false); - if (!initialised.load(std::memory_order_acquire)) + if (unlikely(!initialised.load(std::memory_order_acquire))) { FlagLock lock(flag); if (!initialised) diff --git a/src/ds/mpscq.h b/src/ds/mpscq.h index f554473..1b34a91 100644 --- a/src/ds/mpscq.h +++ b/src/ds/mpscq.h @@ -3,6 +3,7 @@ #include "bits.h" #include "helpers.h" +#include namespace snmalloc { template @@ -13,8 +14,8 @@ namespace snmalloc std::is_same>::value, "T->next must be a std::atomic"); - std::atomic back; - T* front; + std::atomic back = nullptr; + T* front = nullptr; public: void invariant() @@ -59,7 +60,7 @@ namespace snmalloc prev->next.store(first, std::memory_order_relaxed); } - T* dequeue() + std::pair dequeue() { // Returns the front message, or null if not possible to return a message. invariant(); @@ -69,14 +70,14 @@ namespace snmalloc if (next != nullptr) { front = next; - + bits::prefetch(&(next->next)); assert(front); std::atomic_thread_fence(std::memory_order_acquire); invariant(); - return first; + return std::pair(first, true); } - return nullptr; + return std::pair(nullptr, false); } }; } // namespace snmalloc diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 703c163..7e5e569 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1,10 +1,5 @@ #pragma once -#if !defined(NDEBUG) && !defined(OPEN_ENCLAVE) && !defined(FreeBSD_KERNEL) && \ - !defined(USE_SNMALLOC_STATS) -# define USE_SNMALLOC_STATS -#endif - #ifdef _MSC_VER # define ALLOCATOR __declspec(allocator) #else @@ -226,6 +221,19 @@ namespace snmalloc # define SNMALLOC_DEFAULT_PAGEMAP snmalloc::SuperslabMap<> #endif + // This class is just used so that the free lists are the first entry + // in the allocator and hence has better code gen. + // It contains a free list per small size class. These are used for + // allocation on the fast path. This part of the code is inspired by mimalloc. + class FastFreeLists + { + protected: + FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES]; + + public: + FastFreeLists() : small_fast_free_lists() {} + }; + /** * Allocator. This class is parameterised on three template parameters. The * `MemoryProvider` defines the source of memory for this allocator. @@ -247,7 +255,8 @@ namespace snmalloc class PageMap = SNMALLOC_DEFAULT_PAGEMAP, bool IsQueueInline = true> class Allocator - : public Pooled> + : public FastFreeLists, + public Pooled> { LargeAlloc large_allocator; PageMap page_map; @@ -277,32 +286,31 @@ namespace snmalloc else return calloc(1, size); #else - constexpr uint8_t sizeclass = size_to_sizeclass_const(size); + constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size); stats().alloc_request(size); - handle_message_queue(); - // Allocate memory of a statically known size. if constexpr (sizeclass < NUM_SMALL_CLASSES) { - constexpr size_t rsize = sizeclass_to_size(sizeclass); - return small_alloc(sizeclass, rsize); + return small_alloc(size); } else if constexpr (sizeclass < NUM_SIZECLASSES) { + handle_message_queue(); constexpr size_t rsize = sizeclass_to_size(sizeclass); return medium_alloc(sizeclass, rsize, size); } else { + handle_message_queue(); return large_alloc(size); } #endif } template - ALLOCATOR void* alloc(size_t size) + inline ALLOCATOR void* alloc(size_t size) { #ifdef USE_MALLOC static_assert( @@ -315,18 +323,30 @@ namespace snmalloc #else stats().alloc_request(size); - handle_message_queue(); - - uint8_t sizeclass = size_to_sizeclass(size); - // Allocate memory of a dynamically known size. - if (sizeclass < NUM_SMALL_CLASSES) + // Perform the - 1 on size, so that zero wraps around and ends up on + // slow path. + if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1))) { // Allocations smaller than the slab size are more likely. Improve // branch prediction by placing this case first. - size_t rsize = sizeclass_to_size(sizeclass); - return small_alloc(sizeclass, rsize); + return small_alloc(size); } + + return alloc_not_small(size); + } + + template + SNMALLOC_SLOW_PATH ALLOCATOR void* alloc_not_small(size_t size) + { + handle_message_queue(); + + if (size == 0) + { + return small_alloc(1); + } + + sizeclass_t sizeclass = size_to_sizeclass(size); if (sizeclass < NUM_SIZECLASSES) { size_t rsize = sizeclass_to_size(sizeclass); @@ -346,7 +366,7 @@ namespace snmalloc return free(p); #else - constexpr uint8_t sizeclass = size_to_sizeclass_const(size); + constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size); handle_message_queue(); @@ -389,7 +409,7 @@ namespace snmalloc // Free memory of a dynamically known size. Must be called with an // external pointer. - uint8_t sizeclass = size_to_sizeclass(size); + sizeclass_t sizeclass = size_to_sizeclass(size); if (sizeclass < NUM_SMALL_CLASSES) { @@ -418,25 +438,19 @@ namespace snmalloc #endif } - void dealloc(void* p) + SNMALLOC_FAST_PATH void dealloc(void* p) { #ifdef USE_MALLOC return free(p); #else - handle_message_queue(); // Free memory of an unknown size. Must be called with an external // pointer. uint8_t size = pagemap().get(address_cast(p)); - if (size == 0) - { - error("Not allocated by this allocator"); - } - Superslab* super = Superslab::get(p); - if (size == PMSuperslab) + if (likely(size == PMSuperslab)) { RemoteAllocator* target = super->get_allocator(); Slab* slab = Slab::get(p); @@ -445,14 +459,24 @@ namespace snmalloc // Reading a remote sizeclass won't fail, since the other allocator // can't reuse the slab, as we have not yet deallocated this // pointer. - uint8_t sizeclass = meta.sizeclass; + sizeclass_t sizeclass = meta.sizeclass; - if (super->get_allocator() == public_state()) + if (likely(super->get_allocator() == public_state())) small_dealloc(super, p, sizeclass); else remote_dealloc(target, p, sizeclass); return; } + dealloc_not_small(p, size); + } + + SNMALLOC_SLOW_PATH void dealloc_not_small(void* p, uint8_t size) + { + handle_message_queue(); + + if (p == nullptr) + return; + if (size == PMMediumslab) { Mediumslab* slab = Mediumslab::get(p); @@ -460,7 +484,7 @@ namespace snmalloc // Reading a remote sizeclass won't fail, since the other allocator // can't reuse the slab, as we have no yet deallocated this pointer. - uint8_t sizeclass = slab->get_sizeclass(); + sizeclass_t sizeclass = slab->get_sizeclass(); if (target == public_state()) medium_dealloc(slab, p, sizeclass); @@ -469,7 +493,13 @@ namespace snmalloc return; } + if (size == 0) + { + error("Not allocated by this allocator"); + } + # ifdef CHECK_CLIENT + Superslab* super = Superslab::get(p); if (size > 64 || address_cast(super) != address_cast(p)) { error("Not deallocating start of an object"); @@ -494,7 +524,7 @@ namespace snmalloc Slab* slab = Slab::get(p); Metaslab& meta = super->get_meta(slab); - uint8_t sc = meta.sizeclass; + sizeclass_t sc = meta.sizeclass; size_t slab_end = static_cast(address_cast(slab) + SLAB_SIZE); return external_pointer(p, sc, slab_end); @@ -503,7 +533,7 @@ namespace snmalloc { Mediumslab* slab = Mediumslab::get(p); - uint8_t sc = slab->get_sizeclass(); + sizeclass_t sc = slab->get_sizeclass(); size_t slab_end = static_cast(address_cast(slab) + SUPERSLAB_SIZE); @@ -622,7 +652,8 @@ namespace snmalloc return (id >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK; } - void dealloc(alloc_id_t target_id, void* p, uint8_t sizeclass) + SNMALLOC_FAST_PATH void + dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass) { this->size += sizeclass_to_size(sizeclass); @@ -698,7 +729,6 @@ namespace snmalloc DLList super_only_short_available; RemoteCache remote; - Remote stub; std::conditional_t remote_alloc; @@ -706,7 +736,7 @@ namespace snmalloc #ifdef CACHE_FRIENDLY_OFFSET size_t remote_offset = 0; - void* apply_cache_friendly_offset(void* p, uint8_t sizeclass) + void* apply_cache_friendly_offset(void* p, sizeclass_t sizeclass) { size_t mask = sizeclass_to_cache_friendly_mask(sizeclass); @@ -716,7 +746,7 @@ namespace snmalloc return (void*)((uintptr_t)p + offset); } #else - void* apply_cache_friendly_offset(void* p, uint8_t sizeclass) + void* apply_cache_friendly_offset(void* p, sizeclass_t sizeclass) { UNUSED(sizeclass); return p; @@ -770,11 +800,11 @@ namespace snmalloc message_queue().invariant(); #ifndef NDEBUG - for (uint8_t i = 0; i < NUM_SIZECLASSES; i++) + for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) { size_t size = sizeclass_to_size(i); - uint8_t sc1 = size_to_sizeclass(size); - uint8_t sc2 = size_to_sizeclass_const(size); + sizeclass_t sc1 = size_to_sizeclass(size); + sizeclass_t sc2 = size_to_sizeclass_const(size); size_t size1 = sizeclass_to_size(sc1); size_t size2 = sizeclass_to_size(sc2); @@ -794,7 +824,7 @@ namespace snmalloc template static uintptr_t - external_pointer(void* p, uint8_t sizeclass, size_t end_point) + external_pointer(void* p, sizeclass_t sizeclass, size_t end_point) { size_t rsize = sizeclass_to_size(sizeclass); size_t end_point_correction = location == End ? @@ -808,72 +838,83 @@ namespace snmalloc void init_message_queue() { - message_queue().init(&stub); + // Manufacture an allocation to prime the queue + // Using an actual allocation removes a conditional of a critical path. + Remote* dummy = reinterpret_cast(alloc(MIN_ALLOC_SIZE)); + dummy->set_target_id(id()); + message_queue().init(dummy); } - void handle_dealloc_remote(Remote* p) + SNMALLOC_FAST_PATH void handle_dealloc_remote(Remote* p) { - if (p != &stub) - { - Superslab* super = Superslab::get(p); + Superslab* super = Superslab::get(p); #ifdef CHECK_CLIENT - if (p->target_id() != super->get_allocator()->id()) - error("Detected memory corruption. Potential use-after-free"); + if (p->target_id() != super->get_allocator()->id()) + error("Detected memory corruption. Potential use-after-free"); #endif - if (super->get_kind() == Super) + if (likely(super->get_kind() == Super)) + { + Slab* slab = Slab::get(p); + Metaslab& meta = super->get_meta(slab); + if (likely(p->target_id() == id())) { - Slab* slab = Slab::get(p); - Metaslab& meta = super->get_meta(slab); - if (p->target_id() == id()) - { - small_dealloc_offseted(super, p, meta.sizeclass); - } - else - { - // Queue for remote dealloc elsewhere. - remote.dealloc(p->target_id(), p, meta.sizeclass); - } + small_dealloc_offseted(super, p, meta.sizeclass); + return; + } + } + handle_dealloc_remote_slow(p); + } + + SNMALLOC_SLOW_PATH void handle_dealloc_remote_slow(Remote* p) + { + Superslab* super = Superslab::get(p); + if (likely(super->get_kind() == Medium)) + { + Mediumslab* slab = Mediumslab::get(p); + if (p->target_id() == id()) + { + sizeclass_t sizeclass = slab->get_sizeclass(); + void* start = remove_cache_friendly_offset(p, sizeclass); + medium_dealloc(slab, start, sizeclass); } else { - Mediumslab* slab = Mediumslab::get(p); - if (p->target_id() == id()) - { - uint8_t sizeclass = slab->get_sizeclass(); - void* start = remove_cache_friendly_offset(p, sizeclass); - medium_dealloc(slab, start, sizeclass); - } - else - { - // Queue for remote dealloc elsewhere. - remote.dealloc(p->target_id(), p, slab->get_sizeclass()); - } + // Queue for remote dealloc elsewhere. + remote.dealloc(p->target_id(), p, slab->get_sizeclass()); } } + else + { + assert(likely(p->target_id() != id())); + Slab* slab = Slab::get(p); + Metaslab& meta = super->get_meta(slab); + // Queue for remote dealloc elsewhere. + remote.dealloc(p->target_id(), p, meta.sizeclass); + } } - NOINLINE void handle_message_queue_inner() + SNMALLOC_SLOW_PATH void handle_message_queue_inner() { for (size_t i = 0; i < REMOTE_BATCH; i++) { - Remote* r = message_queue().dequeue(); + auto r = message_queue().dequeue(); - if (r == nullptr) + if (unlikely(!r.second)) break; - handle_dealloc_remote(r); + handle_dealloc_remote(r.first); } // Our remote queues may be larger due to forwarding remote frees. - if (remote.size < REMOTE_CACHE) + if (likely(remote.size < REMOTE_CACHE)) return; stats().remote_post(); remote.post(id()); } - ALWAYSINLINE void handle_message_queue() + SNMALLOC_FAST_PATH void handle_message_queue() { // Inline the empty check, but not necessarily the full queue handling. if (likely(message_queue().is_empty())) @@ -938,7 +979,7 @@ namespace snmalloc } template - Slab* alloc_slab(uint8_t sizeclass) + Slab* alloc_slab(sizeclass_t sizeclass) { stats().sizeclass_alloc_slab(sizeclass); if (Superslab::is_short_sizeclass(sizeclass)) @@ -978,7 +1019,7 @@ namespace snmalloc } template - void* small_alloc(uint8_t sizeclass, size_t rsize) + inline void* small_alloc(size_t size) { MEASURE_TIME_MARKERS( small_alloc, @@ -988,14 +1029,41 @@ namespace snmalloc zero_mem == YesZero ? "zeromem" : "nozeromem", allow_reserve == NoReserve ? "noreserve" : "reserve")); + SNMALLOC_ASSUME(size <= SLAB_SIZE); + sizeclass_t sizeclass = size_to_sizeclass(size); stats().sizeclass_alloc(sizeclass); - SlabList* sc = &small_classes[sizeclass]; + assert(sizeclass < NUM_SMALL_CLASSES); + auto& fl = small_fast_free_lists[sizeclass]; + auto head = fl.value; + if (likely((reinterpret_cast(head) & 1) == 0)) + { + void* p = head; + // Read the next slot from the memory that's about to be allocated. + fl.value = Metaslab::follow_next(p); + + if constexpr (zero_mem == YesZero) + { + large_allocator.memory_provider.zero(p, size); + } + return p; + } + + return small_alloc_slow(sizeclass); + } + + template + SNMALLOC_SLOW_PATH void* small_alloc_slow(sizeclass_t sizeclass) + { + handle_message_queue(); + size_t rsize = sizeclass_to_size(sizeclass); + auto& sl = small_classes[sizeclass]; + Slab* slab; - if (!sc->is_empty()) + if (!sl.is_empty()) { - SlabLink* link = sc->get_head(); + SlabLink* link = sl.get_head(); slab = link->get_slab(); } else @@ -1005,13 +1073,15 @@ namespace snmalloc if ((allow_reserve == NoReserve) && (slab == nullptr)) return nullptr; - sc->insert(slab->get_link()); + sl.insert(slab->get_link()); } - - return slab->alloc(sc, rsize, large_allocator.memory_provider); + auto& ffl = small_fast_free_lists[sizeclass]; + return slab->alloc( + sl, ffl, rsize, large_allocator.memory_provider); } - void small_dealloc(Superslab* super, void* p, uint8_t sizeclass) + SNMALLOC_FAST_PATH void + small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) { #ifdef CHECK_CLIENT Slab* slab = Slab::get(p); @@ -1025,19 +1095,29 @@ namespace snmalloc small_dealloc_offseted(super, offseted, sizeclass); } - void small_dealloc_offseted(Superslab* super, void* p, uint8_t sizeclass) + SNMALLOC_FAST_PATH void + small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) { MEASURE_TIME(small_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); - bool was_full = super->is_full(); - SlabList* sc = &small_classes[sizeclass]; Slab* slab = Slab::get(p); - Superslab::Action a = - slab->dealloc(sc, super, p, large_allocator.memory_provider); - if (a == Superslab::NoSlabReturn) + if (likely(slab->dealloc_fast(super, p))) return; + small_dealloc_offseted_slow(super, p, sizeclass); + } + + SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow( + Superslab* super, void* p, sizeclass_t sizeclass) + { + bool was_full = super->is_full(); + SlabList* sl = &small_classes[sizeclass]; + Slab* slab = Slab::get(p); + Superslab::Action a = + slab->dealloc_slow(sl, super, p, large_allocator.memory_provider); + if (likely(a == Superslab::NoSlabReturn)) + return; stats().sizeclass_dealloc_slab(sizeclass); if (a == Superslab::NoStatusChange) @@ -1100,7 +1180,7 @@ namespace snmalloc } template - void* medium_alloc(uint8_t sizeclass, size_t rsize, size_t size) + void* medium_alloc(sizeclass_t sizeclass, size_t rsize, size_t size) { MEASURE_TIME_MARKERS( medium_alloc, @@ -1110,7 +1190,7 @@ namespace snmalloc zero_mem == YesZero ? "zeromem" : "nozeromem", allow_reserve == NoReserve ? "noreserve" : "reserve")); - uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES; + sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; DLList* sc = &medium_classes[medium_class]; Mediumslab* slab = sc->get_head(); @@ -1144,7 +1224,7 @@ namespace snmalloc return p; } - void medium_dealloc(Mediumslab* slab, void* p, uint8_t sizeclass) + void medium_dealloc(Mediumslab* slab, void* p, sizeclass_t sizeclass) { MEASURE_TIME(medium_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); @@ -1163,7 +1243,7 @@ namespace snmalloc { if (!was_full) { - uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES; + sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; DLList* sc = &medium_classes[medium_class]; sc->remove(slab); } @@ -1180,7 +1260,7 @@ namespace snmalloc } else if (was_full) { - uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES; + sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; DLList* sc = &medium_classes[medium_class]; sc->insert(slab); } @@ -1233,10 +1313,13 @@ namespace snmalloc large_allocator.dealloc(slab, large_class); } - void remote_dealloc(RemoteAllocator* target, void* p, uint8_t sizeclass) + SNMALLOC_FAST_PATH void + remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass) { MEASURE_TIME(remote_dealloc, 4, 16); + handle_message_queue(); + void* offseted = apply_cache_friendly_offset(p, sizeclass); stats().remote_free(sizeclass); diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h index b047a3a..83e0b2e 100644 --- a/src/mem/allocstats.h +++ b/src/mem/allocstats.h @@ -1,6 +1,7 @@ #pragma once #include "../ds/bits.h" +#include "../mem/sizeclass.h" #include @@ -165,7 +166,7 @@ namespace snmalloc #endif } - void sizeclass_alloc(uint8_t sc) + void sizeclass_alloc(sizeclass_t sc) { UNUSED(sc); @@ -175,7 +176,7 @@ namespace snmalloc #endif } - void sizeclass_dealloc(uint8_t sc) + void sizeclass_dealloc(sizeclass_t sc) { UNUSED(sc); @@ -194,7 +195,7 @@ namespace snmalloc #endif } - void sizeclass_alloc_slab(uint8_t sc) + void sizeclass_alloc_slab(sizeclass_t sc) { UNUSED(sc); @@ -204,7 +205,7 @@ namespace snmalloc #endif } - void sizeclass_dealloc_slab(uint8_t sc) + void sizeclass_dealloc_slab(sizeclass_t sc) { UNUSED(sc); @@ -251,7 +252,7 @@ namespace snmalloc #endif } - void remote_free(uint8_t sc) + void remote_free(sizeclass_t sc) { UNUSED(sc); @@ -267,7 +268,7 @@ namespace snmalloc #endif } - void remote_receive(uint8_t sc) + void remote_receive(sizeclass_t sc) { UNUSED(sc); @@ -348,7 +349,7 @@ namespace snmalloc << "Count" << csv.endl; } - for (uint8_t i = 0; i < N; i++) + for (sizeclass_t i = 0; i < N; i++) { if (sizeclass[i].count.is_unused()) continue; diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 929cbd7..6d5ee5c 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -189,7 +189,7 @@ namespace snmalloc * (over the lifetime of this process). If the underlying system does not * support low memory notifications, this will return 0. */ - ALWAYSINLINE + SNMALLOC_FAST_PATH uint64_t low_memory_epoch() { if constexpr (pal_supports()) @@ -235,7 +235,7 @@ namespace snmalloc } } - ALWAYSINLINE void lazy_decommit_if_needed() + SNMALLOC_FAST_PATH void lazy_decommit_if_needed() { #ifdef TEST_LAZY_DECOMMIT static_assert( diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index ce533d8..38042ad 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -44,7 +44,7 @@ namespace snmalloc return pointer_cast(address_cast(p) & SUPERSLAB_MASK); } - void init(RemoteAllocator* alloc, uint8_t sc, size_t rsize) + void init(RemoteAllocator* alloc, sizeclass_t sc, size_t rsize) { assert(sc >= NUM_SMALL_CLASSES); assert((sc - NUM_SMALL_CLASSES) < NUM_MEDIUM_CLASSES); @@ -56,7 +56,7 @@ namespace snmalloc // initialise the allocation stack. if ((kind != Medium) || (sizeclass != sc)) { - sizeclass = sc; + sizeclass = static_cast(sc); uint16_t ssize = static_cast(rsize >> 8); kind = Medium; free = medium_slab_free(sc); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 7bb2a32..a7a60b9 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -29,22 +29,24 @@ namespace snmalloc // This can be either a short or a standard slab. class Metaslab { - private: - // How many entries are used in this slab. + public: + // How many entries are not in the free list of slab. uint16_t used = 0; - public: - // Bump free list of unused entries in this sizeclass. - // If the bottom bit is 1, then this represents a bump_ptr - // of where we have allocated up to in this slab. Otherwise, - // it represents the location of the first block in the free - // list. The free list is chained through deallocated blocks. - // It is terminated with a bump ptr. - // - // Note that, the first entry in a slab is never bump allocated - // but is used for the link. This means that 1 represents the fully - // bump allocated slab. + // How many entries have been allocated from this slab. + uint16_t allocated; + + // Index of first entry in this slab that forms the free + // list. The list entries are stored as the first pointer + // in each unused object. The terminator is a pointer or + // offset into the block with the bottom bit set. This means + // I.e. + // * an empty list has a head of 1. + // * a one element list has an head contains an offset to this + // this block, and then contains a pointer with the bottom + // bit set. Mod head; + // When a slab has free space it will be on the has space list for // that size class. We use an empty block in this slab to be the // doubly linked node into that size class's free list. @@ -93,35 +95,37 @@ namespace snmalloc /// Value used to check for corruptions in a block static constexpr size_t POISON = - static_cast(bits::is64() ? 0xDEADBEEFDEAD0000 : 0xDEAD0000); + static_cast(bits::is64() ? 0xDEADBEEFDEADBEEF : 0xDEADBEEF); /// Store next pointer in a block. In Debug using magic value to detect some /// simple corruptions. - static void store_next(void* p, uint16_t head) + static SNMALLOC_FAST_PATH void store_next(void* p, void* head) { #ifndef CHECK_CLIENT - *static_cast(p) = head; + *static_cast(p) = head; #else - *static_cast(p) = - head ^ POISON ^ (static_cast(head) << (bits::BITS - 16)); + *static_cast(p) = head; + *(static_cast(p) + 1) = address_cast(head) ^ POISON; #endif } /// Accessor function for the next pointer in a block. /// In Debug checks for simple corruptions. - static uint16_t follow_next(void* node) + static SNMALLOC_FAST_PATH void* follow_next(void* node) { - size_t next = *static_cast(node); #ifdef CHECK_CLIENT - if (((next ^ POISON) ^ (next << (bits::BITS - 16))) > 0xFFFF) + uintptr_t next = *static_cast(node); + uintptr_t chk = *(static_cast(node) + 1); + if ((next ^ chk) != POISON) error("Detected memory corruption. Use-after-free."); #endif - return static_cast(next); + return *static_cast(node); } + bool valid_head(bool is_short) { size_t size = sizeclass_to_size(sizeclass); - size_t slab_start = get_initial_link(sizeclass, is_short); + size_t slab_start = get_initial_offset(sizeclass, is_short); size_t all_high_bits = ~static_cast(1); size_t head_start = @@ -138,18 +142,19 @@ namespace snmalloc * We don't expect a cycle, so worst case is only followed by a crash, so * slow doesn't mater. **/ - void debug_slab_acyclic_free_list(Slab* slab) + size_t debug_slab_acyclic_free_list(Slab* slab) { #ifndef NDEBUG - uint16_t curr = head; - uint16_t curr_slow = head; + size_t length = 0; + void* curr = pointer_offset(slab, head); + void* curr_slow = pointer_offset(slab, head); bool both = false; - while ((curr & 1) != 1) + while ((reinterpret_cast(curr) & 1) == 0) { - curr = follow_next(pointer_offset(slab, curr)); + curr = follow_next(curr); if (both) { - curr_slow = follow_next(pointer_offset(slab, curr_slow)); + curr_slow = follow_next(curr_slow); } if (curr == curr_slow) @@ -158,9 +163,12 @@ namespace snmalloc } both = !both; + length++; } + return length; #else UNUSED(slab); + return 0; #endif } @@ -168,7 +176,10 @@ namespace snmalloc { #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) size_t size = sizeclass_to_size(sizeclass); - size_t offset = get_initial_link(sizeclass, is_short); + size_t offset = get_initial_offset(sizeclass, is_short); + + if (is_unused()) + return; size_t accounted_for = used * size + offset; @@ -184,38 +195,41 @@ namespace snmalloc // Block is not full assert(SLAB_SIZE > accounted_for); - debug_slab_acyclic_free_list(slab); + // Keep variable so it appears in debugger. + size_t length = debug_slab_acyclic_free_list(slab); + UNUSED(length); // Walk bump-free-list-segment accounting for unused space - uint16_t curr = head; - while ((curr & 1) != 1) + void* curr = pointer_offset(slab, head); + while ((address_cast(curr) & 1) == 0) { // Check we are looking at a correctly aligned block - uint16_t start = remove_cache_friendly_offset(curr, sizeclass); - assert((start - offset) % size == 0); + void* start = curr; + assert( + ((address_cast(start) - address_cast(slab) - offset) % size) == 0); // Account for free elements in free list accounted_for += size; assert(SLAB_SIZE >= accounted_for); // We should never reach the link node in the free list. - assert(curr != link); + assert(curr != pointer_offset(slab, link)); // Iterate bump/free list segment - curr = follow_next(pointer_offset(slab, curr)); + curr = follow_next(curr); } - if (curr != 1) + auto bumpptr = (allocated * size) + offset; + // Check we haven't allocaated more than gits in a slab + assert(bumpptr <= SLAB_SIZE); + + // Account for to be bump allocated space + accounted_for += SLAB_SIZE - bumpptr; + + if (bumpptr != SLAB_SIZE) { - // Check we terminated traversal on a correctly aligned block - uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass); - assert((start - offset) % size == 0); - - // Account for to be bump allocated space - accounted_for += SLAB_SIZE - (curr - 1); - // The link should be the first allocation as we // haven't completely filled this block at any point. - assert(link == get_initial_link(sizeclass, is_short)); + assert(link == get_initial_offset(sizeclass, is_short)); } assert(!is_full()); diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index 89bfd4f..0c264c4 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -105,7 +105,33 @@ namespace snmalloc std::atomic top[TOPLEVEL_ENTRIES]; // = {nullptr}; template - inline PagemapEntry* get_node(std::atomic* e, bool& result) + SNMALLOC_FAST_PATH PagemapEntry* + get_node(std::atomic* e, bool& result) + { + // The page map nodes are all allocated directly from the OS zero + // initialised with a system call. We don't need any ordered to guarantee + // to see that correctly. The only transistions are monotone and handled + // by the slow path. + PagemapEntry* value = e->load(std::memory_order_relaxed); + + if (likely(value > LOCKED_ENTRY)) + { + result = true; + return value; + } + if constexpr (create_addr) + { + return get_node_slow(e, result); + } + else + { + result = false; + return nullptr; + } + } + + SNMALLOC_SLOW_PATH PagemapEntry* + get_node_slow(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -114,31 +140,23 @@ namespace snmalloc if ((value == nullptr) || (value == LOCKED_ENTRY)) { - if constexpr (create_addr) - { - value = nullptr; + value = nullptr; - if (e->compare_exchange_strong( - value, LOCKED_ENTRY, std::memory_order_relaxed)) - { - auto& v = default_memory_provider; - value = v.alloc_chunk(); - e->store(value, std::memory_order_release); - } - else - { - while (address_cast(e->load(std::memory_order_relaxed)) == - LOCKED_ENTRY) - { - bits::pause(); - } - value = e->load(std::memory_order_acquire); - } + if (e->compare_exchange_strong( + value, LOCKED_ENTRY, std::memory_order_relaxed)) + { + auto& v = default_memory_provider; + value = v.alloc_chunk(); + e->store(value, std::memory_order_release); } else { - result = false; - return nullptr; + while (address_cast(e->load(std::memory_order_relaxed)) == + LOCKED_ENTRY) + { + bits::pause(); + } + value = e->load(std::memory_order_acquire); } } result = true; @@ -146,7 +164,8 @@ namespace snmalloc } template - inline std::pair get_leaf_index(uintptr_t addr, bool& result) + SNMALLOC_FAST_PATH std::pair + get_leaf_index(uintptr_t addr, bool& result) { #ifdef FreeBSD_KERNEL // Zero the top 16 bits - kernel addresses all have them set, but the @@ -160,7 +179,7 @@ namespace snmalloc for (size_t i = 0; i < INDEX_LEVELS; i++) { PagemapEntry* value = get_node(e, result); - if (!result) + if (unlikely(!result)) return std::pair(nullptr, 0); shift -= BITS_PER_INDEX_LEVEL; @@ -180,7 +199,7 @@ namespace snmalloc Leaf* leaf = reinterpret_cast(get_node(e, result)); - if (!result) + if (unlikely(!result)) return std::pair(nullptr, 0); shift -= BITS_FOR_LEAF; @@ -189,7 +208,7 @@ namespace snmalloc } template - inline std::atomic* get_addr(uintptr_t p, bool& success) + SNMALLOC_FAST_PATH std::atomic* get_addr(uintptr_t p, bool& success) { auto leaf_ix = get_leaf_index(p, success); return &(leaf_ix.first->values[leaf_ix.second]); diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 724ac28..eb2ef61 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -4,29 +4,31 @@ namespace snmalloc { - constexpr static uint16_t get_initial_bumpptr(uint8_t sc, bool is_short); - constexpr static uint16_t get_initial_link(uint8_t sc, bool is_short); - constexpr static size_t sizeclass_to_size(uint8_t sizeclass); - constexpr static size_t sizeclass_to_cache_friendly_mask(uint8_t sizeclass); - constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(uint8_t sc); - constexpr static uint16_t medium_slab_free(uint8_t sizeclass); + // Both usings should compile + // We use size_t as it generates better code. + using sizeclass_t = size_t; + // using sizeclass_t = uint8_t; - static inline uint8_t size_to_sizeclass(size_t size) + constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short); + constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); + constexpr static size_t + sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); + constexpr static size_t + sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); + constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass); + static sizeclass_t size_to_sizeclass(size_t size); + + constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) { // Don't use sizeclasses that are not a multiple of the alignment. // For example, 24 byte allocations can be // problematic for some data due to alignment issues. - return static_cast( - bits::to_exp_mant(size)); - } - - constexpr static inline uint8_t size_to_sizeclass_const(size_t size) - { - // Don't use sizeclasses that are not a multiple of the alignment. - // For example, 24 byte allocations can be - // problematic for some data due to alignment issues. - return static_cast( + auto sc = static_cast( bits::to_exp_mant_const(size)); + + assert(sc == static_cast(sc)); + + return sc; } constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class) @@ -143,26 +145,29 @@ namespace snmalloc } #ifdef CACHE_FRIENDLY_OFFSET - inline static void* remove_cache_friendly_offset(void* p, uint8_t sizeclass) + SNMALLOC_FAST_PATH static void* + remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) { size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass); return p = (void*)((uintptr_t)p & mask); } - inline static uint16_t - remove_cache_friendly_offset(uint16_t relative, uint8_t sizeclass) + SNMALLOC_FAST_PATH static uint16_t + remove_cache_friendly_offset(uint16_t relative, sizeclass_t sizeclass) { size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass); return relative & mask; } #else - inline static void* remove_cache_friendly_offset(void* p, uint8_t sizeclass) + SNMALLOC_FAST_PATH static void* + remove_cache_friendly_offset(void* p, sizeclass_t sizeclass) { UNUSED(sizeclass); return p; } - inline static uint16_t - remove_cache_friendly_offset(uint16_t relative, uint8_t sizeclass) + + SNMALLOC_FAST_PATH static uint16_t + remove_cache_friendly_offset(uint16_t relative, sizeclass_t sizeclass) { UNUSED(sizeclass); return relative; diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 06c80aa..5e067db 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -5,31 +5,52 @@ namespace snmalloc { + constexpr size_t PTR_BITS = bits::next_pow2_bits_const(sizeof(void*)); + + constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s) + { + // We subtract and shirt to reduce the size of the table, i.e. we don't have + // to store a value for every size class. + // We could shift by MIN_ALLOC_BITS, as this would give us the most + // compressed table, but by shifting by PTR_BITS the code-gen is better + // as the most important path using this subsequently shifts left by + // PTR_BITS, hence they can be fused into a single mask. + return (s - 1) >> PTR_BITS; + } + + constexpr static size_t sizeclass_lookup_size = + sizeclass_lookup_index(SLAB_SIZE + 1); + struct SizeClassTable { + sizeclass_t sizeclass_lookup[sizeclass_lookup_size] = {{}}; ModArray size; ModArray cache_friendly_mask; ModArray inverse_cache_friendly_mask; - ModArray bump_ptr_start; - ModArray short_bump_ptr_start; - ModArray initial_link_ptr; - ModArray short_initial_link_ptr; + ModArray initial_offset_ptr; + ModArray short_initial_offset_ptr; ModArray medium_slab_slots; constexpr SizeClassTable() : size(), cache_friendly_mask(), inverse_cache_friendly_mask(), - bump_ptr_start(), - short_bump_ptr_start(), - initial_link_ptr(), - short_initial_link_ptr(), + initial_offset_ptr(), + short_initial_offset_ptr(), medium_slab_slots() { - for (uint8_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) + size_t curr = 1; + for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { size[sizeclass] = bits::from_exp_mant(sizeclass); + if (sizeclass < NUM_SMALL_CLASSES) + { + for (; curr <= size[sizeclass]; curr += 1 << PTR_BITS) + { + sizeclass_lookup[sizeclass_lookup_index(curr)] = sizeclass; + } + } size_t alignment = bits::min( bits::one_at_bit(bits::ctz_const(size[sizeclass])), OS_PAGE_SIZE); @@ -40,7 +61,7 @@ namespace snmalloc size_t header_size = sizeof(Superslab); size_t short_slab_size = SLAB_SIZE - header_size; - for (uint8_t i = 0; i < NUM_SMALL_CLASSES; i++) + for (sizeclass_t i = 0; i < NUM_SMALL_CLASSES; i++) { // We align to the end of the block to remove special cases for the // short block. Calculate remainders @@ -48,22 +69,12 @@ namespace snmalloc size_t correction = SLAB_SIZE % size[i]; // First element in the block is the link - initial_link_ptr[i] = static_cast(correction); - short_initial_link_ptr[i] = + initial_offset_ptr[i] = static_cast(correction); + short_initial_offset_ptr[i] = static_cast(header_size + short_correction); - - // Move to object after link. - auto short_after_link = short_initial_link_ptr[i] + size[i]; - size_t after_link = initial_link_ptr[i] + size[i]; - - // Bump ptr has bottom bit set. - // In case we only have one object on this slab check for wrap around. - short_bump_ptr_start[i] = - static_cast((short_after_link + 1) % SLAB_SIZE); - bump_ptr_start[i] = static_cast((after_link + 1) % SLAB_SIZE); } - for (uint8_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) + for (sizeclass_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) { medium_slab_slots[i - NUM_SMALL_CLASSES] = static_cast( (SUPERSLAB_SIZE - Mediumslab::header_size()) / size[i]); @@ -74,40 +85,48 @@ namespace snmalloc static constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); static inline constexpr uint16_t - get_initial_bumpptr(uint8_t sc, bool is_short) + get_initial_offset(sizeclass_t sc, bool is_short) { if (is_short) - return sizeclass_metadata.short_bump_ptr_start[sc]; + return sizeclass_metadata.short_initial_offset_ptr[sc]; - return sizeclass_metadata.bump_ptr_start[sc]; + return sizeclass_metadata.initial_offset_ptr[sc]; } - static inline constexpr uint16_t get_initial_link(uint8_t sc, bool is_short) - { - if (is_short) - return sizeclass_metadata.short_initial_link_ptr[sc]; - - return sizeclass_metadata.initial_link_ptr[sc]; - } - - constexpr static inline size_t sizeclass_to_size(uint8_t sizeclass) + constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) { return sizeclass_metadata.size[sizeclass]; } constexpr static inline size_t - sizeclass_to_cache_friendly_mask(uint8_t sizeclass) + sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass) { return sizeclass_metadata.cache_friendly_mask[sizeclass]; } - constexpr static inline size_t - sizeclass_to_inverse_cache_friendly_mask(uint8_t sizeclass) + constexpr static SNMALLOC_FAST_PATH size_t + sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sizeclass) { return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass]; } - constexpr static inline uint16_t medium_slab_free(uint8_t sizeclass) + static inline sizeclass_t size_to_sizeclass(size_t size) + { + if ((size - 1) <= (SLAB_SIZE - 1)) + { + auto index = sizeclass_lookup_index(size); + SNMALLOC_ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE)); + return sizeclass_metadata.sizeclass_lookup[index]; + } + + // Don't use sizeclasses that are not a multiple of the alignment. + // For example, 24 byte allocations can be + // problematic for some data due to alignment issues. + return static_cast( + bits::to_exp_mant(size)); + } + + constexpr static inline uint16_t medium_slab_free(sizeclass_t sizeclass) { return sizeclass_metadata .medium_slab_slots[(sizeclass - NUM_SMALL_CLASSES)]; diff --git a/src/mem/slab.h b/src/mem/slab.h index 0442740..24c4fe2 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -4,6 +4,12 @@ namespace snmalloc { + struct FreeListHead + { + // Use a value with bottom bit set for empty list. + void* value = pointer_offset(nullptr, 1); + }; + class Slab { private: @@ -31,7 +37,11 @@ namespace snmalloc } template - void* alloc(SlabList* sc, size_t rsize, MemoryProvider& memory_provider) + inline void* alloc( + SlabList& sl, + FreeListHead& fast_free_list, + size_t rsize, + MemoryProvider& memory_provider) { // Read the head from the metadata stored in the superslab. Metaslab& meta = get_meta(); @@ -39,38 +49,83 @@ namespace snmalloc assert(rsize == sizeclass_to_size(meta.sizeclass)); meta.debug_slab_invariant(is_short(), this); - assert(sc->get_head() == (SlabLink*)((size_t)this + meta.link)); + assert(sl.get_head() == (SlabLink*)((size_t)this + meta.link)); assert(!meta.is_full()); - meta.add_use(); + void* p = nullptr; + bool p_has_value = false; - void* p; - - if ((head & 1) == 0) + if (head == 1) { - void* node = pointer_offset(this, head); - - // Read the next slot from the memory that's about to be allocated. - meta.head = Metaslab::follow_next(node); - - p = remove_cache_friendly_offset(node, meta.sizeclass); - } - else - { - if (meta.head == 1) + size_t bumpptr = get_initial_offset(meta.sizeclass, is_short()); + bumpptr += meta.allocated * rsize; + if (bumpptr == SLAB_SIZE) { + meta.add_use(); + assert(meta.used == meta.allocated); + p = pointer_offset(this, meta.link); - sc->pop(); meta.set_full(); + sl.pop(); + p_has_value = true; } else { - // This slab is being bump allocated. - p = pointer_offset(this, head - 1); - meta.head = (head + static_cast(rsize)) & (SLAB_SIZE - 1); + void* curr = nullptr; + bool commit = false; + while (true) + { + size_t newbumpptr = bumpptr + rsize; + auto alignedbumpptr = bits::align_up(bumpptr - 1, OS_PAGE_SIZE); + auto alignednewbumpptr = bits::align_up(newbumpptr, OS_PAGE_SIZE); + + if (alignedbumpptr != alignednewbumpptr) + { + // We have committed once already. + if (commit) + break; + + memory_provider.template notify_using( + pointer_offset(this, alignedbumpptr), + alignednewbumpptr - alignedbumpptr); + + commit = true; + } + + if (curr == nullptr) + { + meta.head = static_cast(bumpptr); + } + else + { + Metaslab::store_next(curr, pointer_offset(this, bumpptr)); + } + curr = pointer_offset(this, bumpptr); + bumpptr = newbumpptr; + meta.allocated = meta.allocated + 1; + } + + assert(curr != nullptr); + Metaslab::store_next(curr, pointer_offset(nullptr, 1)); } } + if (!p_has_value) + { + p = pointer_offset(this, meta.head); + + // Read the next slot from the memory that's about to be allocated. + void* next = Metaslab::follow_next(p); + // Put everything in allocators small_class free list. + meta.head = 1; + fast_free_list.value = next; + // Treat stealing the free list as allocating it all. + // Link is not in use, i.e. - 1 is required. + meta.used = meta.allocated - 1; + } + + assert(is_start_of_object(Superslab::get(p), p)); + meta.debug_slab_invariant(is_short(), this); if constexpr (zero_mem == YesZero) @@ -92,38 +147,58 @@ namespace snmalloc address_cast(this) + SLAB_SIZE - address_cast(p)); } - // Returns true, if it alters get_status. - template - inline typename Superslab::Action dealloc( - SlabList* sc, Superslab* super, void* p, MemoryProvider& memory_provider) + // Returns true, if it deallocation can proceed without changing any status + // bits. Note that this does remove the use from the meta slab, so it + // doesn't need doing on the slow path. + SNMALLOC_FAST_PATH bool dealloc_fast(Superslab* super, void* p) { Metaslab& meta = super->get_meta(this); - - bool was_full = meta.is_full(); - #ifdef CHECK_CLIENT if (meta.is_unused()) error("Detected potential double free."); #endif - meta.debug_slab_invariant(is_short(), this); meta.sub_use(); + bool was_full = meta.is_full(); + if (unlikely(was_full)) + return false; + + bool is_unused = meta.is_unused(); + if (unlikely(is_unused)) + return false; + + // Update the head and the next pointer in the free list. + uint16_t head = meta.head; + uint16_t current = pointer_to_index(p); + + // Set the head to the memory being deallocated. + meta.head = current; + assert(meta.valid_head(is_short())); + + // Set the next pointer to the previous head. + Metaslab::store_next(p, pointer_offset(this, head)); + meta.debug_slab_invariant(is_short(), this); + return true; + } + + // If dealloc fast returns false, then call this. + // This does not need to remove the "use" as done by the fast path. + // Returns a complex return code for managing the superslab meta data. + // i.e. This deallocation could make an entire superslab free. + template + SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow( + SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider) + { + Metaslab& meta = super->get_meta(this); + + bool was_full = meta.is_full(); + bool is_unused = meta.is_unused(); + if (was_full) { // We are not on the sizeclass list. - if (!meta.is_unused()) - { - // Update the head and the sizeclass link. - uint16_t index = pointer_to_index(p); - assert(meta.head == 1); - meta.link = index; - - // Push on the list of slabs for this sizeclass. - sc->insert(meta.get_link(this)); - meta.debug_slab_invariant(is_short(), this); - } - else + if (is_unused) { // Dealloc on the superslab. if (is_short()) @@ -131,37 +206,30 @@ namespace snmalloc return super->dealloc_slab(this, memory_provider); } + // Update the head and the sizeclass link. + uint16_t index = pointer_to_index(p); + assert(meta.head == 1); + // assert(meta.fully_allocated(is_short())); + meta.link = index; + + // Push on the list of slabs for this sizeclass. + sl->insert(meta.get_link(this)); + meta.debug_slab_invariant(is_short(), this); + return Superslab::NoSlabReturn; } - else if (meta.is_unused()) + + if (is_unused) { // Remove from the sizeclass list and dealloc on the superslab. - sc->remove(meta.get_link(this)); + sl->remove(meta.get_link(this)); if (is_short()) return super->dealloc_short_slab(memory_provider); return super->dealloc_slab(this, memory_provider); } - else - { -#ifndef NDEBUG - sc->debug_check_contains(meta.get_link(this)); -#endif - // Update the head and the next pointer in the free list. - uint16_t head = meta.head; - uint16_t current = pointer_to_index(p); - - // Set the head to the memory being deallocated. - meta.head = current; - assert(meta.valid_head(is_short())); - - // Set the next pointer to the previous head. - Metaslab::store_next(p, head); - - meta.debug_slab_invariant(is_short(), this); - } - return Superslab::NoSlabReturn; + abort(); } bool is_short() diff --git a/src/mem/superslab.h b/src/mem/superslab.h index c131d78..66d65a5 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -70,9 +70,9 @@ namespace snmalloc return pointer_cast(address_cast(p) & SUPERSLAB_MASK); } - static bool is_short_sizeclass(uint8_t sizeclass) + static bool is_short_sizeclass(sizeclass_t sizeclass) { - constexpr uint8_t h = size_to_sizeclass_const(sizeof(Superslab)); + constexpr sizeclass_t h = size_to_sizeclass_const(sizeof(Superslab)); return sizeclass <= h; } @@ -154,16 +154,17 @@ namespace snmalloc } template - Slab* alloc_short_slab(uint8_t sizeclass, MemoryProvider& memory_provider) + Slab* + alloc_short_slab(sizeclass_t sizeclass, MemoryProvider& memory_provider) { if ((used & 1) == 1) return alloc_slab(sizeclass, memory_provider); - meta[0].head = get_initial_bumpptr(sizeclass, true); - meta[0].sizeclass = sizeclass; - meta[0].link = get_initial_link(sizeclass, true); + meta[0].allocated = 1; + meta[0].head = 1; + meta[0].sizeclass = static_cast(sizeclass); + meta[0].link = get_initial_offset(sizeclass, true); - if constexpr (decommit_strategy == DecommitAll) { memory_provider.template notify_using( pointer_offset(this, OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE); @@ -174,7 +175,7 @@ namespace snmalloc } template - Slab* alloc_slab(uint8_t sizeclass, MemoryProvider& memory_provider) + Slab* alloc_slab(sizeclass_t sizeclass, MemoryProvider& memory_provider) { uint8_t h = head; Slab* slab = pointer_cast( @@ -182,9 +183,10 @@ namespace snmalloc uint8_t n = meta[h].next; - meta[h].head = get_initial_bumpptr(sizeclass, false); - meta[h].sizeclass = sizeclass; - meta[h].link = get_initial_link(sizeclass, false); + meta[h].head = 1; + meta[h].allocated = 1; + meta[h].sizeclass = static_cast(sizeclass); + meta[h].link = get_initial_offset(sizeclass, false); head = h + n + 1; used += 2; diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 59a05e3..d19cd16 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -28,7 +28,7 @@ namespace snmalloc class ThreadAllocUntypedWrapper { public: - static inline Alloc*& get() + static SNMALLOC_FAST_PATH Alloc*& get() { return (Alloc*&)ThreadAllocUntyped::get(); } @@ -76,7 +76,7 @@ namespace snmalloc * The non-create case exists so that the `per_thread` variable can be a * local static and not a global, allowing ODR to deduplicate it. */ - static inline Alloc*& get(bool create = true) + static SNMALLOC_FAST_PATH Alloc*& get(bool create = true) { static thread_local Alloc* per_thread; if (!per_thread && create) @@ -224,7 +224,7 @@ namespace snmalloc * Private accessor to the per thread allocator * Provides no checking for initialization */ - static ALWAYSINLINE Alloc*& inner_get() + static SNMALLOC_FAST_PATH Alloc*& inner_get() { static thread_local Alloc* per_thread; return per_thread; @@ -242,7 +242,7 @@ namespace snmalloc /** * Private initialiser for the per thread allocator */ - static NOINLINE Alloc*& inner_init() + static SNMALLOC_SLOW_PATH Alloc*& inner_init() { Alloc*& per_thread = inner_get(); @@ -280,11 +280,11 @@ namespace snmalloc * Public interface, returns the allocator for the current thread, * constructing it if necessary. */ - static ALWAYSINLINE Alloc*& get() + static SNMALLOC_FAST_PATH Alloc*& get() { Alloc*& per_thread = inner_get(); - if (per_thread != nullptr) + if (likely(per_thread != nullptr)) return per_thread; // Slow path that performs initialization diff --git a/src/override/malloc.cc b/src/override/malloc.cc index db1b1bb..42c1fd2 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -23,17 +23,11 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) { - // Include size 0 in the first sizeclass. - size = ((size - 1) >> (bits::BITS - 1)) + size; - return ThreadAlloc::get()->alloc(size); } SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free)(void* ptr) { - if (ptr == nullptr) - return; - ThreadAlloc::get()->dealloc(ptr); } @@ -46,8 +40,6 @@ extern "C" errno = ENOMEM; return nullptr; } - // Include size 0 in the first sizeclass. - sz = ((sz - 1) >> (bits::BITS - 1)) + sz; return ThreadAlloc::get()->alloc(sz); } @@ -137,7 +129,7 @@ extern "C" } size = bits::max(size, alignment); - uint8_t sc = size_to_sizeclass(size); + snmalloc::sizeclass_t sc = size_to_sizeclass(size); if (sc >= NUM_SIZECLASSES) { // large allocs are 16M aligned. diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index e399786..9f3725b 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -38,7 +38,14 @@ namespace snmalloc assert( bits::is_aligned_block(p, size) || (zero_mem == NoZero)); if constexpr (zero_mem == YesZero) + { zero(p, size); + } + else + { + UNUSED(size); + UNUSED(p); + } } /// OS specific function for zeroing memory diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 508f3c8..e692f3c 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -77,7 +77,7 @@ int main(int argc, char** argv) test_calloc(0, 0, SUCCESS, false); - for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (snmalloc::sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); @@ -103,7 +103,7 @@ int main(int argc, char** argv) for (size_t align = sizeof(size_t); align <= SUPERSLAB_SIZE; align <<= 1) { - for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (snmalloc::sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); test_posix_memalign(size, align, SUCCESS, false); diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index a98c201..6ca6c8d 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -2,7 +2,7 @@ #include NOINLINE -uint8_t size_to_sizeclass(size_t size) +snmalloc::sizeclass_t size_to_sizeclass(size_t size) { return snmalloc::size_to_sizeclass(size); } @@ -17,7 +17,7 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; - for (uint8_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++) + for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++) { // Separate printing for small and medium sizeclasses if (sz == snmalloc::NUM_SMALL_CLASSES)