diff --git a/src/ds/bits.h b/src/ds/bits.h index 9ba07d3..cf6f79c 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -11,6 +11,13 @@ # define HEADER_GLOBAL __declspec(selectany) # define likely(x) !!(x) # define unlikely(x) !!(x) +# define SLOW_PATH NOINLINE +# define FAST_PATH ALWAYSINLINE +# ifdef NDEBUG +# define ASSUME(x) +# else +# define ASSUME(x) assert(x); +# endif #else # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) @@ -18,6 +25,13 @@ # include # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) +# define SLOW_PATH NOINLINE +# define FAST_PATH ALWAYSINLINE +# ifdef NDEBUG +# define ASSUME(x) if (!(x)) __builtin_unreachable(); +# else +# define ASSUME(x) assert(x); +# endif # ifdef __clang__ # define HEADER_GLOBAL __attribute__((selectany)) # else diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 8d4f181..7e8d402 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -339,7 +339,7 @@ namespace snmalloc } template - NOINLINE ALLOCATOR void* alloc_not_small(size_t size) + SLOW_PATH ALLOCATOR void* alloc_not_small(size_t size) { handle_message_queue(); @@ -440,7 +440,7 @@ namespace snmalloc #endif } - ALWAYSINLINE void dealloc(void* p) + FAST_PATH void dealloc(void* p) { #ifdef USE_MALLOC return free(p); @@ -474,7 +474,7 @@ namespace snmalloc dealloc_not_small(p, size); } - NOINLINE void dealloc_not_small(void* p, uint8_t size) + SLOW_PATH void dealloc_not_small(void* p, uint8_t size) { if (size == PMMediumslab) { @@ -882,7 +882,7 @@ namespace snmalloc } } - NOINLINE void handle_message_queue_inner() + SLOW_PATH void handle_message_queue_inner() { for (size_t i = 0; i < REMOTE_BATCH; i++) { @@ -902,7 +902,7 @@ namespace snmalloc remote.post(id()); } - ALWAYSINLINE void handle_message_queue() + FAST_PATH void handle_message_queue() { // Inline the empty check, but not necessarily the full queue handling. if (likely(message_queue().is_empty())) @@ -1017,6 +1017,7 @@ namespace snmalloc zero_mem == YesZero ? "zeromem" : "nozeromem", allow_reserve == NoReserve ? "noreserve" : "reserve")); + ASSUME(size <= SLAB_SIZE); sizeclass_t sizeclass = size_to_sizeclass(size); stats().sizeclass_alloc(sizeclass); @@ -1041,7 +1042,7 @@ namespace snmalloc } template - NOINLINE + SLOW_PATH void* small_alloc_slow(size_t size) { handle_message_queue(); @@ -1069,7 +1070,7 @@ namespace snmalloc return slab->alloc(sl, ffl, rsize, large_allocator.memory_provider); } - ALWAYSINLINE void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) + FAST_PATH void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) { #ifdef CHECK_CLIENT Slab* slab = Slab::get(p); @@ -1083,7 +1084,7 @@ namespace snmalloc small_dealloc_offseted(super, offseted, sizeclass); } - ALWAYSINLINE void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) + FAST_PATH void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) { MEASURE_TIME(small_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); @@ -1095,7 +1096,7 @@ namespace snmalloc small_dealloc_offseted_slow(super, p, sizeclass); } - NOINLINE void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass) + SLOW_PATH void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass) { bool was_full = super->is_full(); SlabList* sl = &small_classes[sizeclass]; diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 929cbd7..4b427c4 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -189,7 +189,7 @@ namespace snmalloc * (over the lifetime of this process). If the underlying system does not * support low memory notifications, this will return 0. */ - ALWAYSINLINE + FAST_PATH uint64_t low_memory_epoch() { if constexpr (pal_supports()) @@ -235,7 +235,7 @@ namespace snmalloc } } - ALWAYSINLINE void lazy_decommit_if_needed() + FAST_PATH void lazy_decommit_if_needed() { #ifdef TEST_LAZY_DECOMMIT static_assert( diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index e6dce02..55176cb 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -105,7 +105,7 @@ namespace snmalloc std::atomic top[TOPLEVEL_ENTRIES]; // = {nullptr}; template - inline PagemapEntry* get_node(std::atomic* e, bool& result) + FAST_PATH PagemapEntry* get_node(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -128,7 +128,7 @@ namespace snmalloc } } - NOINLINE PagemapEntry* get_node_slow(std::atomic* e, bool& result) + SLOW_PATH PagemapEntry* get_node_slow(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -161,7 +161,7 @@ namespace snmalloc } template - inline std::pair get_leaf_index(uintptr_t addr, bool& result) + FAST_PATH std::pair get_leaf_index(uintptr_t addr, bool& result) { #ifdef FreeBSD_KERNEL // Zero the top 16 bits - kernel addresses all have them set, but the @@ -204,7 +204,7 @@ namespace snmalloc } template - inline std::atomic* get_addr(uintptr_t p, bool& success) + FAST_PATH std::atomic* get_addr(uintptr_t p, bool& success) { auto leaf_ix = get_leaf_index(p, success); return &(leaf_ix.first->values[leaf_ix.second]); diff --git a/src/mem/slab.h b/src/mem/slab.h index 141b6e8..3b70680 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -144,7 +144,7 @@ namespace snmalloc // Returns true, if it deallocation can proceed without changing any status bits. // Note that this does remove the use from the meta slab, so it doesn't need doing // on the slow path. - ALWAYSINLINE bool dealloc_fast(Superslab* super, void* p) + FAST_PATH bool dealloc_fast(Superslab* super, void* p) { Metaslab& meta = super->get_meta(this); #ifdef CHECK_CLIENT @@ -179,7 +179,7 @@ namespace snmalloc // Returns a complex return code for managing the superslab meta data. // i.e. This deallocation could make an entire superslab free. template - NOINLINE typename Superslab::Action dealloc_slow( + SLOW_PATH typename Superslab::Action dealloc_slow( SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider) { Metaslab& meta = super->get_meta(this); diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 98f8096..cc17005 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -1,4 +1,4 @@ -#pragma once + #pragma once #include "../ds/helpers.h" #include "globalalloc.h" @@ -28,7 +28,7 @@ namespace snmalloc class ThreadAllocUntypedWrapper { public: - static inline Alloc*& get() + static FAST_PATH Alloc*& get() { return (Alloc*&)ThreadAllocUntyped::get(); } @@ -76,7 +76,7 @@ namespace snmalloc * The non-create case exists so that the `per_thread` variable can be a * local static and not a global, allowing ODR to deduplicate it. */ - static inline Alloc*& get(bool create = true) + static FAST_PATH Alloc*& get(bool create = true) { static thread_local Alloc* per_thread; if (!per_thread && create) @@ -224,7 +224,7 @@ namespace snmalloc * Private accessor to the per thread allocator * Provides no checking for initialization */ - static ALWAYSINLINE Alloc*& inner_get() + static FAST_PATH Alloc*& inner_get() { static thread_local Alloc* per_thread; return per_thread; @@ -242,7 +242,7 @@ namespace snmalloc /** * Private initialiser for the per thread allocator */ - static NOINLINE Alloc*& inner_init() + static SLOW_PATH Alloc*& inner_init() { Alloc*& per_thread = inner_get(); @@ -262,15 +262,15 @@ namespace snmalloc // Associate the new allocator with the destructor. tls_set_value(key, &per_thread); -# ifdef USE_SNMALLOC_STATS + # ifdef USE_SNMALLOC_STATS // Allocator is up and running now, safe to call atexit. if (first) { atexit(print_stats); } -# else + # else UNUSED(first); -# endif + # endif } return per_thread; } @@ -280,7 +280,7 @@ namespace snmalloc * Public interface, returns the allocator for the current thread, * constructing it if necessary. */ - static ALWAYSINLINE Alloc*& get() + static FAST_PATH Alloc*& get() { Alloc*& per_thread = inner_get();