diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index e80d945..48699cd 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -240,7 +240,7 @@ namespace snmalloc { if constexpr (potentially_out_of_range) { - if (unlikely(body_opt == nullptr)) + if (SNMALLOC_UNLIKELY(body_opt == nullptr)) return default_value; } diff --git a/src/ds/defines.h b/src/ds/defines.h index 3019493..946f5b2 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -6,8 +6,8 @@ # define SNMALLOC_FAST_FAIL() __fastfail(28) # define ALWAYSINLINE __forceinline # define NOINLINE __declspec(noinline) -# define likely(x) !!(x) -# define unlikely(x) !!(x) +# define SNMALLOC_LIKELY(x) !!(x) +# define SNMALLOC_UNLIKELY(x) !!(x) # define SNMALLOC_SLOW_PATH NOINLINE # define SNMALLOC_FAST_PATH ALWAYSINLINE /** @@ -26,8 +26,8 @@ # define SNMALLOC_UNUSED_FUNCTION #else # define SNMALLOC_FAST_FAIL() __builtin_trap() -# define likely(x) __builtin_expect(!!(x), 1) -# define unlikely(x) __builtin_expect(!!(x), 0) +# define SNMALLOC_LIKELY(x) __builtin_expect(!!(x), 1) +# define SNMALLOC_UNLIKELY(x) __builtin_expect(!!(x), 0) # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) # define SNMALLOC_SLOW_PATH NOINLINE @@ -152,7 +152,7 @@ inline SNMALLOC_FAST_PATH void check_client_error(const char* const str) inline SNMALLOC_FAST_PATH void check_client_impl(bool test, const char* const str) { - if (unlikely(!test)) + if (SNMALLOC_UNLIKELY(!test)) check_client_error(str); } #ifdef SNMALLOC_CHECK_CLIENT diff --git a/src/ds/helpers.h b/src/ds/helpers.h index b8bc15b..dba6106 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -32,7 +32,7 @@ namespace snmalloc // If defined should be initially false; SNMALLOC_ASSERT(first == nullptr || *first == false); - if (unlikely(!initialised.load(std::memory_order_acquire))) + if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire))) { FlagLock lock(flag); if (!initialised) diff --git a/src/ds/spmcstack.h b/src/ds/spmcstack.h index 5651a09..7c6ea70 100644 --- a/src/ds/spmcstack.h +++ b/src/ds/spmcstack.h @@ -51,7 +51,7 @@ namespace snmalloc if (stack.load(std::memory_order_relaxed) == nullptr) return nullptr; T* old_head = stack.exchange(nullptr); - if (unlikely(old_head == nullptr)) + if (SNMALLOC_UNLIKELY(old_head == nullptr)) return nullptr; auto next = old_head->next.load(std::memory_order_relaxed); diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 5811079..ca9937e 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -513,9 +513,10 @@ namespace snmalloc // TODO this needs to not double revoke if using MTE // TODO thread capabilities? - if (likely(entry.get_remote() == public_state())) + if (SNMALLOC_LIKELY(entry.get_remote() == public_state())) { - if (likely(dealloc_local_object_fast(entry, p.as_void(), entropy))) + if (SNMALLOC_LIKELY( + dealloc_local_object_fast(entry, p.as_void(), entropy))) return; dealloc_local_object_slow(entry); @@ -635,7 +636,7 @@ namespace snmalloc handle_message_queue(Action action, Args... args) { // Inline the empty check, but not necessarily the full queue handling. - if (likely(!has_messages())) + if (SNMALLOC_LIKELY(!has_messages())) { return action(args...); } @@ -648,7 +649,7 @@ namespace snmalloc { auto entry = SharedStateHandle::Pagemap::get_metaentry( backend_state_ptr(), snmalloc::address_cast(p)); - if (likely(dealloc_local_object_fast(entry, p, entropy))) + if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy))) return; dealloc_local_object_slow(entry); @@ -675,7 +676,7 @@ namespace snmalloc // Update the head and the next pointer in the free list. meta->free_queue.add(cp, key, entropy); - return likely(!meta->return_object()); + return SNMALLOC_LIKELY(!meta->return_object()); } template @@ -684,12 +685,12 @@ namespace snmalloc { // Look to see if we can grab a free list. auto& sl = alloc_classes[sizeclass].available; - if (likely(alloc_classes[sizeclass].length > 0)) + if (SNMALLOC_LIKELY(alloc_classes[sizeclass].length > 0)) { #ifdef SNMALLOC_CHECK_CLIENT // Occassionally don't use the last list. if ( - unlikely(alloc_classes[sizeclass].length == 1) && + SNMALLOC_UNLIKELY(alloc_classes[sizeclass].length == 1) && (entropy.next_bit() == 0)) { return small_alloc_slow(sizeclass, fast_free_list); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index f994a4b..21886ba 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -97,7 +97,7 @@ namespace snmalloc template SNMALLOC_FAST_PATH decltype(auto) check_init(Action action, Args... args) { - if (likely(core_alloc != nullptr)) + if (SNMALLOC_LIKELY(core_alloc != nullptr)) { return core_alloc->handle_message_queue(action, core_alloc, args...); } @@ -219,7 +219,7 @@ namespace snmalloc auto slowpath = [&]( smallsizeclass_t sizeclass, freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { - if (likely(core_alloc != nullptr)) + if (SNMALLOC_LIKELY(core_alloc != nullptr)) { return core_alloc->handle_message_queue( []( @@ -428,7 +428,7 @@ namespace snmalloc #else // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. - if (likely( + if (SNMALLOC_LIKELY( (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) { // Small allocations are more likely. Improve @@ -478,16 +478,17 @@ namespace snmalloc const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p_tame)); - if (likely(local_cache.remote_allocator == entry.get_remote())) + if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote())) { - if (likely(CoreAlloc::dealloc_local_object_fast( + if (SNMALLOC_LIKELY(CoreAlloc::dealloc_local_object_fast( entry, p_tame, local_cache.entropy))) return; core_alloc->dealloc_local_object_slow(entry); return; } - if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) + if (SNMALLOC_LIKELY( + entry.get_remote() != SharedStateHandle::fake_large_remote)) { // Check if we have space for the remote deallocation if (local_cache.remote_dealloc_cache.reserve_space(entry)) @@ -507,7 +508,8 @@ namespace snmalloc // Large deallocation or null. // also checks for managed by page map. - if (likely((p_tame != nullptr) && !entry.get_sizeclass().is_default())) + if (SNMALLOC_LIKELY( + (p_tame != nullptr) && !entry.get_sizeclass().is_default())) { size_t entry_sizeclass = entry.get_sizeclass().as_large(); diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 6e98bb8..caa19c6 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -112,7 +112,7 @@ namespace snmalloc stats.alloc_request(size); stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; - if (likely(!fl.empty())) + if (SNMALLOC_LIKELY(!fl.empty())) { auto p = fl.take(key, domesticate); return finish_alloc(p, sizeclass); diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 5520593..6e33b58 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -147,11 +147,11 @@ namespace snmalloc freelist::HeadPtr next = curr->atomic_read_next(key, domesticate_queue); // We have observed a non-linearisable effect of the queue. // Just go back to allocating normally. - if (unlikely(next == nullptr)) + if (SNMALLOC_UNLIKELY(next == nullptr)) break; // We want this element next, so start it loading. Aal::prefetch(next.unsafe_ptr()); - if (unlikely(!cb(curr))) + if (SNMALLOC_UNLIKELY(!cb(curr))) { /* * We've domesticate_queue-d next so that we can read through it, but diff --git a/src/override/malloc.cc b/src/override/malloc.cc index 6db61ae..658e2ba 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -47,7 +47,7 @@ extern "C" { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); - if (unlikely(overflow)) + if (SNMALLOC_UNLIKELY(overflow)) { return SNMALLOC_NAME_MANGLE(snmalloc_set_error)(); } @@ -85,7 +85,7 @@ extern "C" } void* p = a.alloc(size); - if (likely(p != nullptr)) + if (SNMALLOC_LIKELY(p != nullptr)) { sz = bits::min(size, sz); // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy @@ -94,7 +94,7 @@ extern "C" memcpy(p, ptr, sz); a.dealloc(ptr); } - else if (likely(size == 0)) + else if (SNMALLOC_LIKELY(size == 0)) { a.dealloc(ptr); } @@ -150,7 +150,7 @@ extern "C" } void* p = SNMALLOC_NAME_MANGLE(memalign)(alignment, size); - if (unlikely(p == nullptr)) + if (SNMALLOC_UNLIKELY(p == nullptr)) { if (size != 0) return ENOMEM; diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index c193dc2..6d1ebce 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -136,7 +136,7 @@ namespace auto& alloc = ThreadAlloc::get(); void* p = const_cast(ptr); - if (unlikely(alloc.remaining_bytes(ptr) < len)) + if (SNMALLOC_UNLIKELY(alloc.remaining_bytes(ptr) < len)) { if constexpr (FailFast) { @@ -209,7 +209,7 @@ extern "C" // 0 is a very common size for memcpy and we don't need to do external // pointer checks if we hit it. It's also the fastest case, to encourage // the compiler to favour the other cases. - if (unlikely(len == 0)) + if (SNMALLOC_UNLIKELY(len == 0)) { return dst; } diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index 1115e8c..edf511b 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -175,7 +175,7 @@ namespace snmalloc anonymous_memory_fd, 0); - if (likely(r != MAP_FAILED)) + if (SNMALLOC_LIKELY(r != MAP_FAILED)) { return; } @@ -238,7 +238,7 @@ namespace snmalloc VM_PROT_READ | VM_PROT_WRITE, VM_INHERIT_COPY); - if (unlikely(kr != KERN_SUCCESS)) + if (SNMALLOC_UNLIKELY(kr != KERN_SUCCESS)) { return nullptr; }