Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

put likely/unlikely in scope (#420)

Browse Source 
* put likely/unlikely in scope

Signed-off-by: SchrodingerZhu <i@zhuyi.fan>

* make clang-format happy

Signed-off-by: SchrodingerZhu <i@zhuyi.fan>
This commit is contained in:
Schrodinger ZHU Yifan
2021-11-18 00:05:52 +08:00
committed by GitHub
parent cd0311b26f
commit faa80037bb
11 changed files with 36 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/backend/pagemap.h
View File

@@ -240,7 +240,7 @@ namespace snmalloc
{ {
if constexpr (potentially_out_of_range) if constexpr (potentially_out_of_range)
{ {
if (unlikely(body_opt == nullptr)) if (SNMALLOC_UNLIKELY(body_opt == nullptr))
return default_value; return default_value;
} }

10
src/ds/defines.h
View File

@@ -6,8 +6,8 @@
# define SNMALLOC_FAST_FAIL() __fastfail(28) # define SNMALLOC_FAST_FAIL() __fastfail(28)
# define ALWAYSINLINE __forceinline # define ALWAYSINLINE __forceinline
# define NOINLINE __declspec(noinline) # define NOINLINE __declspec(noinline)
# define likely(x) !!(x) # define SNMALLOC_LIKELY(x) !!(x)
# define unlikely(x) !!(x) # define SNMALLOC_UNLIKELY(x) !!(x)
# define SNMALLOC_SLOW_PATH NOINLINE # define SNMALLOC_SLOW_PATH NOINLINE
# define SNMALLOC_FAST_PATH ALWAYSINLINE # define SNMALLOC_FAST_PATH ALWAYSINLINE
/** /**
@@ -26,8 +26,8 @@
# define SNMALLOC_UNUSED_FUNCTION # define SNMALLOC_UNUSED_FUNCTION
#else #else
# define SNMALLOC_FAST_FAIL() __builtin_trap() # define SNMALLOC_FAST_FAIL() __builtin_trap()
# define likely(x) __builtin_expect(!!(x), 1) # define SNMALLOC_LIKELY(x) __builtin_expect(!!(x), 1)
# define unlikely(x) __builtin_expect(!!(x), 0) # define SNMALLOC_UNLIKELY(x) __builtin_expect(!!(x), 0)
# define ALWAYSINLINE __attribute__((always_inline)) # define ALWAYSINLINE __attribute__((always_inline))
# define NOINLINE __attribute__((noinline)) # define NOINLINE __attribute__((noinline))
# define SNMALLOC_SLOW_PATH NOINLINE # define SNMALLOC_SLOW_PATH NOINLINE
@@ -152,7 +152,7 @@ inline SNMALLOC_FAST_PATH void check_client_error(const char* const str)
inline SNMALLOC_FAST_PATH void inline SNMALLOC_FAST_PATH void
check_client_impl(bool test, const char* const str) check_client_impl(bool test, const char* const str)
{ {
if (unlikely(!test)) if (SNMALLOC_UNLIKELY(!test))
check_client_error(str); check_client_error(str);
} }
#ifdef SNMALLOC_CHECK_CLIENT #ifdef SNMALLOC_CHECK_CLIENT

2
src/ds/helpers.h
View File

@@ -32,7 +32,7 @@ namespace snmalloc
// If defined should be initially false; // If defined should be initially false;
SNMALLOC_ASSERT(first == nullptr || *first == false); SNMALLOC_ASSERT(first == nullptr || *first == false);
if (unlikely(!initialised.load(std::memory_order_acquire))) if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire)))
{ {
FlagLock lock(flag); FlagLock lock(flag);
if (!initialised) if (!initialised)

2
src/ds/spmcstack.h
View File

@@ -51,7 +51,7 @@ namespace snmalloc
if (stack.load(std::memory_order_relaxed) == nullptr) if (stack.load(std::memory_order_relaxed) == nullptr)
return nullptr; return nullptr;
T* old_head = stack.exchange(nullptr); T* old_head = stack.exchange(nullptr);
if (unlikely(old_head == nullptr)) if (SNMALLOC_UNLIKELY(old_head == nullptr))
return nullptr; return nullptr;
auto next = old_head->next.load(std::memory_order_relaxed); auto next = old_head->next.load(std::memory_order_relaxed);

15
src/mem/corealloc.h
View File

@@ -513,9 +513,10 @@ namespace snmalloc
// TODO this needs to not double revoke if using MTE // TODO this needs to not double revoke if using MTE
// TODO thread capabilities? // TODO thread capabilities?
if (likely(entry.get_remote() == public_state())) if (SNMALLOC_LIKELY(entry.get_remote() == public_state()))
{ {
if (likely(dealloc_local_object_fast(entry, p.as_void(), entropy))) if (SNMALLOC_LIKELY(
dealloc_local_object_fast(entry, p.as_void(), entropy)))
return; return;
dealloc_local_object_slow(entry); dealloc_local_object_slow(entry);
@@ -635,7 +636,7 @@ namespace snmalloc
handle_message_queue(Action action, Args... args) handle_message_queue(Action action, Args... args)
{ {
// Inline the empty check, but not necessarily the full queue handling. // Inline the empty check, but not necessarily the full queue handling.
if (likely(!has_messages())) if (SNMALLOC_LIKELY(!has_messages()))
{ {
return action(args...); return action(args...);
} }
@@ -648,7 +649,7 @@ namespace snmalloc
{ {
auto entry = SharedStateHandle::Pagemap::get_metaentry( auto entry = SharedStateHandle::Pagemap::get_metaentry(
backend_state_ptr(), snmalloc::address_cast(p)); backend_state_ptr(), snmalloc::address_cast(p));
if (likely(dealloc_local_object_fast(entry, p, entropy))) if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy)))
return; return;
dealloc_local_object_slow(entry); dealloc_local_object_slow(entry);
@@ -675,7 +676,7 @@ namespace snmalloc
// Update the head and the next pointer in the free list. // Update the head and the next pointer in the free list.
meta->free_queue.add(cp, key, entropy); meta->free_queue.add(cp, key, entropy);
return likely(!meta->return_object()); return SNMALLOC_LIKELY(!meta->return_object());
} }
template<ZeroMem zero_mem> template<ZeroMem zero_mem>
@@ -684,12 +685,12 @@ namespace snmalloc
{ {
// Look to see if we can grab a free list. // Look to see if we can grab a free list.
auto& sl = alloc_classes[sizeclass].available; auto& sl = alloc_classes[sizeclass].available;
if (likely(alloc_classes[sizeclass].length > 0)) if (SNMALLOC_LIKELY(alloc_classes[sizeclass].length > 0))
{ {
#ifdef SNMALLOC_CHECK_CLIENT #ifdef SNMALLOC_CHECK_CLIENT
// Occassionally don't use the last list. // Occassionally don't use the last list.
if ( if (
unlikely(alloc_classes[sizeclass].length == 1) && SNMALLOC_UNLIKELY(alloc_classes[sizeclass].length == 1) &&
(entropy.next_bit() == 0)) (entropy.next_bit() == 0))
{ {
return small_alloc_slow<zero_mem>(sizeclass, fast_free_list); return small_alloc_slow<zero_mem>(sizeclass, fast_free_list);

16
src/mem/localalloc.h
View File

@@ -97,7 +97,7 @@ namespace snmalloc
template<typename Action, typename... Args> template<typename Action, typename... Args>
SNMALLOC_FAST_PATH decltype(auto) check_init(Action action, Args... args) SNMALLOC_FAST_PATH decltype(auto) check_init(Action action, Args... args)
{ {
if (likely(core_alloc != nullptr)) if (SNMALLOC_LIKELY(core_alloc != nullptr))
{ {
return core_alloc->handle_message_queue(action, core_alloc, args...); return core_alloc->handle_message_queue(action, core_alloc, args...);
} }
@@ -219,7 +219,7 @@ namespace snmalloc
auto slowpath = [&]( auto slowpath = [&](
smallsizeclass_t sizeclass, smallsizeclass_t sizeclass,
freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA {
if (likely(core_alloc != nullptr)) if (SNMALLOC_LIKELY(core_alloc != nullptr))
{ {
return core_alloc->handle_message_queue( return core_alloc->handle_message_queue(
[]( [](
@@ -428,7 +428,7 @@ namespace snmalloc
#else #else
// Perform the - 1 on size, so that zero wraps around and ends up on // Perform the - 1 on size, so that zero wraps around and ends up on
// slow path. // slow path.
if (likely( if (SNMALLOC_LIKELY(
(size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1)))
{ {
// Small allocations are more likely. Improve // Small allocations are more likely. Improve
@@ -478,16 +478,17 @@ namespace snmalloc
const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry( const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(
core_alloc->backend_state_ptr(), address_cast(p_tame)); core_alloc->backend_state_ptr(), address_cast(p_tame));
if (likely(local_cache.remote_allocator == entry.get_remote())) if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote()))
{ {
if (likely(CoreAlloc::dealloc_local_object_fast( if (SNMALLOC_LIKELY(CoreAlloc::dealloc_local_object_fast(
entry, p_tame, local_cache.entropy))) entry, p_tame, local_cache.entropy)))
return; return;
core_alloc->dealloc_local_object_slow(entry); core_alloc->dealloc_local_object_slow(entry);
return; return;
} }
if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) if (SNMALLOC_LIKELY(
entry.get_remote() != SharedStateHandle::fake_large_remote))
{ {
// Check if we have space for the remote deallocation // Check if we have space for the remote deallocation
if (local_cache.remote_dealloc_cache.reserve_space(entry)) if (local_cache.remote_dealloc_cache.reserve_space(entry))
@@ -507,7 +508,8 @@ namespace snmalloc
// Large deallocation or null. // Large deallocation or null.
// also checks for managed by page map. // also checks for managed by page map.
if (likely((p_tame != nullptr) && !entry.get_sizeclass().is_default())) if (SNMALLOC_LIKELY(
(p_tame != nullptr) && !entry.get_sizeclass().is_default()))
{ {
size_t entry_sizeclass = entry.get_sizeclass().as_large(); size_t entry_sizeclass = entry.get_sizeclass().as_large();

2
src/mem/localcache.h
View File

@@ -112,7 +112,7 @@ namespace snmalloc
stats.alloc_request(size); stats.alloc_request(size);
stats.sizeclass_alloc(sizeclass); stats.sizeclass_alloc(sizeclass);
auto& fl = small_fast_free_lists[sizeclass]; auto& fl = small_fast_free_lists[sizeclass];
if (likely(!fl.empty())) if (SNMALLOC_LIKELY(!fl.empty()))
{ {
auto p = fl.take(key, domesticate); auto p = fl.take(key, domesticate);
return finish_alloc<zero_mem, SharedStateHandle>(p, sizeclass); return finish_alloc<zero_mem, SharedStateHandle>(p, sizeclass);

4
src/mem/remoteallocator.h
View File

@@ -147,11 +147,11 @@ namespace snmalloc
freelist::HeadPtr next = curr->atomic_read_next(key, domesticate_queue); freelist::HeadPtr next = curr->atomic_read_next(key, domesticate_queue);
// We have observed a non-linearisable effect of the queue. // We have observed a non-linearisable effect of the queue.
// Just go back to allocating normally. // Just go back to allocating normally.
if (unlikely(next == nullptr)) if (SNMALLOC_UNLIKELY(next == nullptr))
break; break;
// We want this element next, so start it loading. // We want this element next, so start it loading.
Aal::prefetch(next.unsafe_ptr()); Aal::prefetch(next.unsafe_ptr());
if (unlikely(!cb(curr))) if (SNMALLOC_UNLIKELY(!cb(curr)))
{ {
/* /*
* We've domesticate_queue-d next so that we can read through it, but * We've domesticate_queue-d next so that we can read through it, but

8
src/override/malloc.cc
View File

@@ -47,7 +47,7 @@ extern "C"
{ {
bool overflow = false; bool overflow = false;
size_t sz = bits::umul(size, nmemb, overflow); size_t sz = bits::umul(size, nmemb, overflow);
if (unlikely(overflow)) if (SNMALLOC_UNLIKELY(overflow))
{ {
return SNMALLOC_NAME_MANGLE(snmalloc_set_error)(); return SNMALLOC_NAME_MANGLE(snmalloc_set_error)();
} }
@@ -85,7 +85,7 @@ extern "C"
} }
void* p = a.alloc(size); void* p = a.alloc(size);
if (likely(p != nullptr)) if (SNMALLOC_LIKELY(p != nullptr))
{ {
sz = bits::min(size, sz); sz = bits::min(size, sz);
// Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy
@@ -94,7 +94,7 @@ extern "C"
memcpy(p, ptr, sz); memcpy(p, ptr, sz);
a.dealloc(ptr); a.dealloc(ptr);
} }
else if (likely(size == 0)) else if (SNMALLOC_LIKELY(size == 0))
{ {
a.dealloc(ptr); a.dealloc(ptr);
} }
@@ -150,7 +150,7 @@ extern "C"
} }
void* p = SNMALLOC_NAME_MANGLE(memalign)(alignment, size); void* p = SNMALLOC_NAME_MANGLE(memalign)(alignment, size);
if (unlikely(p == nullptr)) if (SNMALLOC_UNLIKELY(p == nullptr))
{ {
if (size != 0) if (size != 0)
return ENOMEM; return ENOMEM;

4
src/override/memcpy.cc
View File

@@ -136,7 +136,7 @@ namespace
auto& alloc = ThreadAlloc::get(); auto& alloc = ThreadAlloc::get();
void* p = const_cast<void*>(ptr); void* p = const_cast<void*>(ptr);
if (unlikely(alloc.remaining_bytes(ptr) < len)) if (SNMALLOC_UNLIKELY(alloc.remaining_bytes(ptr) < len))
{ {
if constexpr (FailFast) if constexpr (FailFast)
{ {
@@ -209,7 +209,7 @@ extern "C"
// 0 is a very common size for memcpy and we don't need to do external // 0 is a very common size for memcpy and we don't need to do external
// pointer checks if we hit it. It's also the fastest case, to encourage // pointer checks if we hit it. It's also the fastest case, to encourage
// the compiler to favour the other cases. // the compiler to favour the other cases.
if (unlikely(len == 0)) if (SNMALLOC_UNLIKELY(len == 0))
{ {
return dst; return dst;
} }

4
src/pal/pal_apple.h
View File

@@ -175,7 +175,7 @@ namespace snmalloc
anonymous_memory_fd, anonymous_memory_fd,
0); 0);
if (likely(r != MAP_FAILED)) if (SNMALLOC_LIKELY(r != MAP_FAILED))
{ {
return; return;
} }
@@ -238,7 +238,7 @@ namespace snmalloc
VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE,
VM_INHERIT_COPY); VM_INHERIT_COPY);
if (unlikely(kr != KERN_SUCCESS)) if (SNMALLOC_UNLIKELY(kr != KERN_SUCCESS))
{ {
return nullptr; return nullptr;
} }