Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

put likely/unlikely in scope (#420)

Browse Source 
* put likely/unlikely in scope

Signed-off-by: SchrodingerZhu <i@zhuyi.fan>

* make clang-format happy

Signed-off-by: SchrodingerZhu <i@zhuyi.fan>
This commit is contained in:
Schrodinger ZHU Yifan
2021-11-18 00:05:52 +08:00
committed by GitHub
parent cd0311b26f
commit faa80037bb
11 changed files with 36 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/backend/pagemap.h
View File

@@ -240,7 +240,7 @@ namespace snmalloc
{
if constexpr (potentially_out_of_range)
{
if (unlikely(body_opt == nullptr))
if (SNMALLOC_UNLIKELY(body_opt == nullptr))
return default_value;
}

10
src/ds/defines.h
View File

@@ -6,8 +6,8 @@
# define SNMALLOC_FAST_FAIL() __fastfail(28)
# define ALWAYSINLINE __forceinline
# define NOINLINE __declspec(noinline)
# define likely(x) !!(x)
# define unlikely(x) !!(x)
# define SNMALLOC_LIKELY(x) !!(x)
# define SNMALLOC_UNLIKELY(x) !!(x)
# define SNMALLOC_SLOW_PATH NOINLINE
# define SNMALLOC_FAST_PATH ALWAYSINLINE
/**
@@ -26,8 +26,8 @@
# define SNMALLOC_UNUSED_FUNCTION
#else
# define SNMALLOC_FAST_FAIL() __builtin_trap()
# define likely(x) __builtin_expect(!!(x), 1)
# define unlikely(x) __builtin_expect(!!(x), 0)
# define SNMALLOC_LIKELY(x) __builtin_expect(!!(x), 1)
# define SNMALLOC_UNLIKELY(x) __builtin_expect(!!(x), 0)
# define ALWAYSINLINE __attribute__((always_inline))
# define NOINLINE __attribute__((noinline))
# define SNMALLOC_SLOW_PATH NOINLINE
@@ -152,7 +152,7 @@ inline SNMALLOC_FAST_PATH void check_client_error(const char* const str)
inline SNMALLOC_FAST_PATH void
check_client_impl(bool test, const char* const str)
{
if (unlikely(!test))
if (SNMALLOC_UNLIKELY(!test))
check_client_error(str);
}
#ifdef SNMALLOC_CHECK_CLIENT

2
src/ds/helpers.h
View File

@@ -32,7 +32,7 @@ namespace snmalloc
// If defined should be initially false;
SNMALLOC_ASSERT(first == nullptr || *first == false);
if (unlikely(!initialised.load(std::memory_order_acquire)))
if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire)))
{
FlagLock lock(flag);
if (!initialised)

2
src/ds/spmcstack.h
View File

@@ -51,7 +51,7 @@ namespace snmalloc
if (stack.load(std::memory_order_relaxed) == nullptr)
return nullptr;
T* old_head = stack.exchange(nullptr);
if (unlikely(old_head == nullptr))
if (SNMALLOC_UNLIKELY(old_head == nullptr))
return nullptr;
auto next = old_head->next.load(std::memory_order_relaxed);

15
src/mem/corealloc.h
View File

@@ -513,9 +513,10 @@ namespace snmalloc
// TODO this needs to not double revoke if using MTE
// TODO thread capabilities?
if (likely(entry.get_remote() == public_state()))
if (SNMALLOC_LIKELY(entry.get_remote() == public_state()))
{
if (likely(dealloc_local_object_fast(entry, p.as_void(), entropy)))
if (SNMALLOC_LIKELY(
dealloc_local_object_fast(entry, p.as_void(), entropy)))
return;
dealloc_local_object_slow(entry);
@@ -635,7 +636,7 @@ namespace snmalloc
handle_message_queue(Action action, Args... args)
{
// Inline the empty check, but not necessarily the full queue handling.
if (likely(!has_messages()))
if (SNMALLOC_LIKELY(!has_messages()))
{
return action(args...);
}
@@ -648,7 +649,7 @@ namespace snmalloc
{
auto entry = SharedStateHandle::Pagemap::get_metaentry(
backend_state_ptr(), snmalloc::address_cast(p));
if (likely(dealloc_local_object_fast(entry, p, entropy)))
if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy)))
return;
dealloc_local_object_slow(entry);
@@ -675,7 +676,7 @@ namespace snmalloc
// Update the head and the next pointer in the free list.
meta->free_queue.add(cp, key, entropy);
return likely(!meta->return_object());
return SNMALLOC_LIKELY(!meta->return_object());
}
template<ZeroMem zero_mem>
@@ -684,12 +685,12 @@ namespace snmalloc
{
// Look to see if we can grab a free list.
auto& sl = alloc_classes[sizeclass].available;
if (likely(alloc_classes[sizeclass].length > 0))
if (SNMALLOC_LIKELY(alloc_classes[sizeclass].length > 0))
{
#ifdef SNMALLOC_CHECK_CLIENT
// Occassionally don't use the last list.
if (
unlikely(alloc_classes[sizeclass].length == 1) &&
SNMALLOC_UNLIKELY(alloc_classes[sizeclass].length == 1) &&
(entropy.next_bit() == 0))
{
return small_alloc_slow<zero_mem>(sizeclass, fast_free_list);

16
src/mem/localalloc.h
View File

@@ -97,7 +97,7 @@ namespace snmalloc
template<typename Action, typename... Args>
SNMALLOC_FAST_PATH decltype(auto) check_init(Action action, Args... args)
{
if (likely(core_alloc != nullptr))
if (SNMALLOC_LIKELY(core_alloc != nullptr))
{
return core_alloc->handle_message_queue(action, core_alloc, args...);
}
@@ -219,7 +219,7 @@ namespace snmalloc
auto slowpath = [&](
smallsizeclass_t sizeclass,
freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA {
if (likely(core_alloc != nullptr))
if (SNMALLOC_LIKELY(core_alloc != nullptr))
{
return core_alloc->handle_message_queue(
[](
@@ -428,7 +428,7 @@ namespace snmalloc
#else
// Perform the - 1 on size, so that zero wraps around and ends up on
// slow path.
if (likely(
if (SNMALLOC_LIKELY(
(size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1)))
{
// Small allocations are more likely. Improve
@@ -478,16 +478,17 @@ namespace snmalloc
const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(
core_alloc->backend_state_ptr(), address_cast(p_tame));
if (likely(local_cache.remote_allocator == entry.get_remote()))
if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote()))
{
if (likely(CoreAlloc::dealloc_local_object_fast(
if (SNMALLOC_LIKELY(CoreAlloc::dealloc_local_object_fast(
entry, p_tame, local_cache.entropy)))
return;
core_alloc->dealloc_local_object_slow(entry);
return;
}
if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote))
if (SNMALLOC_LIKELY(
entry.get_remote() != SharedStateHandle::fake_large_remote))
{
// Check if we have space for the remote deallocation
if (local_cache.remote_dealloc_cache.reserve_space(entry))
@@ -507,7 +508,8 @@ namespace snmalloc
// Large deallocation or null.
// also checks for managed by page map.
if (likely((p_tame != nullptr) && !entry.get_sizeclass().is_default()))
if (SNMALLOC_LIKELY(
(p_tame != nullptr) && !entry.get_sizeclass().is_default()))
{
size_t entry_sizeclass = entry.get_sizeclass().as_large();

2
src/mem/localcache.h
View File

@@ -112,7 +112,7 @@ namespace snmalloc
stats.alloc_request(size);
stats.sizeclass_alloc(sizeclass);
auto& fl = small_fast_free_lists[sizeclass];
if (likely(!fl.empty()))
if (SNMALLOC_LIKELY(!fl.empty()))
{
auto p = fl.take(key, domesticate);
return finish_alloc<zero_mem, SharedStateHandle>(p, sizeclass);

4
src/mem/remoteallocator.h
View File

@@ -147,11 +147,11 @@ namespace snmalloc
freelist::HeadPtr next = curr->atomic_read_next(key, domesticate_queue);
// We have observed a non-linearisable effect of the queue.
// Just go back to allocating normally.
if (unlikely(next == nullptr))
if (SNMALLOC_UNLIKELY(next == nullptr))
break;
// We want this element next, so start it loading.
Aal::prefetch(next.unsafe_ptr());
if (unlikely(!cb(curr)))
if (SNMALLOC_UNLIKELY(!cb(curr)))
{
/*
* We've domesticate_queue-d next so that we can read through it, but

8
src/override/malloc.cc
View File

@@ -47,7 +47,7 @@ extern "C"
{
bool overflow = false;
size_t sz = bits::umul(size, nmemb, overflow);
if (unlikely(overflow))
if (SNMALLOC_UNLIKELY(overflow))
{
return SNMALLOC_NAME_MANGLE(snmalloc_set_error)();
}
@@ -85,7 +85,7 @@ extern "C"
}
void* p = a.alloc(size);
if (likely(p != nullptr))
if (SNMALLOC_LIKELY(p != nullptr))
{
sz = bits::min(size, sz);
// Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy
@@ -94,7 +94,7 @@ extern "C"
memcpy(p, ptr, sz);
a.dealloc(ptr);
}
else if (likely(size == 0))
else if (SNMALLOC_LIKELY(size == 0))
{
a.dealloc(ptr);
}
@@ -150,7 +150,7 @@ extern "C"
}
void* p = SNMALLOC_NAME_MANGLE(memalign)(alignment, size);
if (unlikely(p == nullptr))
if (SNMALLOC_UNLIKELY(p == nullptr))
{
if (size != 0)
return ENOMEM;

4
src/override/memcpy.cc
View File

@@ -136,7 +136,7 @@ namespace
auto& alloc = ThreadAlloc::get();
void* p = const_cast<void*>(ptr);
if (unlikely(alloc.remaining_bytes(ptr) < len))
if (SNMALLOC_UNLIKELY(alloc.remaining_bytes(ptr) < len))
{
if constexpr (FailFast)
{
@@ -209,7 +209,7 @@ extern "C"
// 0 is a very common size for memcpy and we don't need to do external
// pointer checks if we hit it. It's also the fastest case, to encourage
// the compiler to favour the other cases.
if (unlikely(len == 0))
if (SNMALLOC_UNLIKELY(len == 0))
{
return dst;
}

4
src/pal/pal_apple.h
View File

@@ -175,7 +175,7 @@ namespace snmalloc
anonymous_memory_fd,
0);
if (likely(r != MAP_FAILED))
if (SNMALLOC_LIKELY(r != MAP_FAILED))
{
return;
}
@@ -238,7 +238,7 @@ namespace snmalloc
VM_PROT_READ | VM_PROT_WRITE,
VM_INHERIT_COPY);
if (unlikely(kr != KERN_SUCCESS))
if (SNMALLOC_UNLIKELY(kr != KERN_SUCCESS))
{
return nullptr;
}