put likely/unlikely in scope (#420)

* put likely/unlikely in scope

Signed-off-by: SchrodingerZhu <i@zhuyi.fan>

* make clang-format happy

Signed-off-by: SchrodingerZhu <i@zhuyi.fan>

src/mem/corealloc.h

@@ -513,9 +513,10 @@ namespace snmalloc
       // TODO this needs to not double revoke if using MTE
       // TODO thread capabilities?
 
-      if (likely(entry.get_remote() == public_state()))
+      if (SNMALLOC_LIKELY(entry.get_remote() == public_state()))
       {
-        if (likely(dealloc_local_object_fast(entry, p.as_void(), entropy)))
+        if (SNMALLOC_LIKELY(
+              dealloc_local_object_fast(entry, p.as_void(), entropy)))
           return;
 
         dealloc_local_object_slow(entry);
@@ -635,7 +636,7 @@ namespace snmalloc
     handle_message_queue(Action action, Args... args)
     {
       // Inline the empty check, but not necessarily the full queue handling.
-      if (likely(!has_messages()))
+      if (SNMALLOC_LIKELY(!has_messages()))
       {
         return action(args...);
       }
@@ -648,7 +649,7 @@ namespace snmalloc
     {
       auto entry = SharedStateHandle::Pagemap::get_metaentry(
         backend_state_ptr(), snmalloc::address_cast(p));
-      if (likely(dealloc_local_object_fast(entry, p, entropy)))
+      if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy)))
         return;
 
       dealloc_local_object_slow(entry);
@@ -675,7 +676,7 @@ namespace snmalloc
       // Update the head and the next pointer in the free list.
       meta->free_queue.add(cp, key, entropy);
 
-      return likely(!meta->return_object());
+      return SNMALLOC_LIKELY(!meta->return_object());
     }
 
     template<ZeroMem zero_mem>
@@ -684,12 +685,12 @@ namespace snmalloc
     {
       // Look to see if we can grab a free list.
       auto& sl = alloc_classes[sizeclass].available;
-      if (likely(alloc_classes[sizeclass].length > 0))
+      if (SNMALLOC_LIKELY(alloc_classes[sizeclass].length > 0))
       {
 #ifdef SNMALLOC_CHECK_CLIENT
         // Occassionally don't use the last list.
         if (
-          unlikely(alloc_classes[sizeclass].length == 1) &&
+          SNMALLOC_UNLIKELY(alloc_classes[sizeclass].length == 1) &&
           (entropy.next_bit() == 0))
         {
           return small_alloc_slow<zero_mem>(sizeclass, fast_free_list);

src/mem/localalloc.h

@@ -97,7 +97,7 @@ namespace snmalloc
     template<typename Action, typename... Args>
     SNMALLOC_FAST_PATH decltype(auto) check_init(Action action, Args... args)
     {
-      if (likely(core_alloc != nullptr))
+      if (SNMALLOC_LIKELY(core_alloc != nullptr))
       {
         return core_alloc->handle_message_queue(action, core_alloc, args...);
       }
@@ -219,7 +219,7 @@ namespace snmalloc
       auto slowpath = [&](
                         smallsizeclass_t sizeclass,
                         freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA {
-        if (likely(core_alloc != nullptr))
+        if (SNMALLOC_LIKELY(core_alloc != nullptr))
         {
           return core_alloc->handle_message_queue(
             [](
@@ -428,7 +428,7 @@ namespace snmalloc
 #else
       // Perform the - 1 on size, so that zero wraps around and ends up on
       // slow path.
-      if (likely(
+      if (SNMALLOC_LIKELY(
             (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1)))
       {
         // Small allocations are more likely. Improve
@@ -478,16 +478,17 @@ namespace snmalloc
 
       const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(
         core_alloc->backend_state_ptr(), address_cast(p_tame));
-      if (likely(local_cache.remote_allocator == entry.get_remote()))
+      if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote()))
       {
-        if (likely(CoreAlloc::dealloc_local_object_fast(
+        if (SNMALLOC_LIKELY(CoreAlloc::dealloc_local_object_fast(
               entry, p_tame, local_cache.entropy)))
           return;
         core_alloc->dealloc_local_object_slow(entry);
         return;
       }
 
-      if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote))
+      if (SNMALLOC_LIKELY(
+            entry.get_remote() != SharedStateHandle::fake_large_remote))
       {
         // Check if we have space for the remote deallocation
         if (local_cache.remote_dealloc_cache.reserve_space(entry))
@@ -507,7 +508,8 @@ namespace snmalloc
 
       // Large deallocation or null.
       // also checks for managed by page map.
-      if (likely((p_tame != nullptr) && !entry.get_sizeclass().is_default()))
+      if (SNMALLOC_LIKELY(
+            (p_tame != nullptr) && !entry.get_sizeclass().is_default()))
       {
         size_t entry_sizeclass = entry.get_sizeclass().as_large();
 

src/mem/localcache.h

@@ -112,7 +112,7 @@ namespace snmalloc
       stats.alloc_request(size);
       stats.sizeclass_alloc(sizeclass);
       auto& fl = small_fast_free_lists[sizeclass];
-      if (likely(!fl.empty()))
+      if (SNMALLOC_LIKELY(!fl.empty()))
       {
         auto p = fl.take(key, domesticate);
         return finish_alloc<zero_mem, SharedStateHandle>(p, sizeclass);

src/mem/remoteallocator.h

@@ -147,11 +147,11 @@ namespace snmalloc
         freelist::HeadPtr next = curr->atomic_read_next(key, domesticate_queue);
         // We have observed a non-linearisable effect of the queue.
         // Just go back to allocating normally.
-        if (unlikely(next == nullptr))
+        if (SNMALLOC_UNLIKELY(next == nullptr))
           break;
         // We want this element next, so start it loading.
         Aal::prefetch(next.unsafe_ptr());
-        if (unlikely(!cb(curr)))
+        if (SNMALLOC_UNLIKELY(!cb(curr)))
         {
           /*
            * We've domesticate_queue-d next so that we can read through it, but