Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add macro for ASSUME and FAST_PATH/SLOW_PATH

Browse Source 
Fixes GCC warning that was incorrect using an ASSUME.

Made fast path and slow path Macros so we can add additional attributes.
This commit is contained in:
Matthew Parkinson
2019-07-01 11:49:35 +01:00
parent c2780f99ed
commit 3c7d122dea
6 changed files with 41 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/ds/bits.h
View File

@@ -11,6 +11,13 @@
# define HEADER_GLOBAL __declspec(selectany)
# define likely(x) !!(x)
# define unlikely(x) !!(x)
# define SLOW_PATH NOINLINE
# define FAST_PATH ALWAYSINLINE
# ifdef NDEBUG
# define ASSUME(x)
# else
# define ASSUME(x) assert(x);
# endif
#else
# define likely(x) __builtin_expect(!!(x), 1)
# define unlikely(x) __builtin_expect(!!(x), 0)
@@ -18,6 +25,13 @@
# include <emmintrin.h>
# define ALWAYSINLINE __attribute__((always_inline))
# define NOINLINE __attribute__((noinline))
# define SLOW_PATH NOINLINE
# define FAST_PATH ALWAYSINLINE
# ifdef NDEBUG
# define ASSUME(x) if (!(x)) __builtin_unreachable();
# else
# define ASSUME(x) assert(x);
# endif
# ifdef __clang__
# define HEADER_GLOBAL __attribute__((selectany))
# else

19
src/mem/alloc.h
View File

@@ -339,7 +339,7 @@ namespace snmalloc
}
template<ZeroMem zero_mem = NoZero, AllowReserve allow_reserve = YesReserve>
NOINLINE ALLOCATOR void* alloc_not_small(size_t size)
SLOW_PATH ALLOCATOR void* alloc_not_small(size_t size)
{
handle_message_queue();
@@ -440,7 +440,7 @@ namespace snmalloc
#endif
}
ALWAYSINLINE void dealloc(void* p)
FAST_PATH void dealloc(void* p)
{
#ifdef USE_MALLOC
return free(p);
@@ -474,7 +474,7 @@ namespace snmalloc
dealloc_not_small(p, size);
}
NOINLINE void dealloc_not_small(void* p, uint8_t size)
SLOW_PATH void dealloc_not_small(void* p, uint8_t size)
{
if (size == PMMediumslab)
{
@@ -882,7 +882,7 @@ namespace snmalloc
}
}
NOINLINE void handle_message_queue_inner()
SLOW_PATH void handle_message_queue_inner()
{
for (size_t i = 0; i < REMOTE_BATCH; i++)
{
@@ -902,7 +902,7 @@ namespace snmalloc
remote.post(id());
}
ALWAYSINLINE void handle_message_queue()
FAST_PATH void handle_message_queue()
{
// Inline the empty check, but not necessarily the full queue handling.
if (likely(message_queue().is_empty()))
@@ -1017,6 +1017,7 @@ namespace snmalloc
zero_mem == YesZero ? "zeromem" : "nozeromem",
allow_reserve == NoReserve ? "noreserve" : "reserve"));
ASSUME(size <= SLAB_SIZE);
sizeclass_t sizeclass = size_to_sizeclass(size);
stats().sizeclass_alloc(sizeclass);
@@ -1041,7 +1042,7 @@ namespace snmalloc
}
template<ZeroMem zero_mem, AllowReserve allow_reserve>
NOINLINE
SLOW_PATH
void* small_alloc_slow(size_t size)
{
handle_message_queue();
@@ -1069,7 +1070,7 @@ namespace snmalloc
return slab->alloc<zero_mem>(sl, ffl, rsize, large_allocator.memory_provider);
}
ALWAYSINLINE void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass)
FAST_PATH void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass)
{
#ifdef CHECK_CLIENT
Slab* slab = Slab::get(p);
@@ -1083,7 +1084,7 @@ namespace snmalloc
small_dealloc_offseted(super, offseted, sizeclass);
}
ALWAYSINLINE void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass)
FAST_PATH void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass)
{
MEASURE_TIME(small_dealloc, 4, 16);
stats().sizeclass_dealloc(sizeclass);
@@ -1095,7 +1096,7 @@ namespace snmalloc
small_dealloc_offseted_slow(super, p, sizeclass);
}
NOINLINE void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass)
SLOW_PATH void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass)
{
bool was_full = super->is_full();
SlabList* sl = &small_classes[sizeclass];

4
src/mem/largealloc.h
View File

@@ -189,7 +189,7 @@ namespace snmalloc
* (over the lifetime of this process). If the underlying system does not
* support low memory notifications, this will return 0.
*/
ALWAYSINLINE
FAST_PATH
uint64_t low_memory_epoch()
{
if constexpr (pal_supports<LowMemoryNotification>())
@@ -235,7 +235,7 @@ namespace snmalloc
}
}
ALWAYSINLINE void lazy_decommit_if_needed()
FAST_PATH void lazy_decommit_if_needed()
{
#ifdef TEST_LAZY_DECOMMIT
static_assert(

8
src/mem/pagemap.h
View File

@@ -105,7 +105,7 @@ namespace snmalloc
std::atomic<PagemapEntry*> top[TOPLEVEL_ENTRIES]; // = {nullptr};
template<bool create_addr>
inline PagemapEntry* get_node(std::atomic<PagemapEntry*>* e, bool& result)
FAST_PATH PagemapEntry* get_node(std::atomic<PagemapEntry*>* e, bool& result)
{
// The page map nodes are all allocated directly from the OS zero
// initialised with a system call. We don't need any ordered to guarantee
@@ -128,7 +128,7 @@ namespace snmalloc
}
}
NOINLINE PagemapEntry* get_node_slow(std::atomic<PagemapEntry*>* e, bool& result)
SLOW_PATH PagemapEntry* get_node_slow(std::atomic<PagemapEntry*>* e, bool& result)
{
// The page map nodes are all allocated directly from the OS zero
// initialised with a system call. We don't need any ordered to guarantee
@@ -161,7 +161,7 @@ namespace snmalloc
}
template<bool create_addr>
inline std::pair<Leaf*, size_t> get_leaf_index(uintptr_t addr, bool& result)
FAST_PATH std::pair<Leaf*, size_t> get_leaf_index(uintptr_t addr, bool& result)
{
#ifdef FreeBSD_KERNEL
// Zero the top 16 bits - kernel addresses all have them set, but the
@@ -204,7 +204,7 @@ namespace snmalloc
}
template<bool create_addr>
inline std::atomic<T>* get_addr(uintptr_t p, bool& success)
FAST_PATH std::atomic<T>* get_addr(uintptr_t p, bool& success)
{
auto leaf_ix = get_leaf_index<create_addr>(p, success);
return &(leaf_ix.first->values[leaf_ix.second]);

4
src/mem/slab.h
View File

@@ -144,7 +144,7 @@ namespace snmalloc
// Returns true, if it deallocation can proceed without changing any status bits.
// Note that this does remove the use from the meta slab, so it doesn't need doing
// on the slow path.
ALWAYSINLINE bool dealloc_fast(Superslab* super, void* p)
FAST_PATH bool dealloc_fast(Superslab* super, void* p)
{
Metaslab& meta = super->get_meta(this);
#ifdef CHECK_CLIENT
@@ -179,7 +179,7 @@ namespace snmalloc
// Returns a complex return code for managing the superslab meta data.
// i.e. This deallocation could make an entire superslab free.
template<typename MemoryProvider>
NOINLINE typename Superslab::Action dealloc_slow(
SLOW_PATH typename Superslab::Action dealloc_slow(
SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider)
{
Metaslab& meta = super->get_meta(this);

18
src/mem/threadalloc.h
View File

@@ -1,4 +1,4 @@
#pragma once
#pragma once
#include "../ds/helpers.h"
#include "globalalloc.h"
@@ -28,7 +28,7 @@ namespace snmalloc
class ThreadAllocUntypedWrapper
{
public:
static inline Alloc*& get()
static FAST_PATH Alloc*& get()
{
return (Alloc*&)ThreadAllocUntyped::get();
}
@@ -76,7 +76,7 @@ namespace snmalloc
* The non-create case exists so that the `per_thread` variable can be a
* local static and not a global, allowing ODR to deduplicate it.
*/
static inline Alloc*& get(bool create = true)
static FAST_PATH Alloc*& get(bool create = true)
{
static thread_local Alloc* per_thread;
if (!per_thread && create)
@@ -224,7 +224,7 @@ namespace snmalloc
* Private accessor to the per thread allocator
* Provides no checking for initialization
*/
static ALWAYSINLINE Alloc*& inner_get()
static FAST_PATH Alloc*& inner_get()
{
static thread_local Alloc* per_thread;
return per_thread;
@@ -242,7 +242,7 @@ namespace snmalloc
/**
* Private initialiser for the per thread allocator
*/
static NOINLINE Alloc*& inner_init()
static SLOW_PATH Alloc*& inner_init()
{
Alloc*& per_thread = inner_get();
@@ -262,15 +262,15 @@ namespace snmalloc
// Associate the new allocator with the destructor.
tls_set_value(key, &per_thread);
# ifdef USE_SNMALLOC_STATS
# ifdef USE_SNMALLOC_STATS
// Allocator is up and running now, safe to call atexit.
if (first)
{
atexit(print_stats);
}
# else
# else
UNUSED(first);
# endif
# endif
}
return per_thread;
}
@@ -280,7 +280,7 @@ namespace snmalloc
* Public interface, returns the allocator for the current thread,
* constructing it if necessary.
*/
static ALWAYSINLINE Alloc*& get()
static FAST_PATH Alloc*& get()
{
Alloc*& per_thread = inner_get();