Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clang format.

Browse Source
This commit is contained in:
Matthew Parkinson
2019-07-02 10:51:18 +01:00
parent b14735ff06
commit 621b7e6b9a
8 changed files with 76 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/ds/bits.h
View File

@@ -28,7 +28,9 @@
# define SLOW_PATH NOINLINE
# define FAST_PATH ALWAYSINLINE
# ifdef NDEBUG
# define ASSUME(x) if (!(x)) __builtin_unreachable();
# define ASSUME(x) \
if (!(x)) \
__builtin_unreachable();
# else
# define ASSUME(x) assert(x);
# endif

36
src/mem/alloc.h
View File

@@ -223,15 +223,15 @@ namespace snmalloc
// This class is just used so that the free lists are the first entry
// in the allocator and hence has better code gen.
// It contains a free list per small size class. These are used for allocation
// on the fast path.
// This part of the code is inspired by mimalloc.
// It contains a free list per small size class. These are used for
// allocation on the fast path. This part of the code is inspired by mimalloc.
class FastFreeLists
{
protected:
FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES];
public:
FastFreeLists () : small_fast_free_lists() {}
FastFreeLists() : small_fast_free_lists() {}
};
/**
@@ -255,7 +255,8 @@ namespace snmalloc
class PageMap = SNMALLOC_DEFAULT_PAGEMAP,
bool IsQueueInline = true>
class Allocator
: public FastFreeLists, public Pooled<Allocator<MemoryProvider, PageMap, IsQueueInline>>
: public FastFreeLists,
public Pooled<Allocator<MemoryProvider, PageMap, IsQueueInline>>
{
LargeAlloc<MemoryProvider> large_allocator;
PageMap page_map;
@@ -301,7 +302,7 @@ namespace snmalloc
return medium_alloc<zero_mem, allow_reserve>(sizeclass, rsize, size);
}
else
{
{
handle_message_queue();
return large_alloc<zero_mem, allow_reserve>(size);
}
@@ -323,7 +324,7 @@ namespace snmalloc
stats().alloc_request(size);
// Allocate memory of a dynamically known size.
// Perform the - 1 on size, so that zero wraps around and ends up on
// Perform the - 1 on size, so that zero wraps around and ends up on
// slow path.
if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1)))
{
@@ -447,7 +448,6 @@ namespace snmalloc
// pointer.
uint8_t size = pagemap().get(address_cast(p));
Superslab* super = Superslab::get(p);
if (likely(size == PMSuperslab))
@@ -1019,14 +1019,13 @@ namespace snmalloc
sizeclass_t sizeclass = size_to_sizeclass(size);
stats().sizeclass_alloc(sizeclass);
assert(sizeclass < NUM_SMALL_CLASSES);
auto& fl = small_fast_free_lists[sizeclass];
auto head = fl.value;
if (likely((reinterpret_cast<size_t>(head) & 1) == 0))
{
void * p = head;
// Read the next slot from the memory that's about to be allocated.
void* p = head;
// Read the next slot from the memory that's about to be allocated.
fl.value = Metaslab::follow_next(p);
if constexpr (zero_mem == YesZero)
@@ -1040,8 +1039,7 @@ namespace snmalloc
}
template<ZeroMem zero_mem, AllowReserve allow_reserve>
SLOW_PATH
void* small_alloc_slow(sizeclass_t sizeclass)
SLOW_PATH void* small_alloc_slow(sizeclass_t sizeclass)
{
handle_message_queue();
size_t rsize = sizeclass_to_size(sizeclass);
@@ -1064,10 +1062,12 @@ namespace snmalloc
sl.insert(slab->get_link());
}
auto& ffl = small_fast_free_lists[sizeclass];
return slab->alloc<zero_mem>(sl, ffl, rsize, large_allocator.memory_provider);
return slab->alloc<zero_mem>(
sl, ffl, rsize, large_allocator.memory_provider);
}
FAST_PATH void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass)
FAST_PATH void
small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass)
{
#ifdef CHECK_CLIENT
Slab* slab = Slab::get(p);
@@ -1081,7 +1081,8 @@ namespace snmalloc
small_dealloc_offseted(super, offseted, sizeclass);
}
FAST_PATH void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass)
FAST_PATH void
small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass)
{
MEASURE_TIME(small_dealloc, 4, 16);
stats().sizeclass_dealloc(sizeclass);
@@ -1093,7 +1094,8 @@ namespace snmalloc
small_dealloc_offseted_slow(super, p, sizeclass);
}
SLOW_PATH void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass)
SLOW_PATH void small_dealloc_offseted_slow(
Superslab* super, void* p, sizeclass_t sizeclass)
{
bool was_full = super->is_full();
SlabList* sl = &small_classes[sizeclass];

14
src/mem/metaslab.h
View File

@@ -40,10 +40,10 @@ namespace snmalloc
// list. The list entries are stored as the first pointer
// in each unused object. The terminator is a pointer or
// offset into the block with the bottom bit set. This means
// I.e.
// I.e.
// * an empty list has a head of 1.
// * a one element list has an head contains an offset to this
// this block, and then contains a pointer with the bottom
// this block, and then contains a pointer with the bottom
// bit set.
Mod<SLAB_SIZE, uint16_t> head;
@@ -105,8 +105,7 @@ namespace snmalloc
*static_cast<void**>(p) = head;
#else
*static_cast<void**>(p) = head;
*(static_cast<uintptr_t*>(p) + 1) =
address_cast(head) ^ POISON;
*(static_cast<uintptr_t*>(p) + 1) = address_cast(head) ^ POISON;
#endif
}
@@ -164,7 +163,7 @@ namespace snmalloc
}
both = !both;
length ++;
length++;
}
return length;
#else
@@ -204,7 +203,8 @@ namespace snmalloc
{
// Check we are looking at a correctly aligned block
void* start = curr;
assert(((address_cast(start) - address_cast(slab) - offset) % size) == 0);
assert(
((address_cast(start) - address_cast(slab) - offset) % size) == 0);
// Account for free elements in free list
accounted_for += size;
@@ -229,7 +229,7 @@ namespace snmalloc
// haven't completely filled this block at any point.
assert(link == get_initial_offset(sizeclass, is_short));
}
assert(!is_full());
// Add the link node.
accounted_for += size;

13
src/mem/pagemap.h
View File

@@ -105,7 +105,8 @@ namespace snmalloc
std::atomic<PagemapEntry*> top[TOPLEVEL_ENTRIES]; // = {nullptr};
template<bool create_addr>
FAST_PATH PagemapEntry* get_node(std::atomic<PagemapEntry*>* e, bool& result)
FAST_PATH PagemapEntry*
get_node(std::atomic<PagemapEntry*>* e, bool& result)
{
// The page map nodes are all allocated directly from the OS zero
// initialised with a system call. We don't need any ordered to guarantee
@@ -127,8 +128,9 @@ namespace snmalloc
return nullptr;
}
}
SLOW_PATH PagemapEntry* get_node_slow(std::atomic<PagemapEntry*>* e, bool& result)
SLOW_PATH PagemapEntry*
get_node_slow(std::atomic<PagemapEntry*>* e, bool& result)
{
// The page map nodes are all allocated directly from the OS zero
// initialised with a system call. We don't need any ordered to guarantee
@@ -149,7 +151,7 @@ namespace snmalloc
else
{
while (address_cast(e->load(std::memory_order_relaxed)) ==
LOCKED_ENTRY)
LOCKED_ENTRY)
{
bits::pause();
}
@@ -161,7 +163,8 @@ namespace snmalloc
}
template<bool create_addr>
FAST_PATH std::pair<Leaf*, size_t> get_leaf_index(uintptr_t addr, bool& result)
FAST_PATH std::pair<Leaf*, size_t>
get_leaf_index(uintptr_t addr, bool& result)
{
#ifdef FreeBSD_KERNEL
// Zero the top 16 bits - kernel addresses all have them set, but the

9
src/mem/sizeclass.h
View File

@@ -10,12 +10,13 @@ namespace snmalloc
constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short);
constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass);
constexpr static size_t sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass);
constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc);
constexpr static size_t
sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass);
constexpr static size_t
sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc);
constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass);
static sizeclass_t size_to_sizeclass(size_t size);
constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size)
{
// Don't use sizeclasses that are not a multiple of the alignment.
@@ -28,7 +29,7 @@ namespace snmalloc
constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class)
{
return bits::one_at_bit(large_class + SUPERSLAB_BITS);
}
}
// Small classes range from [MIN, SLAB], i.e. inclusive.
static constexpr size_t NUM_SMALL_CLASSES =

11
src/mem/sizeclasstable.h
View File

@@ -15,7 +15,8 @@ namespace snmalloc
return (s - 1) >> PTR_BITS;
}
constexpr static size_t sizeclass_lookup_size = sizeclass_lookup_index(SLAB_SIZE + 1);
constexpr static size_t sizeclass_lookup_size =
sizeclass_lookup_index(SLAB_SIZE + 1);
struct SizeClassTable
{
@@ -27,7 +28,6 @@ namespace snmalloc
ModArray<NUM_SMALL_CLASSES, uint16_t> short_initial_offset_ptr;
ModArray<NUM_MEDIUM_CLASSES, uint16_t> medium_slab_slots;
constexpr SizeClassTable()
: size(),
cache_friendly_mask(),
@@ -42,8 +42,8 @@ namespace snmalloc
size[sizeclass] =
bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(sizeclass);
if (sizeclass < NUM_SMALL_CLASSES)
{
for ( ; curr <= size[sizeclass]; curr+= 1 << PTR_BITS)
{
for (; curr <= size[sizeclass]; curr += 1 << PTR_BITS)
{
sizeclass_lookup[sizeclass_lookup_index(curr)] = sizeclass;
}
@@ -109,7 +109,7 @@ namespace snmalloc
static inline sizeclass_t size_to_sizeclass(size_t size)
{
if ((size-1) <= (SLAB_SIZE-1))
if ((size - 1) <= (SLAB_SIZE - 1))
{
auto index = sizeclass_lookup_index(size);
ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE));
@@ -123,7 +123,6 @@ namespace snmalloc
bits::to_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(size));
}
constexpr static inline uint16_t medium_slab_free(sizeclass_t sizeclass)
{
return sizeclass_metadata

43
src/mem/slab.h
View File

@@ -37,7 +37,11 @@ namespace snmalloc
}
template<ZeroMem zero_mem, typename MemoryProvider>
inline void* alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize, MemoryProvider& memory_provider)
inline void* alloc(
SlabList& sl,
FreeListHead& fast_free_list,
size_t rsize,
MemoryProvider& memory_provider)
{
// Read the head from the metadata stored in the superslab.
Metaslab& meta = get_meta();
@@ -67,22 +71,23 @@ namespace snmalloc
}
else
{
void* curr = nullptr;
bool commit = false;
while (true)
{
size_t newbumpptr = bumpptr + rsize;
auto alignedbumpptr =
bits::align_up(bumpptr - 1, OS_PAGE_SIZE);
auto alignedbumpptr = bits::align_up(bumpptr - 1, OS_PAGE_SIZE);
auto alignednewbumpptr = bits::align_up(newbumpptr, OS_PAGE_SIZE);
if (alignedbumpptr != alignednewbumpptr)
{
// We have committed once already.
if (commit) break;
if (commit)
break;
memory_provider.template notify_using<NoZero>(pointer_offset(this, alignedbumpptr), alignednewbumpptr - alignedbumpptr);
memory_provider.template notify_using<NoZero>(
pointer_offset(this, alignedbumpptr),
alignednewbumpptr - alignedbumpptr);
commit = true;
}
@@ -109,7 +114,7 @@ namespace snmalloc
{
p = pointer_offset(this, meta.head);
// Read the next slot from the memory that's about to be allocated.
// Read the next slot from the memory that's about to be allocated.
void* next = Metaslab::follow_next(p);
// Put everything in allocators small_class free list.
meta.head = 1;
@@ -119,7 +124,7 @@ namespace snmalloc
meta.used = meta.allocated - 1;
}
assert (is_start_of_object(Superslab::get(p), p));
assert(is_start_of_object(Superslab::get(p), p));
meta.debug_slab_invariant(is_short(), this);
@@ -142,9 +147,9 @@ namespace snmalloc
address_cast(this) + SLAB_SIZE - address_cast(p));
}
// Returns true, if it deallocation can proceed without changing any status bits.
// Note that this does remove the use from the meta slab, so it doesn't need doing
// on the slow path.
// Returns true, if it deallocation can proceed without changing any status
// bits. Note that this does remove the use from the meta slab, so it
// doesn't need doing on the slow path.
FAST_PATH bool dealloc_fast(Superslab* super, void* p)
{
Metaslab& meta = super->get_meta(this);
@@ -156,10 +161,12 @@ namespace snmalloc
meta.sub_use();
bool was_full = meta.is_full();
if (unlikely(was_full)) return false;
if (unlikely(was_full))
return false;
bool is_unused = meta.is_unused();
if (unlikely(is_unused)) return false;
if (unlikely(is_unused))
return false;
// Update the head and the next pointer in the free list.
uint16_t head = meta.head;
@@ -175,7 +182,7 @@ namespace snmalloc
return true;
}
// If dealloc fast returns false, then call this.
// If dealloc fast returns false, then call this.
// This does not need to remove the "use" as done by the fast path.
// Returns a complex return code for managing the superslab meta data.
// i.e. This deallocation could make an entire superslab free.
@@ -184,7 +191,7 @@ namespace snmalloc
SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider)
{
Metaslab& meta = super->get_meta(this);
bool was_full = meta.is_full();
bool is_unused = meta.is_unused();
@@ -202,7 +209,7 @@ namespace snmalloc
// Update the head and the sizeclass link.
uint16_t index = pointer_to_index(p);
assert(meta.head == 1);
// assert(meta.fully_allocated(is_short()));
// assert(meta.fully_allocated(is_short()));
meta.link = index;
// Push on the list of slabs for this sizeclass.
@@ -210,7 +217,7 @@ namespace snmalloc
meta.debug_slab_invariant(is_short(), this);
return Superslab::NoSlabReturn;
}
if (is_unused)
{
// Remove from the sizeclass list and dealloc on the superslab.
@@ -221,7 +228,7 @@ namespace snmalloc
return super->dealloc_slab(this, memory_provider);
}
abort();
}

8
src/mem/threadalloc.h
View File

@@ -1,4 +1,4 @@
#pragma once
#pragma once
#include "../ds/helpers.h"
#include "globalalloc.h"
@@ -262,15 +262,15 @@ namespace snmalloc
// Associate the new allocator with the destructor.
tls_set_value(key, &per_thread);
# ifdef USE_SNMALLOC_STATS
# ifdef USE_SNMALLOC_STATS
// Allocator is up and running now, safe to call atexit.
if (first)
{
atexit(print_stats);
}
# else
# else
UNUSED(first);
# endif
# endif
}
return per_thread;
}