From 621b7e6b9a34e5d10ed8caa46d5cd0713eaefa51 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 2 Jul 2019 10:51:18 +0100 Subject: [PATCH] Clang format. --- src/ds/bits.h | 4 +++- src/mem/alloc.h | 36 +++++++++++++++++---------------- src/mem/metaslab.h | 14 ++++++------- src/mem/pagemap.h | 13 +++++++----- src/mem/sizeclass.h | 9 +++++---- src/mem/sizeclasstable.h | 11 +++++----- src/mem/slab.h | 43 +++++++++++++++++++++++----------------- src/mem/threadalloc.h | 8 ++++---- 8 files changed, 76 insertions(+), 62 deletions(-) diff --git a/src/ds/bits.h b/src/ds/bits.h index cf6f79c..f7c3d77 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -28,7 +28,9 @@ # define SLOW_PATH NOINLINE # define FAST_PATH ALWAYSINLINE # ifdef NDEBUG -# define ASSUME(x) if (!(x)) __builtin_unreachable(); +# define ASSUME(x) \ + if (!(x)) \ + __builtin_unreachable(); # else # define ASSUME(x) assert(x); # endif diff --git a/src/mem/alloc.h b/src/mem/alloc.h index c03de63..ffd4425 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -223,15 +223,15 @@ namespace snmalloc // This class is just used so that the free lists are the first entry // in the allocator and hence has better code gen. - // It contains a free list per small size class. These are used for allocation - // on the fast path. - // This part of the code is inspired by mimalloc. + // It contains a free list per small size class. These are used for + // allocation on the fast path. This part of the code is inspired by mimalloc. class FastFreeLists { protected: FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES]; + public: - FastFreeLists () : small_fast_free_lists() {} + FastFreeLists() : small_fast_free_lists() {} }; /** @@ -255,7 +255,8 @@ namespace snmalloc class PageMap = SNMALLOC_DEFAULT_PAGEMAP, bool IsQueueInline = true> class Allocator - : public FastFreeLists, public Pooled> + : public FastFreeLists, + public Pooled> { LargeAlloc large_allocator; PageMap page_map; @@ -301,7 +302,7 @@ namespace snmalloc return medium_alloc(sizeclass, rsize, size); } else - { + { handle_message_queue(); return large_alloc(size); } @@ -323,7 +324,7 @@ namespace snmalloc stats().alloc_request(size); // Allocate memory of a dynamically known size. - // Perform the - 1 on size, so that zero wraps around and ends up on + // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1))) { @@ -447,7 +448,6 @@ namespace snmalloc // pointer. uint8_t size = pagemap().get(address_cast(p)); - Superslab* super = Superslab::get(p); if (likely(size == PMSuperslab)) @@ -1019,14 +1019,13 @@ namespace snmalloc sizeclass_t sizeclass = size_to_sizeclass(size); stats().sizeclass_alloc(sizeclass); - assert(sizeclass < NUM_SMALL_CLASSES); auto& fl = small_fast_free_lists[sizeclass]; auto head = fl.value; if (likely((reinterpret_cast(head) & 1) == 0)) { - void * p = head; - // Read the next slot from the memory that's about to be allocated. + void* p = head; + // Read the next slot from the memory that's about to be allocated. fl.value = Metaslab::follow_next(p); if constexpr (zero_mem == YesZero) @@ -1040,8 +1039,7 @@ namespace snmalloc } template - SLOW_PATH - void* small_alloc_slow(sizeclass_t sizeclass) + SLOW_PATH void* small_alloc_slow(sizeclass_t sizeclass) { handle_message_queue(); size_t rsize = sizeclass_to_size(sizeclass); @@ -1064,10 +1062,12 @@ namespace snmalloc sl.insert(slab->get_link()); } auto& ffl = small_fast_free_lists[sizeclass]; - return slab->alloc(sl, ffl, rsize, large_allocator.memory_provider); + return slab->alloc( + sl, ffl, rsize, large_allocator.memory_provider); } - FAST_PATH void small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) + FAST_PATH void + small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass) { #ifdef CHECK_CLIENT Slab* slab = Slab::get(p); @@ -1081,7 +1081,8 @@ namespace snmalloc small_dealloc_offseted(super, offseted, sizeclass); } - FAST_PATH void small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) + FAST_PATH void + small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass) { MEASURE_TIME(small_dealloc, 4, 16); stats().sizeclass_dealloc(sizeclass); @@ -1093,7 +1094,8 @@ namespace snmalloc small_dealloc_offseted_slow(super, p, sizeclass); } - SLOW_PATH void small_dealloc_offseted_slow(Superslab* super, void* p, sizeclass_t sizeclass) + SLOW_PATH void small_dealloc_offseted_slow( + Superslab* super, void* p, sizeclass_t sizeclass) { bool was_full = super->is_full(); SlabList* sl = &small_classes[sizeclass]; diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index df03708..f737e71 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -40,10 +40,10 @@ namespace snmalloc // list. The list entries are stored as the first pointer // in each unused object. The terminator is a pointer or // offset into the block with the bottom bit set. This means - // I.e. + // I.e. // * an empty list has a head of 1. // * a one element list has an head contains an offset to this - // this block, and then contains a pointer with the bottom + // this block, and then contains a pointer with the bottom // bit set. Mod head; @@ -105,8 +105,7 @@ namespace snmalloc *static_cast(p) = head; #else *static_cast(p) = head; - *(static_cast(p) + 1) = - address_cast(head) ^ POISON; + *(static_cast(p) + 1) = address_cast(head) ^ POISON; #endif } @@ -164,7 +163,7 @@ namespace snmalloc } both = !both; - length ++; + length++; } return length; #else @@ -204,7 +203,8 @@ namespace snmalloc { // Check we are looking at a correctly aligned block void* start = curr; - assert(((address_cast(start) - address_cast(slab) - offset) % size) == 0); + assert( + ((address_cast(start) - address_cast(slab) - offset) % size) == 0); // Account for free elements in free list accounted_for += size; @@ -229,7 +229,7 @@ namespace snmalloc // haven't completely filled this block at any point. assert(link == get_initial_offset(sizeclass, is_short)); } - + assert(!is_full()); // Add the link node. accounted_for += size; diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index 4b944c2..e870084 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -105,7 +105,8 @@ namespace snmalloc std::atomic top[TOPLEVEL_ENTRIES]; // = {nullptr}; template - FAST_PATH PagemapEntry* get_node(std::atomic* e, bool& result) + FAST_PATH PagemapEntry* + get_node(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -127,8 +128,9 @@ namespace snmalloc return nullptr; } } - - SLOW_PATH PagemapEntry* get_node_slow(std::atomic* e, bool& result) + + SLOW_PATH PagemapEntry* + get_node_slow(std::atomic* e, bool& result) { // The page map nodes are all allocated directly from the OS zero // initialised with a system call. We don't need any ordered to guarantee @@ -149,7 +151,7 @@ namespace snmalloc else { while (address_cast(e->load(std::memory_order_relaxed)) == - LOCKED_ENTRY) + LOCKED_ENTRY) { bits::pause(); } @@ -161,7 +163,8 @@ namespace snmalloc } template - FAST_PATH std::pair get_leaf_index(uintptr_t addr, bool& result) + FAST_PATH std::pair + get_leaf_index(uintptr_t addr, bool& result) { #ifdef FreeBSD_KERNEL // Zero the top 16 bits - kernel addresses all have them set, but the diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 735adab..0665fed 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -10,12 +10,13 @@ namespace snmalloc constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short); constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); - constexpr static size_t sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); - constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); + constexpr static size_t + sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass); + constexpr static size_t + sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass); static sizeclass_t size_to_sizeclass(size_t size); - constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) { // Don't use sizeclasses that are not a multiple of the alignment. @@ -28,7 +29,7 @@ namespace snmalloc constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class) { return bits::one_at_bit(large_class + SUPERSLAB_BITS); - } + } // Small classes range from [MIN, SLAB], i.e. inclusive. static constexpr size_t NUM_SMALL_CLASSES = diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index ebf8116..d0e3bb1 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -15,7 +15,8 @@ namespace snmalloc return (s - 1) >> PTR_BITS; } - constexpr static size_t sizeclass_lookup_size = sizeclass_lookup_index(SLAB_SIZE + 1); + constexpr static size_t sizeclass_lookup_size = + sizeclass_lookup_index(SLAB_SIZE + 1); struct SizeClassTable { @@ -27,7 +28,6 @@ namespace snmalloc ModArray short_initial_offset_ptr; ModArray medium_slab_slots; - constexpr SizeClassTable() : size(), cache_friendly_mask(), @@ -42,8 +42,8 @@ namespace snmalloc size[sizeclass] = bits::from_exp_mant(sizeclass); if (sizeclass < NUM_SMALL_CLASSES) - { - for ( ; curr <= size[sizeclass]; curr+= 1 << PTR_BITS) + { + for (; curr <= size[sizeclass]; curr += 1 << PTR_BITS) { sizeclass_lookup[sizeclass_lookup_index(curr)] = sizeclass; } @@ -109,7 +109,7 @@ namespace snmalloc static inline sizeclass_t size_to_sizeclass(size_t size) { - if ((size-1) <= (SLAB_SIZE-1)) + if ((size - 1) <= (SLAB_SIZE - 1)) { auto index = sizeclass_lookup_index(size); ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE)); @@ -123,7 +123,6 @@ namespace snmalloc bits::to_exp_mant(size)); } - constexpr static inline uint16_t medium_slab_free(sizeclass_t sizeclass) { return sizeclass_metadata diff --git a/src/mem/slab.h b/src/mem/slab.h index 850b595..387bce6 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -37,7 +37,11 @@ namespace snmalloc } template - inline void* alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize, MemoryProvider& memory_provider) + inline void* alloc( + SlabList& sl, + FreeListHead& fast_free_list, + size_t rsize, + MemoryProvider& memory_provider) { // Read the head from the metadata stored in the superslab. Metaslab& meta = get_meta(); @@ -67,22 +71,23 @@ namespace snmalloc } else { - void* curr = nullptr; bool commit = false; while (true) { size_t newbumpptr = bumpptr + rsize; - auto alignedbumpptr = - bits::align_up(bumpptr - 1, OS_PAGE_SIZE); + auto alignedbumpptr = bits::align_up(bumpptr - 1, OS_PAGE_SIZE); auto alignednewbumpptr = bits::align_up(newbumpptr, OS_PAGE_SIZE); if (alignedbumpptr != alignednewbumpptr) { // We have committed once already. - if (commit) break; + if (commit) + break; - memory_provider.template notify_using(pointer_offset(this, alignedbumpptr), alignednewbumpptr - alignedbumpptr); + memory_provider.template notify_using( + pointer_offset(this, alignedbumpptr), + alignednewbumpptr - alignedbumpptr); commit = true; } @@ -109,7 +114,7 @@ namespace snmalloc { p = pointer_offset(this, meta.head); - // Read the next slot from the memory that's about to be allocated. + // Read the next slot from the memory that's about to be allocated. void* next = Metaslab::follow_next(p); // Put everything in allocators small_class free list. meta.head = 1; @@ -119,7 +124,7 @@ namespace snmalloc meta.used = meta.allocated - 1; } - assert (is_start_of_object(Superslab::get(p), p)); + assert(is_start_of_object(Superslab::get(p), p)); meta.debug_slab_invariant(is_short(), this); @@ -142,9 +147,9 @@ namespace snmalloc address_cast(this) + SLAB_SIZE - address_cast(p)); } - // Returns true, if it deallocation can proceed without changing any status bits. - // Note that this does remove the use from the meta slab, so it doesn't need doing - // on the slow path. + // Returns true, if it deallocation can proceed without changing any status + // bits. Note that this does remove the use from the meta slab, so it + // doesn't need doing on the slow path. FAST_PATH bool dealloc_fast(Superslab* super, void* p) { Metaslab& meta = super->get_meta(this); @@ -156,10 +161,12 @@ namespace snmalloc meta.sub_use(); bool was_full = meta.is_full(); - if (unlikely(was_full)) return false; + if (unlikely(was_full)) + return false; bool is_unused = meta.is_unused(); - if (unlikely(is_unused)) return false; + if (unlikely(is_unused)) + return false; // Update the head and the next pointer in the free list. uint16_t head = meta.head; @@ -175,7 +182,7 @@ namespace snmalloc return true; } - // If dealloc fast returns false, then call this. + // If dealloc fast returns false, then call this. // This does not need to remove the "use" as done by the fast path. // Returns a complex return code for managing the superslab meta data. // i.e. This deallocation could make an entire superslab free. @@ -184,7 +191,7 @@ namespace snmalloc SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider) { Metaslab& meta = super->get_meta(this); - + bool was_full = meta.is_full(); bool is_unused = meta.is_unused(); @@ -202,7 +209,7 @@ namespace snmalloc // Update the head and the sizeclass link. uint16_t index = pointer_to_index(p); assert(meta.head == 1); -// assert(meta.fully_allocated(is_short())); + // assert(meta.fully_allocated(is_short())); meta.link = index; // Push on the list of slabs for this sizeclass. @@ -210,7 +217,7 @@ namespace snmalloc meta.debug_slab_invariant(is_short(), this); return Superslab::NoSlabReturn; } - + if (is_unused) { // Remove from the sizeclass list and dealloc on the superslab. @@ -221,7 +228,7 @@ namespace snmalloc return super->dealloc_slab(this, memory_provider); } - + abort(); } diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index cc17005..ef3fd8a 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -1,4 +1,4 @@ - #pragma once +#pragma once #include "../ds/helpers.h" #include "globalalloc.h" @@ -262,15 +262,15 @@ namespace snmalloc // Associate the new allocator with the destructor. tls_set_value(key, &per_thread); - # ifdef USE_SNMALLOC_STATS +# ifdef USE_SNMALLOC_STATS // Allocator is up and running now, safe to call atexit. if (first) { atexit(print_stats); } - # else +# else UNUSED(first); - # endif +# endif } return per_thread; }