From 7de46182a804606a9d32d872aa49da02f02af1c7 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 8 May 2019 15:29:43 +0100 Subject: [PATCH 1/2] Used start of a slab for link. --- .gitignore | 2 ++ difference.md | 20 +++++++++++++++++ src/mem/metaslab.h | 54 +++++++++++++++++++++------------------------ src/mem/slab.h | 18 +++++++-------- src/mem/superslab.h | 13 +++++++---- 5 files changed, 65 insertions(+), 42 deletions(-) create mode 100644 difference.md diff --git a/.gitignore b/.gitignore index 4b9d305..f25dbf6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +release*/ +debug*/ build*/ CMakeFiles/ .vscode/ diff --git a/difference.md b/difference.md new file mode 100644 index 0000000..20f4f35 --- /dev/null +++ b/difference.md @@ -0,0 +1,20 @@ +# Difference from published paper + +This document outlines the changes that have diverged from the published paper +on `snmalloc`. + + 1. Link no longer terminates the bump-free list. The paper describes a + complex invariant for how the final element of the bump-free list can + also be the link node. + + We now have a much simpler invariant. The link is either 1, signifying + the block is completely full. Or not 1, signifying it has at least one + free element at the offset contained in link, and that contains the DLL + node for this sizeclass. + + The bump-free list contains additional free elements, and the remaining + bump allocated space. + + The value 1, is never a valid bump allocation value, as we initially + allocate the first entry as the link, so we can use 1 as the no more bump + space value. \ No newline at end of file diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 67b4546..e258543 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -19,15 +19,12 @@ namespace snmalloc } }; - using SlabList = DLList>; + using SlabList = DLList; static_assert( sizeof(SlabLink) <= MIN_ALLOC_SIZE, "Need to be able to pack a SlabLink into any free small alloc"); - static constexpr uint16_t SLABLINK_INDEX = - static_cast(SLAB_SIZE - sizeof(SlabLink)); - // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. class Metaslab @@ -42,16 +39,11 @@ namespace snmalloc // of where we have allocated up to in this slab. Otherwise, // it represents the location of the first block in the free // list. The free list is chained through deallocated blocks. - // It either terminates with a bump ptr, or if all the space is in - // the free list, then the last block will be also referenced by - // link. - // Note that, in the case that this is the first block in the size - // class list, where all the unused memory is in the free list, - // then the last block can both be interpreted as a final bump - // pointer entry, and the first entry in the doubly linked list. - // The terminal value in the free list, and the terminal value in - // the SlabLink previous field will alias. The SlabLink uses ~0 for - // its terminal value to be a valid terminal bump ptr. + // It is terminated with a bump ptr. + // + // Note that, the first entry in a slab is never bump allocated + // but is used for the link. This means that 1 represents the fully + // bump allocated slab. Mod head; // When a slab has free space it will be on the has space list for // that size class. We use an empty block in this slab to be the @@ -84,13 +76,14 @@ namespace snmalloc bool is_full() { - return (head & 2) != 0; + return link == 1; } void set_full() { assert(head == 1); - head = static_cast(~0); + assert(link != 1); + link = 1; } SlabLink* get_link(Slab* slab) @@ -141,28 +134,31 @@ namespace snmalloc // Account for free elements in free list accounted_for += size; assert(SLAB_SIZE >= accounted_for); - // We are not guaranteed to hit a bump ptr unless - // we are the top element on the size class, so treat as - // a list segment. - if (curr == link) - break; + // We should never reach the link node in the free list. + assert(curr != link); + // Iterate bump/free list segment curr = *reinterpret_cast(pointer_offset(slab, curr)); } - // Check we terminated traversal on a correctly aligned block - uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass); - assert((start - offset) % size == 0); - - if (curr != link) + if (curr != 1) { - // The link should be at the special end location as we - // haven't completely filled this block at any point. - assert(link == SLABLINK_INDEX); + // Check we terminated traversal on a correctly aligned block + uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass); + assert((start - offset) % size == 0); + // Account for to be bump allocated space accounted_for += SLAB_SIZE - (curr - 1); + + // The link should be the first allocation as we + // haven't completely filled this block at any point. + assert(link == (get_slab_offset(sizeclass, is_short) - 1)); } + assert(!is_full()); + // Add the link node. + accounted_for += size; + // All space accounted for assert(SLAB_SIZE == accounted_for); #else diff --git a/src/mem/slab.h b/src/mem/slab.h index 87b745e..21afeb9 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -58,19 +58,20 @@ namespace snmalloc } else { - // This slab is being bump allocated. - p = pointer_offset(this, head - 1); - meta.head = (head + static_cast(rsize)) & (SLAB_SIZE - 1); if (meta.head == 1) { + p = pointer_offset(this, meta.link); + sc->pop(); meta.set_full(); } + else + { + // This slab is being bump allocated. + p = pointer_offset(this, head - 1); + meta.head = (head + static_cast(rsize)) & (SLAB_SIZE - 1); + } } - // If we're full, we're no longer the current slab for this sizeclass - if (meta.is_full()) - sc->pop(); - meta.debug_slab_invariant(is_short(), this); if constexpr (zero_mem == YesZero) @@ -110,8 +111,7 @@ namespace snmalloc { // Update the head and the sizeclass link. uint16_t index = pointer_to_index(p); - meta.head = index; - assert(meta.valid_head(is_short())); + assert(meta.head == 1); meta.link = index; // Push on the list of slabs for this sizeclass. diff --git a/src/mem/superslab.h b/src/mem/superslab.h index 9cd3cdf..afa9e98 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -159,9 +159,12 @@ namespace snmalloc if ((used & 1) == 1) return alloc_slab(sizeclass, memory_provider); - meta[0].head = get_slab_offset(sizeclass, true); + auto rsize = sizeclass_to_size(sizeclass); + + meta[0].head = + (get_slab_offset(sizeclass, true) + rsize) & (SLAB_SIZE - 1); meta[0].sizeclass = sizeclass; - meta[0].link = SLABLINK_INDEX; + meta[0].link = get_slab_offset(sizeclass, true) - 1; if constexpr (decommit_strategy == DecommitAll) { @@ -182,9 +185,11 @@ namespace snmalloc uint8_t n = meta[h].next; - meta[h].head = get_slab_offset(sizeclass, false); + auto rsize = sizeclass_to_size(sizeclass); + meta[h].head = + (get_slab_offset(sizeclass, false) + rsize) & (SLAB_SIZE - 1); meta[h].sizeclass = sizeclass; - meta[h].link = SLABLINK_INDEX; + meta[h].link = get_slab_offset(sizeclass, false) - 1; head = h + n + 1; used += 2; From ef18d7d8d594f227ff2603b79582fa653b1a073d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 21 May 2019 15:25:33 +0100 Subject: [PATCH 2/2] Made initialization of slab clearer. --- src/mem/metaslab.h | 7 +++---- src/mem/sizeclass.h | 3 ++- src/mem/sizeclasstable.h | 43 ++++++++++++++++++++++++++++++---------- src/mem/superslab.h | 13 ++++-------- 4 files changed, 41 insertions(+), 25 deletions(-) diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index e258543..e933b19 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -94,12 +94,11 @@ namespace snmalloc bool valid_head(bool is_short) { size_t size = sizeclass_to_size(sizeclass); - size_t offset = get_slab_offset(sizeclass, is_short); + size_t slab_start = get_initial_link(sizeclass, is_short); size_t all_high_bits = ~static_cast(1); size_t head_start = remove_cache_friendly_offset(head & all_high_bits, sizeclass); - size_t slab_start = offset & all_high_bits; return ((head_start - slab_start) % size) == 0; } @@ -108,7 +107,7 @@ namespace snmalloc { #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) size_t size = sizeclass_to_size(sizeclass); - size_t offset = get_slab_offset(sizeclass, is_short) - 1; + size_t offset = get_initial_link(sizeclass, is_short); size_t accounted_for = used * size + offset; @@ -152,7 +151,7 @@ namespace snmalloc // The link should be the first allocation as we // haven't completely filled this block at any point. - assert(link == (get_slab_offset(sizeclass, is_short) - 1)); + assert(link == get_initial_link(sizeclass, is_short)); } assert(!is_full()); diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 77ce364..724ac28 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -4,7 +4,8 @@ namespace snmalloc { - constexpr static uint16_t get_slab_offset(uint8_t sc, bool is_short); + constexpr static uint16_t get_initial_bumpptr(uint8_t sc, bool is_short); + constexpr static uint16_t get_initial_link(uint8_t sc, bool is_short); constexpr static size_t sizeclass_to_size(uint8_t sizeclass); constexpr static size_t sizeclass_to_cache_friendly_mask(uint8_t sizeclass); constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(uint8_t sc); diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index a9a80c6..06c80aa 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -12,7 +12,8 @@ namespace snmalloc ModArray inverse_cache_friendly_mask; ModArray bump_ptr_start; ModArray short_bump_ptr_start; - ModArray count_per_slab; + ModArray initial_link_ptr; + ModArray short_initial_link_ptr; ModArray medium_slab_slots; constexpr SizeClassTable() @@ -21,7 +22,8 @@ namespace snmalloc inverse_cache_friendly_mask(), bump_ptr_start(), short_bump_ptr_start(), - count_per_slab(), + initial_link_ptr(), + short_initial_link_ptr(), medium_slab_slots() { for (uint8_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) @@ -40,10 +42,25 @@ namespace snmalloc for (uint8_t i = 0; i < NUM_SMALL_CLASSES; i++) { + // We align to the end of the block to remove special cases for the + // short block. Calculate remainders + size_t short_correction = short_slab_size % size[i]; + size_t correction = SLAB_SIZE % size[i]; + + // First element in the block is the link + initial_link_ptr[i] = static_cast(correction); + short_initial_link_ptr[i] = + static_cast(header_size + short_correction); + + // Move to object after link. + auto short_after_link = short_initial_link_ptr[i] + size[i]; + size_t after_link = initial_link_ptr[i] + size[i]; + + // Bump ptr has bottom bit set. + // In case we only have one object on this slab check for wrap around. short_bump_ptr_start[i] = - static_cast(1 + (short_slab_size % size[i]) + header_size); - bump_ptr_start[i] = static_cast(1 + (SLAB_SIZE % size[i])); - count_per_slab[i] = static_cast(SLAB_SIZE / size[i]); + static_cast((short_after_link + 1) % SLAB_SIZE); + bump_ptr_start[i] = static_cast((after_link + 1) % SLAB_SIZE); } for (uint8_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) @@ -56,7 +73,8 @@ namespace snmalloc static constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); - static inline constexpr uint16_t get_slab_offset(uint8_t sc, bool is_short) + static inline constexpr uint16_t + get_initial_bumpptr(uint8_t sc, bool is_short) { if (is_short) return sizeclass_metadata.short_bump_ptr_start[sc]; @@ -64,6 +82,14 @@ namespace snmalloc return sizeclass_metadata.bump_ptr_start[sc]; } + static inline constexpr uint16_t get_initial_link(uint8_t sc, bool is_short) + { + if (is_short) + return sizeclass_metadata.short_initial_link_ptr[sc]; + + return sizeclass_metadata.initial_link_ptr[sc]; + } + constexpr static inline size_t sizeclass_to_size(uint8_t sizeclass) { return sizeclass_metadata.size[sizeclass]; @@ -81,11 +107,6 @@ namespace snmalloc return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass]; } - constexpr static inline size_t sizeclass_to_count(uint8_t sizeclass) - { - return sizeclass_metadata.count_per_slab[sizeclass]; - } - constexpr static inline uint16_t medium_slab_free(uint8_t sizeclass) { return sizeclass_metadata diff --git a/src/mem/superslab.h b/src/mem/superslab.h index afa9e98..c131d78 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -159,12 +159,9 @@ namespace snmalloc if ((used & 1) == 1) return alloc_slab(sizeclass, memory_provider); - auto rsize = sizeclass_to_size(sizeclass); - - meta[0].head = - (get_slab_offset(sizeclass, true) + rsize) & (SLAB_SIZE - 1); + meta[0].head = get_initial_bumpptr(sizeclass, true); meta[0].sizeclass = sizeclass; - meta[0].link = get_slab_offset(sizeclass, true) - 1; + meta[0].link = get_initial_link(sizeclass, true); if constexpr (decommit_strategy == DecommitAll) { @@ -185,11 +182,9 @@ namespace snmalloc uint8_t n = meta[h].next; - auto rsize = sizeclass_to_size(sizeclass); - meta[h].head = - (get_slab_offset(sizeclass, false) + rsize) & (SLAB_SIZE - 1); + meta[h].head = get_initial_bumpptr(sizeclass, false); meta[h].sizeclass = sizeclass; - meta[h].link = get_slab_offset(sizeclass, false) - 1; + meta[h].link = get_initial_link(sizeclass, false); head = h + n + 1; used += 2;