diff --git a/.gitignore b/.gitignore index 4b9d305..f25dbf6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +release*/ +debug*/ build*/ CMakeFiles/ .vscode/ diff --git a/difference.md b/difference.md new file mode 100644 index 0000000..20f4f35 --- /dev/null +++ b/difference.md @@ -0,0 +1,20 @@ +# Difference from published paper + +This document outlines the changes that have diverged from the published paper +on `snmalloc`. + + 1. Link no longer terminates the bump-free list. The paper describes a + complex invariant for how the final element of the bump-free list can + also be the link node. + + We now have a much simpler invariant. The link is either 1, signifying + the block is completely full. Or not 1, signifying it has at least one + free element at the offset contained in link, and that contains the DLL + node for this sizeclass. + + The bump-free list contains additional free elements, and the remaining + bump allocated space. + + The value 1, is never a valid bump allocation value, as we initially + allocate the first entry as the link, so we can use 1 as the no more bump + space value. \ No newline at end of file diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 67b4546..e258543 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -19,15 +19,12 @@ namespace snmalloc } }; - using SlabList = DLList>; + using SlabList = DLList; static_assert( sizeof(SlabLink) <= MIN_ALLOC_SIZE, "Need to be able to pack a SlabLink into any free small alloc"); - static constexpr uint16_t SLABLINK_INDEX = - static_cast(SLAB_SIZE - sizeof(SlabLink)); - // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. class Metaslab @@ -42,16 +39,11 @@ namespace snmalloc // of where we have allocated up to in this slab. Otherwise, // it represents the location of the first block in the free // list. The free list is chained through deallocated blocks. - // It either terminates with a bump ptr, or if all the space is in - // the free list, then the last block will be also referenced by - // link. - // Note that, in the case that this is the first block in the size - // class list, where all the unused memory is in the free list, - // then the last block can both be interpreted as a final bump - // pointer entry, and the first entry in the doubly linked list. - // The terminal value in the free list, and the terminal value in - // the SlabLink previous field will alias. The SlabLink uses ~0 for - // its terminal value to be a valid terminal bump ptr. + // It is terminated with a bump ptr. + // + // Note that, the first entry in a slab is never bump allocated + // but is used for the link. This means that 1 represents the fully + // bump allocated slab. Mod head; // When a slab has free space it will be on the has space list for // that size class. We use an empty block in this slab to be the @@ -84,13 +76,14 @@ namespace snmalloc bool is_full() { - return (head & 2) != 0; + return link == 1; } void set_full() { assert(head == 1); - head = static_cast(~0); + assert(link != 1); + link = 1; } SlabLink* get_link(Slab* slab) @@ -141,28 +134,31 @@ namespace snmalloc // Account for free elements in free list accounted_for += size; assert(SLAB_SIZE >= accounted_for); - // We are not guaranteed to hit a bump ptr unless - // we are the top element on the size class, so treat as - // a list segment. - if (curr == link) - break; + // We should never reach the link node in the free list. + assert(curr != link); + // Iterate bump/free list segment curr = *reinterpret_cast(pointer_offset(slab, curr)); } - // Check we terminated traversal on a correctly aligned block - uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass); - assert((start - offset) % size == 0); - - if (curr != link) + if (curr != 1) { - // The link should be at the special end location as we - // haven't completely filled this block at any point. - assert(link == SLABLINK_INDEX); + // Check we terminated traversal on a correctly aligned block + uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass); + assert((start - offset) % size == 0); + // Account for to be bump allocated space accounted_for += SLAB_SIZE - (curr - 1); + + // The link should be the first allocation as we + // haven't completely filled this block at any point. + assert(link == (get_slab_offset(sizeclass, is_short) - 1)); } + assert(!is_full()); + // Add the link node. + accounted_for += size; + // All space accounted for assert(SLAB_SIZE == accounted_for); #else diff --git a/src/mem/slab.h b/src/mem/slab.h index 87b745e..21afeb9 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -58,19 +58,20 @@ namespace snmalloc } else { - // This slab is being bump allocated. - p = pointer_offset(this, head - 1); - meta.head = (head + static_cast(rsize)) & (SLAB_SIZE - 1); if (meta.head == 1) { + p = pointer_offset(this, meta.link); + sc->pop(); meta.set_full(); } + else + { + // This slab is being bump allocated. + p = pointer_offset(this, head - 1); + meta.head = (head + static_cast(rsize)) & (SLAB_SIZE - 1); + } } - // If we're full, we're no longer the current slab for this sizeclass - if (meta.is_full()) - sc->pop(); - meta.debug_slab_invariant(is_short(), this); if constexpr (zero_mem == YesZero) @@ -110,8 +111,7 @@ namespace snmalloc { // Update the head and the sizeclass link. uint16_t index = pointer_to_index(p); - meta.head = index; - assert(meta.valid_head(is_short())); + assert(meta.head == 1); meta.link = index; // Push on the list of slabs for this sizeclass. diff --git a/src/mem/superslab.h b/src/mem/superslab.h index 9cd3cdf..afa9e98 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -159,9 +159,12 @@ namespace snmalloc if ((used & 1) == 1) return alloc_slab(sizeclass, memory_provider); - meta[0].head = get_slab_offset(sizeclass, true); + auto rsize = sizeclass_to_size(sizeclass); + + meta[0].head = + (get_slab_offset(sizeclass, true) + rsize) & (SLAB_SIZE - 1); meta[0].sizeclass = sizeclass; - meta[0].link = SLABLINK_INDEX; + meta[0].link = get_slab_offset(sizeclass, true) - 1; if constexpr (decommit_strategy == DecommitAll) { @@ -182,9 +185,11 @@ namespace snmalloc uint8_t n = meta[h].next; - meta[h].head = get_slab_offset(sizeclass, false); + auto rsize = sizeclass_to_size(sizeclass); + meta[h].head = + (get_slab_offset(sizeclass, false) + rsize) & (SLAB_SIZE - 1); meta[h].sizeclass = sizeclass; - meta[h].link = SLABLINK_INDEX; + meta[h].link = get_slab_offset(sizeclass, false) - 1; head = h + n + 1; used += 2;