diff --git a/src/mem/entropy.h b/src/mem/entropy.h index 17b69a2..fa1b4ec 100644 --- a/src/mem/entropy.h +++ b/src/mem/entropy.h @@ -27,6 +27,8 @@ namespace snmalloc uint64_t local_key; uint64_t local_counter; address_t constant_key; + uint64_t fresh_bits; + uint64_t count; public: template @@ -93,5 +95,44 @@ namespace snmalloc { bit_source = get_next(); } + + /** + * Pseudo random bit source. + * + * Does not cycle as frequently as `next_bit`. + */ + uint16_t next_fresh_bits(size_t n) + { + if (count <= n) + { + fresh_bits = get_next(); + count = 64; + } + uint16_t result = + static_cast(fresh_bits & (bits::one_at_bit(n) - 1)); + fresh_bits >>= n; + count -= n; + return result; + } + + /*** + * Approximation of a uniform distribution + * + * Biases high numbers. A proper uniform distribution + * was too expensive. This maps a uniform distribution + * over the next power of two (2^m), and for numbers that + * are drawn larger then n-1, they are mapped onto uniform + * top range of n. + */ + uint16_t sample(uint16_t n) + { + size_t i = bits::next_pow2_bits(n); + uint16_t bits = next_fresh_bits(i); + uint16_t result = bits; + // Put over flowing bits at the top. + if (bits >= n) + result = n - (1 + bits - n); + return result; + } }; } // namespace snmalloc \ No newline at end of file diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 1af77ac..1f9035d 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -251,23 +251,28 @@ namespace snmalloc * The fields are paired up to give better codegen as then they are offset * by a power of 2, and the bit extract from the interleaving seed can * be shifted to calculate the relevant offset to index the fields. + * + * If RANDOM is set to false, then the code does not perform any + * randomisation. */ - template + template class FreeListBuilder { + static constexpr size_t LENGTH = RANDOM ? 2 : 1; + // Pointer to the first element. - EncodeFreeObjectReference head[2]; + EncodeFreeObjectReference head[LENGTH]; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - EncodeFreeObjectReference* end[2]; + EncodeFreeObjectReference* end[LENGTH]; #ifdef CHECK_CLIENT // The bottom 16 bits of the previous pointer - uint16_t prev[2]; + uint16_t prev[LENGTH]; // The bottom 16 bits of the current pointer // This needs to be stored for the empty case // where it is `initial_key()` for the slab. - uint16_t curr[2]; + uint16_t curr[LENGTH]; #endif public: S s; @@ -307,16 +312,16 @@ namespace snmalloc void open(void* p) { SNMALLOC_ASSERT(empty()); + for (size_t i = 0; i < LENGTH; i++) + { #ifdef CHECK_CLIENT - prev[0] = HEAD_KEY; - curr[0] = initial_key(p) & 0xffff; - prev[1] = HEAD_KEY; - curr[1] = initial_key(p) & 0xffff; + prev[i] = HEAD_KEY; + curr[i] = initial_key(p) & 0xffff; #else - UNUSED(p); + UNUSED(p); #endif - end[0] = &head[0]; - end[1] = &head[1]; + end[i] = &head[i]; + } } /** @@ -324,7 +329,22 @@ namespace snmalloc */ bool empty() { - return end[0] == &head[0] && end[1] == &head[1]; + for (size_t i = 0; i < LENGTH; i++) + { + if (end[i] != &head[i]) + return false; + } + return true; + } + + bool debug_different_slab(void* n) + { + for (size_t i = 0; i < LENGTH; i++) + { + if (!different_slab(end[i], n)) + return false; + } + return true; } /** @@ -332,11 +352,10 @@ namespace snmalloc */ void add(void* n, LocalEntropy& entropy) { - SNMALLOC_ASSERT( - !different_slab(end[0], n) || !different_slab(end[1], n) || empty()); + SNMALLOC_ASSERT(!debug_different_slab(n) || empty()); FreeObject* next = FreeObject::make(n); - auto index = entropy.next_bit(); + auto index = RANDOM ? entropy.next_bit() : 0; end[index]->store(next, get_prev(index), entropy); end[index] = &(next->next_object); @@ -355,7 +374,7 @@ namespace snmalloc size_t debug_length(LocalEntropy& entropy) { size_t count = 0; - for (size_t i = 0; i < 2; i++) + for (size_t i = 0; i < LENGTH; i++) { uint16_t local_prev = HEAD_KEY; EncodeFreeObjectReference* iter = &head[i]; @@ -392,53 +411,60 @@ namespace snmalloc */ FreeListIter terminate(LocalEntropy& entropy, bool preserve_queue = true) { - SNMALLOC_ASSERT(end[1] != &head[0]); - SNMALLOC_ASSERT(end[0] != &head[1]); - - // If second list is empty, then append is trivial. - if (end[1] == &head[1]) + if constexpr (RANDOM) { - end[0]->store(nullptr, get_prev(0), entropy); - return {head[0].read(HEAD_KEY, entropy)}; - } + SNMALLOC_ASSERT(end[1] != &head[0]); + SNMALLOC_ASSERT(end[0] != &head[1]); - end[1]->store(nullptr, get_prev(1), entropy); + // If second list is non-empty, perform append. + if (end[1] != &head[1]) + { + end[1]->store(nullptr, get_prev(1), entropy); - // Append 1 to 0 - auto mid = head[1].read(HEAD_KEY, entropy); - end[0]->store(mid, get_prev(0), entropy); - // Re-code first link in second list (if there is one). - // The first link in the second list will be encoded with initial_key, - // But that needs to be changed to the curr of the first list. - if (mid != nullptr) - { - auto mid_next = mid->read_next(initial_key(mid) & 0xffff, entropy); - mid->next_object.store(mid_next, get_curr(0), entropy); - } + // Append 1 to 0 + auto mid = head[1].read(HEAD_KEY, entropy); + end[0]->store(mid, get_prev(0), entropy); + // Re-code first link in second list (if there is one). + // The first link in the second list will be encoded with initial_key, + // But that needs to be changed to the curr of the first list. + if (mid != nullptr) + { + auto mid_next = mid->read_next(initial_key(mid) & 0xffff, entropy); + mid->next_object.store(mid_next, get_curr(0), entropy); + } - auto h = head[0].read(HEAD_KEY, entropy); + auto h = head[0].read(HEAD_KEY, entropy); - // If we need to continue adding to the builder - // Set up the second list as empty, - // and extend the first list to cover all of the second. - if (preserve_queue && h != nullptr) - { + // If we need to continue adding to the builder + // Set up the second list as empty, + // and extend the first list to cover all of the second. + if (preserve_queue && h != nullptr) + { #ifdef CHECK_CLIENT - prev[0] = prev[1]; - curr[0] = curr[1]; + prev[0] = prev[1]; + curr[0] = curr[1]; #endif - end[0] = end[1]; + end[0] = end[1]; #ifdef CHECK_CLIENT - prev[1] = HEAD_KEY; - curr[1] = initial_key(h) & 0xffff; + prev[1] = HEAD_KEY; + curr[1] = initial_key(h) & 0xffff; #endif - end[1] = &(head[1]); + end[1] = &(head[1]); + } + + SNMALLOC_ASSERT(end[1] != &head[0]); + SNMALLOC_ASSERT(end[0] != &head[1]); + + return {h}; + } + } + else + { + UNUSED(preserve_queue); } - SNMALLOC_ASSERT(end[1] != &head[0]); - SNMALLOC_ASSERT(end[0] != &head[1]); - - return {h}; + end[0]->store(nullptr, get_prev(0), entropy); + return {head[0].read(HEAD_KEY, entropy)}; } /** @@ -456,8 +482,10 @@ namespace snmalloc */ void init() { - end[0] = &head[0]; - end[1] = &head[1]; + for (size_t i = 0; i < LENGTH; i++) + { + end[i] = &head[i]; + } } }; } // namespace snmalloc diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index bd90c53..b815978 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -50,7 +50,11 @@ namespace snmalloc * * Spare 32bits are used for the fields in MetaslabEnd. */ - FreeListBuilder free_queue; +#ifdef CHECK_CLIENT + FreeListBuilder free_queue; +#else + FreeListBuilder free_queue; +#endif uint16_t& needed() { diff --git a/src/mem/slab.h b/src/mem/slab.h index d117c14..c0ac680 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -37,24 +37,54 @@ namespace snmalloc { void* slab_end = pointer_align_up(pointer_offset(bumpptr, 1)); - FreeListBuilder b; + FreeListBuilder b; SNMALLOC_ASSERT(b.empty()); b.open(bumpptr); - // This code needs generalising, but currently applies - // various offsets with a stride of seven to increase chance of catching - // accidental OOB write. - std::array start_index = {3, 5, 0, 2, 4, 1, 6}; - for (size_t offset : start_index) +#ifdef CHECK_CLIENT + // Structure to represent the temporary list elements + struct PreAllocObject { - void* newbumpptr = pointer_offset(bumpptr, rsize * offset); - while (newbumpptr < slab_end) - { - b.add(newbumpptr, entropy); - newbumpptr = pointer_offset(newbumpptr, rsize * start_index.size()); - } + PreAllocObject* next; + }; + // The following code implements Sattolo's algorithm for generating + // random cyclic permutations. This implementation is in the opposite + // direction, so that the original space does not need initialising. This + // is described as outside-in without citation on Wikipedia, appears to be + // Folklore algorithm. + PreAllocObject* curr = pointer_offset(bumpptr, 0); + curr->next = curr; + uint16_t count = 1; + for (PreAllocObject* p = pointer_offset(bumpptr, rsize); + p < slab_end; + p = pointer_offset(p, rsize)) + { + size_t insert_index = entropy.sample(count); + p->next = std::exchange( + pointer_offset(bumpptr, insert_index * rsize)->next, + p); + count++; } + + // Pick entry into space, and then build linked list by traversing cycle + // to the start. + auto start_index = entropy.sample(count); + auto start_ptr = + pointer_offset(bumpptr, start_index * rsize); + auto curr_ptr = start_ptr; + do + { + b.add(curr_ptr, entropy); + curr_ptr = curr_ptr->next; + } while (curr_ptr != start_ptr); +#else + for (void* p = bumpptr; p < slab_end; p = pointer_offset(p, rsize)) + { + b.add(p, entropy); + } +#endif + // This code consumes everything up to slab_end. bumpptr = slab_end; SNMALLOC_ASSERT(!b.empty());