From f15dc6ee2ed69d03060f93e1c9bb2fa285138a73 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 26 Mar 2021 16:22:32 +0000 Subject: [PATCH] Expose Entropy Define various parts of random that can be used to make the layout of memory more random. Thread this through the allocator. Expose the concept as part of the Pal. Subsequent commits will expose that on different platforms. --- src/mem/alloc.h | 24 ++++++----- src/mem/chunkmap.h | 2 +- src/mem/entropy.h | 96 ++++++++++++++++++++++++++++++++++++++++++++ src/mem/freelist.h | 82 ++++++++++++++----------------------- src/mem/metaslab.h | 22 ++++++---- src/mem/slab.h | 33 +++++++++------ src/pal/pal.h | 6 --- src/pal/pal_consts.h | 10 +++++ 8 files changed, 186 insertions(+), 89 deletions(-) create mode 100644 src/mem/entropy.h diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 231c4de..e563784 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -98,6 +98,7 @@ namespace snmalloc { LargeAlloc large_allocator; ChunkMap chunk_map; + LocalEntropy entropy; /** * Per size class bumpptr for building new free lists @@ -683,6 +684,11 @@ namespace snmalloc if (isFake) return; + // Entropy must be first, so that all data-structures can use the key + // it generates. + // This must occur before any freelists are constructed. + entropy.init(); + init_message_queue(); message_queue().invariant(); @@ -745,10 +751,10 @@ namespace snmalloc Slab* slab = Metaslab::get_slab(bp); while (pointer_align_up(bp, SLAB_SIZE) != bp) { - Slab::alloc_new_list(bp, ffl, rsize); + Slab::alloc_new_list(bp, ffl, rsize, entropy); while (!ffl.empty()) { - small_dealloc_offseted_inner(super, slab, ffl.take(), i); + small_dealloc_offseted_inner(super, slab, ffl.take(entropy), i); } } } @@ -762,7 +768,7 @@ namespace snmalloc auto slab = Metaslab::get_slab(head); do { - auto curr = small_fast_free_lists[i].take(); + auto curr = small_fast_free_lists[i].take(entropy); small_dealloc_offseted_inner(super, slab, curr, i); } while (!small_fast_free_lists[i].empty()); @@ -1016,7 +1022,7 @@ namespace snmalloc { stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); - void* p = remove_cache_friendly_offset(fl.take(), sizeclass); + void* p = remove_cache_friendly_offset(fl.take(entropy), sizeclass); if constexpr (zero_mem == YesZero) { MemoryProvider::Pal::zero(p, sizeclass_to_size(sizeclass)); @@ -1061,7 +1067,7 @@ namespace snmalloc auto meta = reinterpret_cast(sl.get_next()); auto& ffl = small_fast_free_lists[sizeclass]; return Metaslab::alloc( - meta, ffl, rsize); + meta, ffl, rsize, entropy); } return small_alloc_rare(sizeclass, size); } @@ -1125,9 +1131,9 @@ namespace snmalloc auto rsize = sizeclass_to_size(sizeclass); auto& ffl = small_fast_free_lists[sizeclass]; SNMALLOC_ASSERT(ffl.empty()); - Slab::alloc_new_list(bp, ffl, rsize); + Slab::alloc_new_list(bp, ffl, rsize, entropy); - auto p = remove_cache_friendly_offset(ffl.take(), sizeclass); + auto p = remove_cache_friendly_offset(ffl.take(entropy), sizeclass); if constexpr (zero_mem == YesZero) { @@ -1213,7 +1219,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass) { - if (likely(Slab::dealloc_fast(slab, super, p))) + if (likely(Slab::dealloc_fast(slab, super, p, entropy))) return; small_dealloc_offseted_slow(super, slab, p, sizeclass); @@ -1224,7 +1230,7 @@ namespace snmalloc { bool was_full = super->is_full(); SlabList* sl = &small_classes[sizeclass]; - Superslab::Action a = Slab::dealloc_slow(slab, sl, super, p); + Superslab::Action a = Slab::dealloc_slow(slab, sl, super, p, entropy); if (likely(a == Superslab::NoSlabReturn)) return; stats().sizeclass_dealloc_slab(sizeclass); diff --git a/src/mem/chunkmap.h b/src/mem/chunkmap.h index 37a35db..1924d02 100644 --- a/src/mem/chunkmap.h +++ b/src/mem/chunkmap.h @@ -72,7 +72,7 @@ namespace snmalloc * internal tree node in the non-flat pagemap). */ # define SNMALLOC_MAX_FLATPAGEMAP_SIZE \ - (pal_supports ? 256ULL * 1024 * 1024 : PAGEMAP_NODE_SIZE) + (pal_supports ? 256ULL * 1024 * 1024 : PAGEMAP_NODE_SIZE) #endif static constexpr bool CHUNKMAP_USE_FLATPAGEMAP = SNMALLOC_MAX_FLATPAGEMAP_SIZE >= diff --git a/src/mem/entropy.h b/src/mem/entropy.h new file mode 100644 index 0000000..359e185 --- /dev/null +++ b/src/mem/entropy.h @@ -0,0 +1,96 @@ +#include "../ds/address.h" + +#include +#include +#include + +namespace snmalloc +{ + template + std::enable_if_t, uint64_t> get_entropy64() + { + return PAL::get_entropy64(); + } + + template + std::enable_if_t, uint64_t> get_entropy64() + { + std::random_device rd; + uint64_t a = rd(); + return (a << 32) ^ rd(); + } + + class LocalEntropy + { + uint64_t bit_source; + uint64_t local_key; + uint64_t local_counter; + address_t constant_key; + + public: + template + void init() + { + local_key = get_entropy64(); + local_counter = get_entropy64(); + if constexpr (bits::BITS == 64) + constant_key = get_next(); + else + constant_key = get_next() & 0xffff'ffff; + bit_source = get_next(); + } + + /** + * Returns a bit. + * + * The bit returned is cycled every 64 calls. + * This is a very cheap source of some randomness. + * Returns the bottom bit. + */ + uint32_t next_bit() + { + uint64_t bottom_bit = bit_source & 1; + bit_source = (bottom_bit << 63) | (bit_source >> 1); + return bottom_bit & 1; + } + + /** + * A key that is not changed or used to create other keys + * + * This is for use when there is no storage for the key. + */ + address_t get_constant_key() + { + return constant_key; + } + + /** + * Source of random 64bit values + * + * Has a 2^64 period. + * + * Applies a Feistel cipher to a counter + */ + uint64_t get_next() + { + uint64_t c = ++local_counter; + uint64_t bottom; + for (int i = 0; i < 2; i++) + { + bottom = c & 0xffff'fffff; + c = (c << 32) | (((bottom * local_key) ^ c) >> 32); + } + return c; + } + + /** + * Refresh `next_bit` source of bits. + * + * This loads new entropy into the `next_bit` values. + */ + void refresh_bits() + { + bit_source = get_next(); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 5feed0c..1af77ac 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -9,6 +9,7 @@ #include "../ds/dllist.h" #include "../ds/helpers.h" #include "allocconfig.h" +#include "entropy.h" #include @@ -16,13 +17,6 @@ namespace snmalloc { #ifdef CHECK_CLIENT static constexpr std::size_t PRESERVE_BOTTOM_BITS = 16; - - /** - * The key that is used to encode free list pointers. - * This should be randomised at startup in the future. - */ - inline static address_t global_key = static_cast( - bits::is64() ? 0x5a59'DEAD'BEEF'5A59 : 0x5A59'BEEF); #endif /** @@ -85,7 +79,7 @@ namespace snmalloc #ifdef CHECK_CLIENT template static std::enable_if_t - encode(uint16_t local_key, T* next_object) + encode(uint16_t local_key, T* next_object, LocalEntropy& entropy) { // Simple involutional encoding. The bottom half of each word is // multiplied by a function of both global and local keys (the latter, @@ -95,7 +89,7 @@ namespace snmalloc auto next = address_cast(next_object); constexpr address_t MASK = bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1; // Mix in local_key - address_t key = (local_key + 1) * global_key; + address_t key = (local_key + 1) * entropy.get_constant_key(); next ^= (((next & MASK) + 1) * key) & ~(bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1); return reinterpret_cast(next); @@ -104,20 +98,21 @@ namespace snmalloc template static std::enable_if_t - encode(uint16_t local_key, T* next_object) + encode(uint16_t local_key, T* next_object, LocalEntropy& entropy) { UNUSED(local_key); + UNUSED(entropy); return next_object; } - void store(FreeObject* value, uint16_t local_key) + void store(FreeObject* value, uint16_t local_key, LocalEntropy& entropy) { - reference = encode(local_key, value); + reference = encode(local_key, value, entropy); } - FreeObject* read(uint16_t local_key) + FreeObject* read(uint16_t local_key, LocalEntropy& entropy) { - return encode(local_key, reference); + return encode(local_key, reference, entropy); } }; @@ -142,9 +137,9 @@ namespace snmalloc /** * Read the next pointer handling any required decoding of the pointer */ - FreeObject* read_next(uint16_t key) + FreeObject* read_next(uint16_t key, LocalEntropy& entropy) { - return next_object.read(key); + return next_object.read(key, entropy); } }; @@ -223,14 +218,14 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - void* take() + void* take(LocalEntropy& entropy) { #ifdef CHECK_CLIENT check_client( !different_slab(prev, curr), "Heap corruption - free list corrupted!"); #endif auto c = curr; - update_cursor(curr->read_next(get_prev())); + update_cursor(curr->read_next(get_prev(), entropy)); return c; } }; @@ -266,7 +261,6 @@ namespace snmalloc // In the empty case end[i] == &head[i] // This enables branch free enqueuing. EncodeFreeObjectReference* end[2]; - uint32_t interleave; #ifdef CHECK_CLIENT // The bottom 16 bits of the previous pointer uint16_t prev[2]; @@ -300,18 +294,6 @@ namespace snmalloc static constexpr uint16_t HEAD_KEY = 1; - /** - * Rotate the bits for interleaving. - * - * Returns the bottom bit. - */ - uint32_t next_interleave() - { - uint32_t bottom_bit = interleave & 1; - interleave = (bottom_bit << 31) | (interleave >> 1); - return bottom_bit; - } - public: FreeListBuilder() { @@ -324,8 +306,6 @@ namespace snmalloc */ void open(void* p) { - interleave = 0xDEADBEEF; // TODO RANDOM - SNMALLOC_ASSERT(empty()); #ifdef CHECK_CLIENT prev[0] = HEAD_KEY; @@ -337,8 +317,6 @@ namespace snmalloc #endif end[0] = &head[0]; end[1] = &head[1]; - - SNMALLOC_ASSERT(debug_length() == 0); } /** @@ -352,15 +330,15 @@ namespace snmalloc /** * Adds an element to the builder */ - void add(void* n) + void add(void* n, LocalEntropy& entropy) { SNMALLOC_ASSERT( !different_slab(end[0], n) || !different_slab(end[1], n) || empty()); FreeObject* next = FreeObject::make(n); - uint32_t index = next_interleave(); + auto index = entropy.next_bit(); - end[index]->store(next, get_prev(index)); + end[index]->store(next, get_prev(index), entropy); end[index] = &(next->next_object); #ifdef CHECK_CLIENT prev[index] = curr[index]; @@ -374,18 +352,18 @@ namespace snmalloc * If this is needed in a non-debug setting then * we should look at redesigning the queue. */ - size_t debug_length() + size_t debug_length(LocalEntropy& entropy) { size_t count = 0; for (size_t i = 0; i < 2; i++) { uint16_t local_prev = HEAD_KEY; EncodeFreeObjectReference* iter = &head[i]; - FreeObject* prev_obj = iter->read(local_prev); + FreeObject* prev_obj = iter->read(local_prev, entropy); uint16_t local_curr = initial_key(prev_obj) & 0xffff; while (end[i] != iter) { - FreeObject* next = iter->read(local_prev); + FreeObject* next = iter->read(local_prev, entropy); check_client(!different_slab(next, prev_obj), "Heap corruption"); local_prev = local_curr; local_curr = address_cast(next) & 0xffff; @@ -412,7 +390,7 @@ namespace snmalloc * * It is used with preserve_queue disabled by close. */ - FreeListIter terminate(bool preserve_queue = true) + FreeListIter terminate(LocalEntropy& entropy, bool preserve_queue = true) { SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); @@ -420,25 +398,25 @@ namespace snmalloc // If second list is empty, then append is trivial. if (end[1] == &head[1]) { - end[0]->store(nullptr, get_prev(0)); - return {head[0].read(HEAD_KEY)}; + end[0]->store(nullptr, get_prev(0), entropy); + return {head[0].read(HEAD_KEY, entropy)}; } - end[1]->store(nullptr, get_prev(1)); + end[1]->store(nullptr, get_prev(1), entropy); // Append 1 to 0 - auto mid = head[1].read(HEAD_KEY); - end[0]->store(mid, get_prev(0)); + auto mid = head[1].read(HEAD_KEY, entropy); + end[0]->store(mid, get_prev(0), entropy); // Re-code first link in second list (if there is one). // The first link in the second list will be encoded with initial_key, // But that needs to be changed to the curr of the first list. if (mid != nullptr) { - auto mid_next = mid->read_next(initial_key(mid) & 0xffff); - mid->next_object.store(mid_next, get_curr(0)); + auto mid_next = mid->read_next(initial_key(mid) & 0xffff, entropy); + mid->next_object.store(mid_next, get_curr(0), entropy); } - auto h = head[0].read(HEAD_KEY); + auto h = head[0].read(HEAD_KEY, entropy); // If we need to continue adding to the builder // Set up the second list as empty, @@ -467,9 +445,9 @@ namespace snmalloc * Close a free list, and set the iterator parameter * to iterate it. */ - void close(FreeListIter& dst) + void close(FreeListIter& dst, LocalEntropy& entropy) { - dst = terminate(false); + dst = terminate(entropy, false); init(); } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 6d8ff23..d5c60c9 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -160,14 +160,19 @@ namespace snmalloc * eventually annotate that pointer with additional information. */ template - static SNMALLOC_FAST_PATH void* - alloc(Metaslab* self, FreeListIter& fast_free_list, size_t rsize) + static SNMALLOC_FAST_PATH void* alloc( + Metaslab* self, + FreeListIter& fast_free_list, + size_t rsize, + LocalEntropy& entropy) { SNMALLOC_ASSERT(rsize == sizeclass_to_size(self->sizeclass())); SNMALLOC_ASSERT(!self->is_full()); - self->free_queue.close(fast_free_list); - void* n = fast_free_list.take(); + self->free_queue.close(fast_free_list, entropy); + void* n = fast_free_list.take(entropy); + + entropy.refresh_bits(); // Treat stealing the free list as allocating it all. self->remove(); @@ -176,7 +181,7 @@ namespace snmalloc void* p = remove_cache_friendly_offset(n, self->sizeclass()); SNMALLOC_ASSERT(is_start_of_object(self, p)); - self->debug_slab_invariant(Metaslab::get_slab(p)); + self->debug_slab_invariant(Metaslab::get_slab(p), entropy); if constexpr (zero_mem == YesZero) { @@ -193,14 +198,14 @@ namespace snmalloc return p; } - void debug_slab_invariant(Slab* slab) + void debug_slab_invariant(Slab* slab, LocalEntropy& entropy) { #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) bool is_short = Metaslab::is_short(slab); if (is_full()) { - size_t count = free_queue.debug_length(); + size_t count = free_queue.debug_length(entropy); SNMALLOC_ASSERT(count < threshold_for_waking_slab(is_short)); return; } @@ -216,7 +221,7 @@ namespace snmalloc SNMALLOC_ASSERT(SLAB_SIZE > accounted_for); // Account for list size - size_t count = free_queue.debug_length(); + size_t count = free_queue.debug_length(entropy); accounted_for += count * size; SNMALLOC_ASSERT(count <= get_slab_capacity(sizeclass(), is_short)); @@ -234,6 +239,7 @@ namespace snmalloc SNMALLOC_ASSERT(SLAB_SIZE == accounted_for); #else UNUSED(slab); + UNUSED(entropy); #endif } }; diff --git a/src/mem/slab.h b/src/mem/slab.h index b37eedc..d117c14 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -29,8 +29,11 @@ namespace snmalloc * worth of allocations, or one if the allocation size is larger than a * page. */ - static SNMALLOC_FAST_PATH void - alloc_new_list(void*& bumpptr, FreeListIter& fast_free_list, size_t rsize) + static SNMALLOC_FAST_PATH void alloc_new_list( + void*& bumpptr, + FreeListIter& fast_free_list, + size_t rsize, + LocalEntropy& entropy) { void* slab_end = pointer_align_up(pointer_offset(bumpptr, 1)); @@ -48,14 +51,14 @@ namespace snmalloc void* newbumpptr = pointer_offset(bumpptr, rsize * offset); while (newbumpptr < slab_end) { - b.add(newbumpptr); + b.add(newbumpptr, entropy); newbumpptr = pointer_offset(newbumpptr, rsize * start_index.size()); } } bumpptr = slab_end; SNMALLOC_ASSERT(!b.empty()); - b.close(fast_free_list); + b.close(fast_free_list, entropy); } // Returns true, if it deallocation can proceed without changing any status @@ -65,7 +68,7 @@ namespace snmalloc // This is pre-factored to take an explicit self parameter so that we can // eventually annotate that pointer with additional information. static SNMALLOC_FAST_PATH bool - dealloc_fast(Slab* self, Superslab* super, void* p) + dealloc_fast(Slab* self, Superslab* super, void* p, LocalEntropy& entropy) { Metaslab& meta = super->get_meta(self); SNMALLOC_ASSERT(!meta.is_unused()); @@ -74,7 +77,7 @@ namespace snmalloc return false; // Update the head and the next pointer in the free list. - meta.free_queue.add(p); + meta.free_queue.add(p, entropy); return true; } @@ -86,11 +89,15 @@ namespace snmalloc // // This is pre-factored to take an explicit self parameter so that we can // eventually annotate that pointer with additional information. - static SNMALLOC_SLOW_PATH typename Superslab::Action - dealloc_slow(Slab* self, SlabList* sl, Superslab* super, void* p) + static SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow( + Slab* self, + SlabList* sl, + Superslab* super, + void* p, + LocalEntropy& entropy) { Metaslab& meta = super->get_meta(self); - meta.debug_slab_invariant(self); + meta.debug_slab_invariant(self, entropy); if (meta.is_full()) { @@ -106,7 +113,7 @@ namespace snmalloc return super->dealloc_slab(self); } - meta.free_queue.add(p); + meta.free_queue.add(p, entropy); // Remove trigger threshold from how many we need before we have fully // freed the slab. meta.needed() = @@ -114,7 +121,7 @@ namespace snmalloc // Push on the list of slabs for this sizeclass. sl->insert_prev(&meta); - meta.debug_slab_invariant(self); + meta.debug_slab_invariant(self, entropy); return Superslab::NoSlabReturn; } @@ -123,11 +130,11 @@ namespace snmalloc // Check free list is well-formed on platforms with // integers as pointers. FreeListIter fl; - meta.free_queue.close(fl); + meta.free_queue.close(fl, entropy); while (!fl.empty()) { - fl.take(); + fl.take(entropy); count++; } #endif diff --git a/src/pal/pal.h b/src/pal/pal.h index b112c94..2f58c7c 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -66,12 +66,6 @@ namespace snmalloc Pal::error(str); } - /** - * Query whether the PAL supports a specific feature. - */ - template - constexpr static bool pal_supports = (PAL::pal_features & F) == F; - // Used to keep Superslab metadata committed. static constexpr size_t OS_PAGE_SIZE = Pal::page_size; diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 58e6454..90ad321 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -41,6 +41,10 @@ namespace snmalloc * should be pre-allocated. */ NoAllocation = (1 << 3), + /** + * This Pal provides a source of Entropy + */ + Entropy = (1 << 4), }; /** * Flag indicating whether requested memory should be zeroed. @@ -125,4 +129,10 @@ namespace snmalloc } } }; + + /** + * Query whether the PAL supports a specific feature. + */ + template + constexpr static bool pal_supports = (PAL::pal_features & F) == F; } // namespace snmalloc