diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 231c4de..e563784 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -98,6 +98,7 @@ namespace snmalloc { LargeAlloc large_allocator; ChunkMap chunk_map; + LocalEntropy entropy; /** * Per size class bumpptr for building new free lists @@ -683,6 +684,11 @@ namespace snmalloc if (isFake) return; + // Entropy must be first, so that all data-structures can use the key + // it generates. + // This must occur before any freelists are constructed. + entropy.init(); + init_message_queue(); message_queue().invariant(); @@ -745,10 +751,10 @@ namespace snmalloc Slab* slab = Metaslab::get_slab(bp); while (pointer_align_up(bp, SLAB_SIZE) != bp) { - Slab::alloc_new_list(bp, ffl, rsize); + Slab::alloc_new_list(bp, ffl, rsize, entropy); while (!ffl.empty()) { - small_dealloc_offseted_inner(super, slab, ffl.take(), i); + small_dealloc_offseted_inner(super, slab, ffl.take(entropy), i); } } } @@ -762,7 +768,7 @@ namespace snmalloc auto slab = Metaslab::get_slab(head); do { - auto curr = small_fast_free_lists[i].take(); + auto curr = small_fast_free_lists[i].take(entropy); small_dealloc_offseted_inner(super, slab, curr, i); } while (!small_fast_free_lists[i].empty()); @@ -1016,7 +1022,7 @@ namespace snmalloc { stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); - void* p = remove_cache_friendly_offset(fl.take(), sizeclass); + void* p = remove_cache_friendly_offset(fl.take(entropy), sizeclass); if constexpr (zero_mem == YesZero) { MemoryProvider::Pal::zero(p, sizeclass_to_size(sizeclass)); @@ -1061,7 +1067,7 @@ namespace snmalloc auto meta = reinterpret_cast(sl.get_next()); auto& ffl = small_fast_free_lists[sizeclass]; return Metaslab::alloc( - meta, ffl, rsize); + meta, ffl, rsize, entropy); } return small_alloc_rare(sizeclass, size); } @@ -1125,9 +1131,9 @@ namespace snmalloc auto rsize = sizeclass_to_size(sizeclass); auto& ffl = small_fast_free_lists[sizeclass]; SNMALLOC_ASSERT(ffl.empty()); - Slab::alloc_new_list(bp, ffl, rsize); + Slab::alloc_new_list(bp, ffl, rsize, entropy); - auto p = remove_cache_friendly_offset(ffl.take(), sizeclass); + auto p = remove_cache_friendly_offset(ffl.take(entropy), sizeclass); if constexpr (zero_mem == YesZero) { @@ -1213,7 +1219,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass) { - if (likely(Slab::dealloc_fast(slab, super, p))) + if (likely(Slab::dealloc_fast(slab, super, p, entropy))) return; small_dealloc_offseted_slow(super, slab, p, sizeclass); @@ -1224,7 +1230,7 @@ namespace snmalloc { bool was_full = super->is_full(); SlabList* sl = &small_classes[sizeclass]; - Superslab::Action a = Slab::dealloc_slow(slab, sl, super, p); + Superslab::Action a = Slab::dealloc_slow(slab, sl, super, p, entropy); if (likely(a == Superslab::NoSlabReturn)) return; stats().sizeclass_dealloc_slab(sizeclass); diff --git a/src/mem/chunkmap.h b/src/mem/chunkmap.h index 37a35db..1924d02 100644 --- a/src/mem/chunkmap.h +++ b/src/mem/chunkmap.h @@ -72,7 +72,7 @@ namespace snmalloc * internal tree node in the non-flat pagemap). */ # define SNMALLOC_MAX_FLATPAGEMAP_SIZE \ - (pal_supports ? 256ULL * 1024 * 1024 : PAGEMAP_NODE_SIZE) + (pal_supports ? 256ULL * 1024 * 1024 : PAGEMAP_NODE_SIZE) #endif static constexpr bool CHUNKMAP_USE_FLATPAGEMAP = SNMALLOC_MAX_FLATPAGEMAP_SIZE >= diff --git a/src/mem/entropy.h b/src/mem/entropy.h new file mode 100644 index 0000000..359e185 --- /dev/null +++ b/src/mem/entropy.h @@ -0,0 +1,96 @@ +#include "../ds/address.h" + +#include +#include +#include + +namespace snmalloc +{ + template + std::enable_if_t, uint64_t> get_entropy64() + { + return PAL::get_entropy64(); + } + + template + std::enable_if_t, uint64_t> get_entropy64() + { + std::random_device rd; + uint64_t a = rd(); + return (a << 32) ^ rd(); + } + + class LocalEntropy + { + uint64_t bit_source; + uint64_t local_key; + uint64_t local_counter; + address_t constant_key; + + public: + template + void init() + { + local_key = get_entropy64(); + local_counter = get_entropy64(); + if constexpr (bits::BITS == 64) + constant_key = get_next(); + else + constant_key = get_next() & 0xffff'ffff; + bit_source = get_next(); + } + + /** + * Returns a bit. + * + * The bit returned is cycled every 64 calls. + * This is a very cheap source of some randomness. + * Returns the bottom bit. + */ + uint32_t next_bit() + { + uint64_t bottom_bit = bit_source & 1; + bit_source = (bottom_bit << 63) | (bit_source >> 1); + return bottom_bit & 1; + } + + /** + * A key that is not changed or used to create other keys + * + * This is for use when there is no storage for the key. + */ + address_t get_constant_key() + { + return constant_key; + } + + /** + * Source of random 64bit values + * + * Has a 2^64 period. + * + * Applies a Feistel cipher to a counter + */ + uint64_t get_next() + { + uint64_t c = ++local_counter; + uint64_t bottom; + for (int i = 0; i < 2; i++) + { + bottom = c & 0xffff'fffff; + c = (c << 32) | (((bottom * local_key) ^ c) >> 32); + } + return c; + } + + /** + * Refresh `next_bit` source of bits. + * + * This loads new entropy into the `next_bit` values. + */ + void refresh_bits() + { + bit_source = get_next(); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 5feed0c..1af77ac 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -9,6 +9,7 @@ #include "../ds/dllist.h" #include "../ds/helpers.h" #include "allocconfig.h" +#include "entropy.h" #include @@ -16,13 +17,6 @@ namespace snmalloc { #ifdef CHECK_CLIENT static constexpr std::size_t PRESERVE_BOTTOM_BITS = 16; - - /** - * The key that is used to encode free list pointers. - * This should be randomised at startup in the future. - */ - inline static address_t global_key = static_cast( - bits::is64() ? 0x5a59'DEAD'BEEF'5A59 : 0x5A59'BEEF); #endif /** @@ -85,7 +79,7 @@ namespace snmalloc #ifdef CHECK_CLIENT template static std::enable_if_t - encode(uint16_t local_key, T* next_object) + encode(uint16_t local_key, T* next_object, LocalEntropy& entropy) { // Simple involutional encoding. The bottom half of each word is // multiplied by a function of both global and local keys (the latter, @@ -95,7 +89,7 @@ namespace snmalloc auto next = address_cast(next_object); constexpr address_t MASK = bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1; // Mix in local_key - address_t key = (local_key + 1) * global_key; + address_t key = (local_key + 1) * entropy.get_constant_key(); next ^= (((next & MASK) + 1) * key) & ~(bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1); return reinterpret_cast(next); @@ -104,20 +98,21 @@ namespace snmalloc template static std::enable_if_t - encode(uint16_t local_key, T* next_object) + encode(uint16_t local_key, T* next_object, LocalEntropy& entropy) { UNUSED(local_key); + UNUSED(entropy); return next_object; } - void store(FreeObject* value, uint16_t local_key) + void store(FreeObject* value, uint16_t local_key, LocalEntropy& entropy) { - reference = encode(local_key, value); + reference = encode(local_key, value, entropy); } - FreeObject* read(uint16_t local_key) + FreeObject* read(uint16_t local_key, LocalEntropy& entropy) { - return encode(local_key, reference); + return encode(local_key, reference, entropy); } }; @@ -142,9 +137,9 @@ namespace snmalloc /** * Read the next pointer handling any required decoding of the pointer */ - FreeObject* read_next(uint16_t key) + FreeObject* read_next(uint16_t key, LocalEntropy& entropy) { - return next_object.read(key); + return next_object.read(key, entropy); } }; @@ -223,14 +218,14 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - void* take() + void* take(LocalEntropy& entropy) { #ifdef CHECK_CLIENT check_client( !different_slab(prev, curr), "Heap corruption - free list corrupted!"); #endif auto c = curr; - update_cursor(curr->read_next(get_prev())); + update_cursor(curr->read_next(get_prev(), entropy)); return c; } }; @@ -266,7 +261,6 @@ namespace snmalloc // In the empty case end[i] == &head[i] // This enables branch free enqueuing. EncodeFreeObjectReference* end[2]; - uint32_t interleave; #ifdef CHECK_CLIENT // The bottom 16 bits of the previous pointer uint16_t prev[2]; @@ -300,18 +294,6 @@ namespace snmalloc static constexpr uint16_t HEAD_KEY = 1; - /** - * Rotate the bits for interleaving. - * - * Returns the bottom bit. - */ - uint32_t next_interleave() - { - uint32_t bottom_bit = interleave & 1; - interleave = (bottom_bit << 31) | (interleave >> 1); - return bottom_bit; - } - public: FreeListBuilder() { @@ -324,8 +306,6 @@ namespace snmalloc */ void open(void* p) { - interleave = 0xDEADBEEF; // TODO RANDOM - SNMALLOC_ASSERT(empty()); #ifdef CHECK_CLIENT prev[0] = HEAD_KEY; @@ -337,8 +317,6 @@ namespace snmalloc #endif end[0] = &head[0]; end[1] = &head[1]; - - SNMALLOC_ASSERT(debug_length() == 0); } /** @@ -352,15 +330,15 @@ namespace snmalloc /** * Adds an element to the builder */ - void add(void* n) + void add(void* n, LocalEntropy& entropy) { SNMALLOC_ASSERT( !different_slab(end[0], n) || !different_slab(end[1], n) || empty()); FreeObject* next = FreeObject::make(n); - uint32_t index = next_interleave(); + auto index = entropy.next_bit(); - end[index]->store(next, get_prev(index)); + end[index]->store(next, get_prev(index), entropy); end[index] = &(next->next_object); #ifdef CHECK_CLIENT prev[index] = curr[index]; @@ -374,18 +352,18 @@ namespace snmalloc * If this is needed in a non-debug setting then * we should look at redesigning the queue. */ - size_t debug_length() + size_t debug_length(LocalEntropy& entropy) { size_t count = 0; for (size_t i = 0; i < 2; i++) { uint16_t local_prev = HEAD_KEY; EncodeFreeObjectReference* iter = &head[i]; - FreeObject* prev_obj = iter->read(local_prev); + FreeObject* prev_obj = iter->read(local_prev, entropy); uint16_t local_curr = initial_key(prev_obj) & 0xffff; while (end[i] != iter) { - FreeObject* next = iter->read(local_prev); + FreeObject* next = iter->read(local_prev, entropy); check_client(!different_slab(next, prev_obj), "Heap corruption"); local_prev = local_curr; local_curr = address_cast(next) & 0xffff; @@ -412,7 +390,7 @@ namespace snmalloc * * It is used with preserve_queue disabled by close. */ - FreeListIter terminate(bool preserve_queue = true) + FreeListIter terminate(LocalEntropy& entropy, bool preserve_queue = true) { SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); @@ -420,25 +398,25 @@ namespace snmalloc // If second list is empty, then append is trivial. if (end[1] == &head[1]) { - end[0]->store(nullptr, get_prev(0)); - return {head[0].read(HEAD_KEY)}; + end[0]->store(nullptr, get_prev(0), entropy); + return {head[0].read(HEAD_KEY, entropy)}; } - end[1]->store(nullptr, get_prev(1)); + end[1]->store(nullptr, get_prev(1), entropy); // Append 1 to 0 - auto mid = head[1].read(HEAD_KEY); - end[0]->store(mid, get_prev(0)); + auto mid = head[1].read(HEAD_KEY, entropy); + end[0]->store(mid, get_prev(0), entropy); // Re-code first link in second list (if there is one). // The first link in the second list will be encoded with initial_key, // But that needs to be changed to the curr of the first list. if (mid != nullptr) { - auto mid_next = mid->read_next(initial_key(mid) & 0xffff); - mid->next_object.store(mid_next, get_curr(0)); + auto mid_next = mid->read_next(initial_key(mid) & 0xffff, entropy); + mid->next_object.store(mid_next, get_curr(0), entropy); } - auto h = head[0].read(HEAD_KEY); + auto h = head[0].read(HEAD_KEY, entropy); // If we need to continue adding to the builder // Set up the second list as empty, @@ -467,9 +445,9 @@ namespace snmalloc * Close a free list, and set the iterator parameter * to iterate it. */ - void close(FreeListIter& dst) + void close(FreeListIter& dst, LocalEntropy& entropy) { - dst = terminate(false); + dst = terminate(entropy, false); init(); } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 6d8ff23..d5c60c9 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -160,14 +160,19 @@ namespace snmalloc * eventually annotate that pointer with additional information. */ template - static SNMALLOC_FAST_PATH void* - alloc(Metaslab* self, FreeListIter& fast_free_list, size_t rsize) + static SNMALLOC_FAST_PATH void* alloc( + Metaslab* self, + FreeListIter& fast_free_list, + size_t rsize, + LocalEntropy& entropy) { SNMALLOC_ASSERT(rsize == sizeclass_to_size(self->sizeclass())); SNMALLOC_ASSERT(!self->is_full()); - self->free_queue.close(fast_free_list); - void* n = fast_free_list.take(); + self->free_queue.close(fast_free_list, entropy); + void* n = fast_free_list.take(entropy); + + entropy.refresh_bits(); // Treat stealing the free list as allocating it all. self->remove(); @@ -176,7 +181,7 @@ namespace snmalloc void* p = remove_cache_friendly_offset(n, self->sizeclass()); SNMALLOC_ASSERT(is_start_of_object(self, p)); - self->debug_slab_invariant(Metaslab::get_slab(p)); + self->debug_slab_invariant(Metaslab::get_slab(p), entropy); if constexpr (zero_mem == YesZero) { @@ -193,14 +198,14 @@ namespace snmalloc return p; } - void debug_slab_invariant(Slab* slab) + void debug_slab_invariant(Slab* slab, LocalEntropy& entropy) { #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) bool is_short = Metaslab::is_short(slab); if (is_full()) { - size_t count = free_queue.debug_length(); + size_t count = free_queue.debug_length(entropy); SNMALLOC_ASSERT(count < threshold_for_waking_slab(is_short)); return; } @@ -216,7 +221,7 @@ namespace snmalloc SNMALLOC_ASSERT(SLAB_SIZE > accounted_for); // Account for list size - size_t count = free_queue.debug_length(); + size_t count = free_queue.debug_length(entropy); accounted_for += count * size; SNMALLOC_ASSERT(count <= get_slab_capacity(sizeclass(), is_short)); @@ -234,6 +239,7 @@ namespace snmalloc SNMALLOC_ASSERT(SLAB_SIZE == accounted_for); #else UNUSED(slab); + UNUSED(entropy); #endif } }; diff --git a/src/mem/slab.h b/src/mem/slab.h index b37eedc..d117c14 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -29,8 +29,11 @@ namespace snmalloc * worth of allocations, or one if the allocation size is larger than a * page. */ - static SNMALLOC_FAST_PATH void - alloc_new_list(void*& bumpptr, FreeListIter& fast_free_list, size_t rsize) + static SNMALLOC_FAST_PATH void alloc_new_list( + void*& bumpptr, + FreeListIter& fast_free_list, + size_t rsize, + LocalEntropy& entropy) { void* slab_end = pointer_align_up(pointer_offset(bumpptr, 1)); @@ -48,14 +51,14 @@ namespace snmalloc void* newbumpptr = pointer_offset(bumpptr, rsize * offset); while (newbumpptr < slab_end) { - b.add(newbumpptr); + b.add(newbumpptr, entropy); newbumpptr = pointer_offset(newbumpptr, rsize * start_index.size()); } } bumpptr = slab_end; SNMALLOC_ASSERT(!b.empty()); - b.close(fast_free_list); + b.close(fast_free_list, entropy); } // Returns true, if it deallocation can proceed without changing any status @@ -65,7 +68,7 @@ namespace snmalloc // This is pre-factored to take an explicit self parameter so that we can // eventually annotate that pointer with additional information. static SNMALLOC_FAST_PATH bool - dealloc_fast(Slab* self, Superslab* super, void* p) + dealloc_fast(Slab* self, Superslab* super, void* p, LocalEntropy& entropy) { Metaslab& meta = super->get_meta(self); SNMALLOC_ASSERT(!meta.is_unused()); @@ -74,7 +77,7 @@ namespace snmalloc return false; // Update the head and the next pointer in the free list. - meta.free_queue.add(p); + meta.free_queue.add(p, entropy); return true; } @@ -86,11 +89,15 @@ namespace snmalloc // // This is pre-factored to take an explicit self parameter so that we can // eventually annotate that pointer with additional information. - static SNMALLOC_SLOW_PATH typename Superslab::Action - dealloc_slow(Slab* self, SlabList* sl, Superslab* super, void* p) + static SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow( + Slab* self, + SlabList* sl, + Superslab* super, + void* p, + LocalEntropy& entropy) { Metaslab& meta = super->get_meta(self); - meta.debug_slab_invariant(self); + meta.debug_slab_invariant(self, entropy); if (meta.is_full()) { @@ -106,7 +113,7 @@ namespace snmalloc return super->dealloc_slab(self); } - meta.free_queue.add(p); + meta.free_queue.add(p, entropy); // Remove trigger threshold from how many we need before we have fully // freed the slab. meta.needed() = @@ -114,7 +121,7 @@ namespace snmalloc // Push on the list of slabs for this sizeclass. sl->insert_prev(&meta); - meta.debug_slab_invariant(self); + meta.debug_slab_invariant(self, entropy); return Superslab::NoSlabReturn; } @@ -123,11 +130,11 @@ namespace snmalloc // Check free list is well-formed on platforms with // integers as pointers. FreeListIter fl; - meta.free_queue.close(fl); + meta.free_queue.close(fl, entropy); while (!fl.empty()) { - fl.take(); + fl.take(entropy); count++; } #endif diff --git a/src/pal/pal.h b/src/pal/pal.h index b112c94..2f58c7c 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -66,12 +66,6 @@ namespace snmalloc Pal::error(str); } - /** - * Query whether the PAL supports a specific feature. - */ - template - constexpr static bool pal_supports = (PAL::pal_features & F) == F; - // Used to keep Superslab metadata committed. static constexpr size_t OS_PAGE_SIZE = Pal::page_size; diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 58e6454..90ad321 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -41,6 +41,10 @@ namespace snmalloc * should be pre-allocated. */ NoAllocation = (1 << 3), + /** + * This Pal provides a source of Entropy + */ + Entropy = (1 << 4), }; /** * Flag indicating whether requested memory should be zeroed. @@ -125,4 +129,10 @@ namespace snmalloc } } }; + + /** + * Query whether the PAL supports a specific feature. + */ + template + constexpr static bool pal_supports = (PAL::pal_features & F) == F; } // namespace snmalloc