From 81bf3417327d4ee2e39037bd5af7365942c2eacd Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 26 Jul 2021 09:56:48 +0100 Subject: [PATCH] XOR encoded next_object This commit adds a simple XOR encoding to the next_object pointer in FreeObjects. This removes the trivial way of getting hold of a physical address from the system by observing the free list pointers in deallocated objects. --- src/ds/defines.h | 11 +++- src/mem/corealloc.h | 8 +-- src/mem/entropy.h | 31 +++++++--- src/mem/freelist.h | 123 +++++++++++++++++++++++++------------- src/mem/globalconfig.h | 6 +- src/mem/localcache.h | 4 +- src/mem/metaslab.h | 2 +- src/mem/remoteallocator.h | 10 ++-- src/mem/remotecache.h | 6 +- 9 files changed, 136 insertions(+), 65 deletions(-) diff --git a/src/ds/defines.h b/src/ds/defines.h index f5d6646..37095f8 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -130,4 +130,13 @@ check_client_impl(bool test, const char* const str) # define check_client(test, str) check_client_impl(test, str) #else # define check_client(test, str) -#endif \ No newline at end of file +#endif + +namespace snmalloc +{ +#ifdef SNMALLOC_CHECK_CLIENT + static constexpr bool CHECK_CLIENT = true; +#else + static constexpr bool CHECK_CLIENT = false; +#endif +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index fb4ff91..96490a7 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -141,7 +141,7 @@ namespace snmalloc { auto slab_end = pointer_offset(bumpptr, slab_size + 1 - rsize); - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); FreeListBuilder b; SNMALLOC_ASSERT(b.empty()); @@ -209,7 +209,7 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); FreeListIter fl; meta->free_queue.close(fl, key); void* p = finish_alloc_no_zero(fl.take(key), sizeclass); @@ -475,7 +475,7 @@ namespace snmalloc auto cp = CapPtr(reinterpret_cast(p)); - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); // Update the head and the next pointer in the free list. meta->free_queue.add(cp, key, entropy); @@ -538,7 +538,7 @@ namespace snmalloc // Set meta slab to empty. meta->initialise(sizeclass); - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); // take an allocation from the free list auto p = fast_free_list.take(key); diff --git a/src/mem/entropy.h b/src/mem/entropy.h index cf5f179..9a56cb2 100644 --- a/src/mem/entropy.h +++ b/src/mem/entropy.h @@ -29,14 +29,25 @@ namespace snmalloc #endif } + struct FreeListKey + { + address_t key; + address_t key_next; + + constexpr FreeListKey(uint64_t key, uint64_t key_next) + : key(static_cast(key)), + key_next(static_cast(key_next)) + {} + }; + class LocalEntropy { uint64_t bit_source{0}; uint64_t local_key{0}; uint64_t local_counter{0}; - address_t constant_key{0}; uint64_t fresh_bits{0}; uint64_t count{0}; + FreeListKey key{0, 0}; public: constexpr LocalEntropy() = default; @@ -47,9 +58,15 @@ namespace snmalloc local_key = get_entropy64(); local_counter = get_entropy64(); if constexpr (bits::BITS == 64) - constant_key = get_next(); + { + key.key = get_next(); + key.key_next = get_next(); + } else - constant_key = get_next() & 0xffff'ffff; + { + key.key = get_next() & 0xffff'ffff; + key.key_next = get_next() & 0xffff'ffff; + } bit_source = get_next(); } @@ -68,13 +85,11 @@ namespace snmalloc } /** - * A key that is not changed or used to create other keys - * - * This is for use when there is no storage for the key. + * A key for the free lists for this thread. */ - address_t get_constant_key() + const FreeListKey& get_free_list_key() { - return constant_key; + return key; } /** diff --git a/src/mem/freelist.h b/src/mem/freelist.h index e6a173b..31500bb 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -41,19 +41,6 @@ namespace snmalloc { - struct FreeListKey - { - address_t key; - - FreeListKey(uint64_t key_) - { - if constexpr (bits::BITS == 64) - key = static_cast(key_); - else - key = key_ & 0xffff'ffff; - } - }; - /** * This function is used to sign back pointers in the free list. * @@ -64,7 +51,7 @@ namespace snmalloc * list. */ inline static uintptr_t - signed_prev(address_t curr, address_t next, FreeListKey& key) + signed_prev(address_t curr, address_t next, const FreeListKey& key) { auto c = curr; auto n = next; @@ -103,6 +90,30 @@ namespace snmalloc return p.template as_static(); } + /** + * Encode next + */ + inline static CapPtr encode_next( + address_t curr, CapPtr next, const FreeListKey& key) + { + // Note we can consider other encoding schemes here. + // * XORing curr and next. This doesn't require any key material + // * XORing (curr * key). This makes it harder to guess the underlying + // key, as each location effectively has its own key. + // Curr is not used in the current encoding scheme. + UNUSED(curr); + + if constexpr (CHECK_CLIENT && !aal_supports) + { + return CapPtr(address_cast(next) ^ key.key_next); + } + else + { + UNUSED(key); + return next; + } + } + /** * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT. * Static so that it can be used on reference to a FreeObject. @@ -114,7 +125,7 @@ namespace snmalloc static CapPtr* store_next( CapPtr* curr, CapPtr next, - FreeListKey& key) + const FreeListKey& key) { #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = @@ -122,16 +133,23 @@ namespace snmalloc #else UNUSED(key); #endif - *curr = next; + *curr = encode_next(address_cast(curr), next, key); return &(next->next_object); } + static void + store_null(CapPtr* curr, const FreeListKey& key) + { + *curr = encode_next(address_cast(curr), nullptr, key); + } + /** * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT * * Uses the atomic view of next, so can be used in the message queues. */ - void atomic_store_next(CapPtr next, FreeListKey& key) + void + atomic_store_next(CapPtr next, const FreeListKey& key) { #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = @@ -141,17 +159,24 @@ namespace snmalloc #endif // Signature needs to be visible before item is linked in // so requires release semantics. - atomic_next_object.store(next, std::memory_order_release); + atomic_next_object.store( + encode_next(address_cast(&next_object), next, key), + std::memory_order_release); } - void atomic_store_null() + void atomic_store_null(const FreeListKey& key) { - atomic_next_object.store(nullptr, std::memory_order_relaxed); + atomic_next_object.store( + encode_next(address_cast(&next_object), nullptr, key), + std::memory_order_relaxed); } - CapPtr atomic_read_next(FreeListKey& key) + CapPtr atomic_read_next(const FreeListKey& key) { - auto n = atomic_next_object.load(std::memory_order_acquire); + auto n = encode_next( + address_cast(&next_object), + atomic_next_object.load(std::memory_order_acquire), + key); #ifdef SNMALLOC_CHECK_CLIENT if (n != nullptr) { @@ -176,9 +201,9 @@ namespace snmalloc /** * Read the next pointer */ - CapPtr read_next() + CapPtr read_next(const FreeListKey& key) { - return next_object; + return encode_next(address_cast(&next_object), next_object, key); } }; @@ -230,10 +255,10 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - CapPtr take(FreeListKey& key) + CapPtr take(const FreeListKey& key) { auto c = curr; - auto next = curr->read_next(); + auto next = curr->read_next(key); Aal::prefetch(next.unsafe_ptr()); curr = next; @@ -306,8 +331,10 @@ namespace snmalloc /** * Adds an element to the builder */ - void - add(CapPtr n, FreeListKey& key, LocalEntropy& entropy) + void add( + CapPtr n, + const FreeListKey& key, + LocalEntropy& entropy) { uint32_t index; if constexpr (RANDOM) @@ -328,7 +355,7 @@ namespace snmalloc */ template std::enable_if_t - add(CapPtr n, FreeListKey& key) + add(CapPtr n, const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); end[0] = FreeObject::store_next(end[0], n, key); @@ -337,23 +364,39 @@ namespace snmalloc /** * Makes a terminator to a free list. */ - SNMALLOC_FAST_PATH void terminate_list(uint32_t index, FreeListKey& key) + SNMALLOC_FAST_PATH void + terminate_list(uint32_t index, const FreeListKey& key) { - UNUSED(key); - *end[index] = nullptr; + FreeObject::store_null(end[index], key); } - address_t get_fake_signed_prev(uint32_t index, FreeListKey& key) + /** + * Read head removing potential encoding + * + * Although, head does not require meta-data protection + * as it is not stored in an object allocation. For uniformity + * it is treated like the next_object field in a FreeObject + * and is thus subject to encoding if the next_object pointers + * encoded. + */ + CapPtr + read_head(uint32_t index, const FreeListKey& key) + { + return FreeObject::encode_next( + address_cast(&head[index]), head[index], key); + } + + address_t get_fake_signed_prev(uint32_t index, const FreeListKey& key) { return signed_prev( - address_cast(&head[index]), address_cast(head[index]), key); + address_cast(&head[index]), address_cast(read_head(index, key)), key); } /** * Close a free list, and set the iterator parameter * to iterate it. */ - SNMALLOC_FAST_PATH void close(FreeListIter& fl, FreeListKey& key) + SNMALLOC_FAST_PATH void close(FreeListIter& fl, const FreeListKey& key) { if constexpr (RANDOM) { @@ -365,12 +408,12 @@ namespace snmalloc { // The start token has been corrupted. // TOCTTOU issue, but small window here. - head[1]->check_prev(get_fake_signed_prev(1, key)); + read_head(1, key)->check_prev(get_fake_signed_prev(1, key)); terminate_list(1, key); // Append 1 to 0 - FreeObject::store_next(end[0], head[1], key); + FreeObject::store_next(end[0], read_head(1, key), key); SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); @@ -385,7 +428,7 @@ namespace snmalloc terminate_list(0, key); } - fl = {head[0], get_fake_signed_prev(0, key)}; + fl = {read_head(0, key), get_fake_signed_prev(0, key)}; init(); } @@ -401,12 +444,12 @@ namespace snmalloc } std::pair, CapPtr> - extract_segment() + extract_segment(const FreeListKey& key) { SNMALLOC_ASSERT(!empty()); SNMALLOC_ASSERT(!RANDOM); // TODO: Turn this into a static failure. - auto first = head[0]; + auto first = read_head(0, key); // end[0] is pointing to the first field in the object, // this is doing a CONTAINING_RECORD like cast to get back // to the actual object. This isn't true if the builder is diff --git a/src/mem/globalconfig.h b/src/mem/globalconfig.h index 981f09a..cc5dc76 100644 --- a/src/mem/globalconfig.h +++ b/src/mem/globalconfig.h @@ -6,6 +6,8 @@ #include "../mem/slaballocator.h" #include "commonconfig.h" +#include + namespace snmalloc { // Forward reference to thread local cleanup. @@ -73,8 +75,10 @@ namespace snmalloc if (initialised) return; + LocalEntropy entropy; + entropy.init(); // Initialise key for remote deallocation lists - key_global = FreeListKey(get_entropy64()); + key_global = FreeListKey(entropy.get_free_list_key()); // Need to initialise pagemap. backend_state.init(); diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 2070966..ae871c9 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -76,7 +76,7 @@ namespace snmalloc typename SharedStateHandle> bool flush(DeallocFun dealloc, SharedStateHandle handle) { - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); // Return all the free lists to the allocator. // Used during thread teardown @@ -98,7 +98,7 @@ namespace snmalloc template SNMALLOC_FAST_PATH void* alloc(size_t size, Slowpath slowpath) { - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); sizeclass_t sizeclass = size_to_sizeclass(size); stats.alloc_request(size); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index d62db6a..2c88528 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -142,7 +142,7 @@ namespace snmalloc LocalEntropy& entropy, sizeclass_t sizeclass) { - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); FreeListIter tmp_fl; meta->free_queue.close(tmp_fl, key); diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 5b70ac1..6e8ea47 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -27,7 +27,7 @@ namespace snmalloc /** * Global key for all remote lists. */ - inline static FreeListKey key_global(0xdeadbeef); + inline static FreeListKey key_global(0xdeadbeef, 0xdeadbeef); struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator { @@ -50,7 +50,7 @@ namespace snmalloc void init(CapPtr stub) { - stub->atomic_store_null(); + stub->atomic_store_null(key_global); front = stub; back.store(stub, std::memory_order_relaxed); invariant(); @@ -74,12 +74,12 @@ namespace snmalloc void enqueue( CapPtr first, CapPtr last, - FreeListKey& key) + const FreeListKey& key) { // Pushes a list of messages to the queue. Each message from first to // last should be linked together through their next pointers. invariant(); - last->atomic_store_null(); + last->atomic_store_null(key); // exchange needs to be a release, so nullptr in next is visible. CapPtr prev = @@ -93,7 +93,7 @@ namespace snmalloc return front; } - std::pair, bool> dequeue(FreeListKey& key) + std::pair, bool> dequeue(const FreeListKey& key) { // Returns the front message, or null if not possible to return a message. invariant(); diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index af71c22..25680b0 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -67,7 +67,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc( RemoteAllocator::alloc_id_t target_id, CapPtr p, - FreeListKey& key) + const FreeListKey& key) { SNMALLOC_ASSERT(initialised); auto r = p.template as_reinterpret(); @@ -79,7 +79,7 @@ namespace snmalloc bool post( SharedStateHandle handle, RemoteAllocator::alloc_id_t id, - FreeListKey& key) + const FreeListKey& key) { SNMALLOC_ASSERT(initialised); size_t post_round = 0; @@ -96,7 +96,7 @@ namespace snmalloc if (!list[i].empty()) { - auto [first, last] = list[i].extract_segment(); + auto [first, last] = list[i].extract_segment(key); MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(first)); entry.get_remote()->enqueue(first, last, key);