diff --git a/CMakeLists.txt b/CMakeLists.txt index 675d661..004f629 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -266,9 +266,11 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) if(NOT WIN32) set(SHARED_FILES src/override/new.cc src/override/malloc.cc) add_shim(snmallocshim SHARED ${SHARED_FILES}) + add_shim(snmallocshim-checks SHARED ${SHARED_FILES}) add_shim(snmallocshim-1mib SHARED ${SHARED_FILES}) add_shim(snmallocshim-16mib SHARED ${SHARED_FILES}) target_compile_definitions(snmallocshim-16mib PRIVATE SNMALLOC_USE_LARGE_CHUNKS) + target_compile_definitions(snmallocshim-checks PRIVATE CHECK_CLIENT) # Build a shim with some settings from oe. add_shim(snmallocshim-oe SHARED ${SHARED_FILES}) oe_simulate(snmallocshim-oe) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index c44330a..e4e47d8 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -47,7 +47,7 @@ namespace snmalloc class FastFreeLists { protected: - FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES]; + FreeListIter small_fast_free_lists[NUM_SMALL_CLASSES]; public: FastFreeLists() : small_fast_free_lists() {} @@ -747,35 +747,29 @@ namespace snmalloc { auto& bp = bump_ptrs[i]; auto rsize = sizeclass_to_size(i); - FreeListHead ffl; + FreeListIter ffl; while (pointer_align_up(bp, SLAB_SIZE) != bp) { Slab::alloc_new_list(bp, ffl, rsize); - SlabNext* prev = ffl.value; - while (prev != nullptr) + while (!ffl.empty()) { - auto n = Metaslab::follow_next(prev); - Superslab* super = Superslab::get(prev); - Slab* slab = Metaslab::get_slab(prev); - small_dealloc_offseted_inner(super, slab, prev, i); - prev = n; + auto curr = ffl.take(); + Superslab* super = Superslab::get(curr); + Slab* slab = Metaslab::get_slab(curr); + small_dealloc_offseted_inner(super, slab, curr, i); } } } for (size_t i = 0; i < NUM_SMALL_CLASSES; i++) { - auto prev = small_fast_free_lists[i].value; - small_fast_free_lists[i].value = nullptr; - while (prev != nullptr) + while (!small_fast_free_lists[i].empty()) { - auto n = Metaslab::follow_next(prev); + auto curr = small_fast_free_lists[i].take(); - Superslab* super = Superslab::get(prev); - Slab* slab = Metaslab::get_slab(prev); - small_dealloc_offseted_inner(super, slab, prev, i); - - prev = n; + Superslab* super = Superslab::get(curr); + Slab* slab = Metaslab::get_slab(curr); + small_dealloc_offseted_inner(super, slab, curr, i); } test(small_classes[i]); @@ -1035,15 +1029,11 @@ namespace snmalloc { SNMALLOC_ASSUME(sizeclass < NUM_SMALL_CLASSES); auto& fl = small_fast_free_lists[sizeclass]; - SlabNext* head = fl.value; - if (likely(head != nullptr)) + if (likely(!fl.empty())) { stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); - // Read the next slot from the memory that's about to be allocated. - fl.value = Metaslab::follow_next(head); - - void* p = remove_cache_friendly_offset(head, sizeclass); + void* p = remove_cache_friendly_offset(fl.take(), sizeclass); if constexpr (zero_mem == YesZero) { MemoryProvider::Pal::zero(p, sizeclass_to_size(sizeclass)); @@ -1154,12 +1144,10 @@ namespace snmalloc auto& bp = bump_ptrs[sizeclass]; auto rsize = sizeclass_to_size(sizeclass); auto& ffl = small_fast_free_lists[sizeclass]; - SNMALLOC_ASSERT(ffl.value == nullptr); + SNMALLOC_ASSERT(ffl.empty()); Slab::alloc_new_list(bp, ffl, rsize); - SlabNext* p = static_cast( - remove_cache_friendly_offset(ffl.value, sizeclass)); - ffl.value = Metaslab::follow_next(p); + auto p = remove_cache_friendly_offset(ffl.take(), sizeclass); if constexpr (zero_mem == YesZero) { @@ -1181,7 +1169,7 @@ namespace snmalloc Slab* slab = alloc_slab(sizeclass); if (slab == nullptr) return nullptr; - bp = pointer_offset( + bp = pointer_offset( slab, get_initial_offset(sizeclass, Metaslab::is_short(slab))); return small_alloc_build_free_list(sizeclass); diff --git a/src/mem/freelist.h b/src/mem/freelist.h new file mode 100644 index 0000000..773759b --- /dev/null +++ b/src/mem/freelist.h @@ -0,0 +1,317 @@ +#pragma once +/** + * This file encapsulates the in disused object free lists + * that are used per slab of small objects. + */ + +#include "../ds/address.h" +#include "../ds/cdllist.h" +#include "../ds/dllist.h" +#include "../ds/helpers.h" +#include "allocconfig.h" + +#include + +namespace snmalloc +{ +#ifdef CHECK_CLIENT + /** + * The key that is used to encode free list pointers. + * This should be randomised at startup in the future. + */ + inline static address_t global_key = + static_cast(bits::is64() ? 0x9999'9999'9999'9999 : 0x9999'9999); + + /** + * Used to turn a location into a key. This is currently + * just the value of the previous location + 1. + */ + inline static uintptr_t initial_key(void* p) + { + return address_cast(p) + 1; + } +#endif + + static inline bool different_slab(uintptr_t p1, uintptr_t p2) + { + return ((p1 ^ p2) >= SLAB_SIZE); + } + + static inline bool different_slab(uintptr_t p1, void* p2) + { + return different_slab(p1, address_cast(p2)); + } + + static inline bool different_slab(void* p1, void* p2) + { + return different_slab(address_cast(p1), address_cast(p2)); + } + + /** + * Free objects within each slab point directly to the next. + * The next_object pointer can be encoded to detect + * corruption caused by writes in a UAF or a double free. + * + * If cache-friendly offsets are used, then the FreeObject is + * potentially offset from the start of the object. + */ + class FreeObject + { + FreeObject* next_object; + + static FreeObject* encode(uintptr_t local_key, FreeObject* next_object) + { +#ifdef CHECK_CLIENT + if constexpr (aal_supports) + { + // Simple involutional encoding. The bottom half of each word is + // multiplied by a function of both global and local keys (the latter, + // in practice, being the address of the previous list entry) and the + // resulting word's top half is XORed into the pointer value before it + // is stored. + auto next = address_cast(next_object); + constexpr uintptr_t MASK = bits::one_at_bit(bits::BITS / 2) - 1; + // Mix in local_key + auto key = local_key ^ global_key; + next ^= (((next & MASK) + 1) * key) & ~MASK; + next_object = reinterpret_cast(next); + } +#else + UNUSED(local_key); +#endif + return next_object; + } + + public: + static FreeObject* make(void* p) + { + return static_cast(p); + } + + /** + * Read the next pointer handling any required decoding of the pointer + */ + FreeObject* read_next(uintptr_t key) + { + auto next = encode(key, next_object); + return next; + } + + /** + * Store the next pointer handling any required encoding of the pointer + */ + void store_next(FreeObject* next, uintptr_t key) + { + next_object = encode(key, next); + SNMALLOC_ASSERT(next == read_next(key)); + } + }; + + /** + * Wrapper class that allows the keys for pointer encoding to be + * conditionally compiled. + */ + class FreeObjectCursor + { + FreeObject* curr = nullptr; +#ifdef CHECK_CLIENT + uintptr_t prev = 0; +#endif + + uintptr_t get_prev() + { +#ifdef CHECK_CLIENT + return prev; +#else + return 0; +#endif + } + + /** + * Updates the cursor to the new value, + * importantly this updates the key being used. + * Currently this is just the value of current before this call. + * Other schemes could be used. + */ + void update_cursor(FreeObject* next) + { +#ifdef CHECK_CLIENT +# ifndef NDEBUG + if (next != nullptr) + { + if (unlikely(different_slab(prev, next))) + { + error("Heap corruption - free list corrupted!"); + } + } +# endif + prev = address_cast(curr); +#endif + curr = next; + } + + public: + FreeObject* get_curr() + { + return curr; + } + + /** + * Advance the cursor through the list + */ + void move_next() + { +#ifdef CHECK_CLIENT + if (unlikely(different_slab(prev, curr))) + { + error("Heap corruption - free list corrupted!"); + } +#endif + update_cursor(curr->read_next(get_prev())); + } + + /** + * Update the next pointer at the location in the list pointed to + * by the cursor. + */ + void set_next(FreeObject* next) + { + curr->store_next(next, get_prev()); + } + + /** + * Update the next pointer at the location in the list pointed to + * by the cursor, and move the cursor to that new value. + */ + void set_next_and_move(FreeObject* next) + { + set_next(next); + update_cursor(next); + } + + /** + * Resets the key to an initial value. So the cursor can be used + * on a new sequence. + */ + void reset_cursor(FreeObject* next) + { +#ifdef CHECK_CLIENT + prev = initial_key(next); +#endif + curr = next; + } + }; + + /** + * Used to iterate a free list in object space. + * + * Checks signing of pointers + */ + class FreeListIter + { + protected: + FreeObjectCursor front; + + public: + /** + * Checks if there are any more values to iterate. + */ + bool empty() + { + return front.get_curr() == nullptr; + } + + /** + * Moves the iterator on, and returns the current value. + */ + void* take() + { + auto c = front.get_curr(); + front.move_next(); + return c; + } + }; + + /** + * Used to build a free list in object space. + * + * Checks signing of pointers + */ + class FreeListBuilder : FreeListIter + { + FreeObjectCursor end; + + public: + /** + * Start building a new free list. + */ + void open(void* n) + { + SNMALLOC_ASSERT(empty()); + FreeObject* next = FreeObject::make(n); + end.reset_cursor(next); + front.reset_cursor(next); + } + + /** + * Returns current head without affecting the builder. + */ + void* peek_head() + { + return front.get_curr(); + } + + /** + * Checks if there are any more values to iterate. + */ + bool empty() + { + return FreeListIter::empty(); + } + + /** + * Adds an element to the free list + */ + void add(void* n) + { + SNMALLOC_ASSERT(!different_slab(end.get_curr(), n)); + FreeObject* next = FreeObject::make(n); + end.set_next_and_move(next); + } + + /** + * Adds a terminator at the end of a free list, + * but does not close the builder. Thus new elements + * can still be added. It returns a new iterator to the + * list. + * + * This is used to iterate an list that is being constructed. + * It is currently only used to check invariants in Debug builds. + */ + FreeListIter terminate() + { + if (!empty()) + end.set_next(nullptr); + return *this; + } + + /** + * Close a free list, and set the iterator parameter + * to iterate it. + */ + void close(FreeListIter& dst) + { + terminate(); + dst = *this; + init(); + } + + /** + * Set the builder to a not building state. + */ + void init() + { + front.reset_cursor(nullptr); + } + }; +} // namespace snmalloc diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 04a1d38..509604c 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -3,30 +3,13 @@ #include "../ds/cdllist.h" #include "../ds/dllist.h" #include "../ds/helpers.h" +#include "freelist.h" #include "sizeclass.h" namespace snmalloc { class Slab; - /** - * Free objects within each slab point directly to the next (contrast - * SlabLink, which chain different Slabs of the same sizeclass together). - */ - struct SlabNext - { - struct SlabNext* next; -#ifdef CHECK_CLIENT - uintptr_t guard; -#endif - }; - - struct FreeListHead - { - // Use a value with bottom bit set for empty list. - SlabNext* value = nullptr; - }; - using SlabList = CDLLNode<>; using SlabLink = CDLLNode<>; @@ -49,7 +32,7 @@ namespace snmalloc * * The list will be (allocated - needed) long. */ - SlabNext* head = nullptr; + FreeListBuilder free_queue; /** * How many entries are not in the free list of slab, i.e. @@ -90,57 +73,25 @@ namespace snmalloc bool is_full() { auto result = get_prev() == nullptr; - SNMALLOC_ASSERT(!result || head == nullptr); + SNMALLOC_ASSERT(!result || free_queue.empty()); return result; } SNMALLOC_FAST_PATH void set_full() { - SNMALLOC_ASSERT(head == nullptr); + SNMALLOC_ASSERT(free_queue.empty()); // Set needed to 1, so that "return_object" will return true after calling // set_full needed = 1; null_prev(); } - /// Value used to check for corruptions in a block - static constexpr size_t POISON = - static_cast(bits::is64() ? 0xDEADBEEFDEADBEEF : 0xDEADBEEF); - - /// Store next pointer in a block. In Debug using magic value to detect some - /// simple corruptions. - static SNMALLOC_FAST_PATH void store_next(SlabNext* p, SlabNext* head) - { - p->next = head; -#if defined(CHECK_CLIENT) - if constexpr (aal_supports) - { - p->guard = address_cast(head) ^ POISON; - } -#endif - } - - /// Accessor function for the next pointer in a block. - /// In Debug checks for simple corruptions. - static SNMALLOC_FAST_PATH SlabNext* follow_next(SlabNext* node) - { -#if defined(CHECK_CLIENT) - if constexpr (aal_supports) - { - uintptr_t next = address_cast(node->next); - if ((next ^ node->guard) != POISON) - error("Detected memory corruption. Use-after-free."); - } -#endif - return node->next; - } - bool valid_head() { size_t size = sizeclass_to_size(sizeclass); - size_t slab_end = (address_cast(head) | ~SLAB_MASK) + 1; - uintptr_t allocation_start = - remove_cache_friendly_offset(address_cast(head), sizeclass); + auto h = address_cast(free_queue.peek_head()); + address_t slab_end = (h | ~SLAB_MASK) + 1; + address_t allocation_start = remove_cache_friendly_offset(h, sizeclass); return (slab_end - allocation_start) % size == 0; } @@ -155,7 +106,7 @@ namespace snmalloc return pointer_align_down(p) == p; } - static bool is_start_of_object(Metaslab* self, void* p) + SNMALLOC_FAST_PATH static bool is_start_of_object(Metaslab* self, void* p) { return is_multiple_of_sizeclass( sizeclass_to_size(self->sizeclass), @@ -172,26 +123,24 @@ namespace snmalloc */ template static SNMALLOC_FAST_PATH void* - alloc(Metaslab* self, FreeListHead& fast_free_list, size_t rsize) + alloc(Metaslab* self, FreeListIter& fast_free_list, size_t rsize) { SNMALLOC_ASSERT(rsize == sizeclass_to_size(self->sizeclass)); SNMALLOC_ASSERT(!self->is_full()); - auto slab = get_slab(self->head); + auto slab = get_slab(self->free_queue.peek_head()); + self->debug_slab_invariant(slab); - // Use first element as the allocation - SlabNext* h = self->head; - // Put the rest in allocators small_class fast free list. - fast_free_list.value = Metaslab::follow_next(h); - self->head = nullptr; + self->free_queue.close(fast_free_list); + void* n = fast_free_list.take(); // Treat stealing the free list as allocating it all. self->needed = self->allocated; self->remove(); self->set_full(); - void* p = remove_cache_friendly_offset(h, self->sizeclass); + void* p = remove_cache_friendly_offset(n, self->sizeclass); SNMALLOC_ASSERT(is_start_of_object(self, p)); self->debug_slab_invariant(slab); @@ -211,48 +160,6 @@ namespace snmalloc return p; } - /** - * Check bump-free-list-segment for cycles - * - * Using - * https://en.wikipedia.org/wiki/Cycle_detection#Floyd's_Tortoise_and_Hare - * We don't expect a cycle, so worst case is only followed by a crash, so - * slow doesn't mater. - */ - size_t debug_slab_acyclic_free_list(Slab* slab) - { -#ifndef NDEBUG - size_t length = 0; - SlabNext* curr = head; - SlabNext* curr_slow = head; - bool both = false; - while (curr != nullptr) - { - if (get_slab(curr) != slab) - { - error("Free list corruption, not correct slab."); - } - curr = follow_next(curr); - if (both) - { - curr_slow = follow_next(curr_slow); - } - - if (curr == curr_slow) - { - error("Free list contains a cycle, typically indicates double free."); - } - - both = !both; - length++; - } - return length; -#else - UNUSED(slab); - return 0; -#endif - } - void debug_slab_invariant(Slab* slab) { #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) @@ -274,24 +181,18 @@ namespace snmalloc // Block is not full SNMALLOC_ASSERT(SLAB_SIZE > accounted_for); - // Keep variable so it appears in debugger. - size_t length = debug_slab_acyclic_free_list(slab); - UNUSED(length); - // Walk bump-free-list-segment accounting for unused space - SlabNext* curr = head; - while (curr != nullptr) + FreeListIter fl = free_queue.terminate(); + + while (!fl.empty()) { // Check we are looking at a correctly aligned block - void* start = remove_cache_friendly_offset(curr, sizeclass); + void* start = remove_cache_friendly_offset(fl.take(), sizeclass); SNMALLOC_ASSERT(((pointer_diff(slab, start) - offset) % size) == 0); // Account for free elements in free list accounted_for += size; SNMALLOC_ASSERT(SLAB_SIZE >= accounted_for); - - // Iterate bump/free list segment - curr = follow_next(curr); } auto bumpptr = (allocated * size) + offset; diff --git a/src/mem/slab.h b/src/mem/slab.h index ae2f0eb..85f1665 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -1,5 +1,6 @@ #pragma once +#include "freelist.h" #include "superslab.h" namespace snmalloc @@ -27,10 +28,10 @@ namespace snmalloc * page. */ static SNMALLOC_FAST_PATH void - alloc_new_list(void*& bumpptr, FreeListHead& fast_free_list, size_t rsize) + alloc_new_list(void*& bumpptr, FreeListIter& fast_free_list, size_t rsize) { - auto snbumpptr = static_cast(bumpptr); - fast_free_list.value = snbumpptr; + FreeListBuilder b; + b.open(bumpptr); void* newbumpptr = pointer_offset(bumpptr, rsize); void* slab_end = pointer_align_up(newbumpptr); @@ -39,17 +40,14 @@ namespace snmalloc if (slab_end2 < slab_end) slab_end = slab_end2; - while (newbumpptr < slab_end) + bumpptr = newbumpptr; + while (bumpptr < slab_end) { - auto newsnbumpptr = static_cast(newbumpptr); - Metaslab::store_next(snbumpptr, newsnbumpptr); - snbumpptr = newsnbumpptr; - bumpptr = newbumpptr; - newbumpptr = pointer_offset(bumpptr, rsize); + b.add(bumpptr); + bumpptr = pointer_offset(bumpptr, rsize); } - Metaslab::store_next(snbumpptr, nullptr); - bumpptr = newbumpptr; + b.close(fast_free_list); } // Returns true, if it deallocation can proceed without changing any status @@ -71,17 +69,10 @@ namespace snmalloc return false; // Update the head and the next pointer in the free list. - SlabNext* head = meta.head; + meta.free_queue.add(p); - SlabNext* psn = static_cast(p); - - // Set the head to the memory being deallocated. - meta.head = psn; SNMALLOC_ASSERT(meta.valid_head()); - // Set the next pointer to the previous head. - Metaslab::store_next(psn, head); - return true; } @@ -109,10 +100,8 @@ namespace snmalloc return super->dealloc_slab(self); } - SNMALLOC_ASSERT(meta.head == nullptr); - SlabNext* psn = static_cast(p); - meta.head = psn; - Metaslab::store_next(psn, nullptr); + SNMALLOC_ASSERT(meta.free_queue.empty()); + meta.free_queue.open(p); meta.needed = meta.allocated - 1; // Push on the list of slabs for this sizeclass. @@ -121,12 +110,24 @@ namespace snmalloc return Superslab::NoSlabReturn; } - // Remove from the sizeclass list and dealloc on the superslab. +#ifdef CHECK_CLIENT + size_t count = 1; + // Check free list is well-formed on platforms with + // integers as pointers. + FreeListIter fl; + meta.free_queue.close(fl); + + while (!fl.empty()) + { + fl.take(); + count++; + } +#endif + meta.remove(); if (Metaslab::is_short(self)) return super->dealloc_short_slab(); - return super->dealloc_slab(self); } }; diff --git a/src/mem/superslab.h b/src/mem/superslab.h index 61e7f90..4398fa4 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -164,7 +164,7 @@ namespace snmalloc if ((used & 1) == 1) return alloc_slab(sizeclass); - meta[0].head = nullptr; + meta[0].free_queue.init(); // Set up meta data as if the entire slab has been turned into a free // list. This means we don't have to check for special cases where we have // returned all the elements, but this is a slab that is still being bump @@ -188,7 +188,7 @@ namespace snmalloc uint8_t n = meta[h].next; - meta[h].head = nullptr; + meta[h].free_queue.init(); // Set up meta data as if the entire slab has been turned into a free // list. This means we don't have to check for special cases where we have // returned all the elements, but this is a slab that is still being bump