From f7821e11ebe3fc4f8532f7d6d1ff9e8bebaeddce Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Thu, 1 Apr 2021 13:20:17 +0100 Subject: [PATCH] SP: LargeAlloc return CBChunk & chase consequences Even if we opt not to bound these pointers internally (if they aren't headed out to the user program or we later derive bounded pointers), they should still be annotated as something other than CBArena, ensuring that we do not attempt to use them for general amplification. --- src/mem/address_space.h | 41 +++++++------ src/mem/alloc.h | 99 +++++++++++++++++--------------- src/mem/chunkmap.h | 12 ++-- src/mem/freelist.h | 3 +- src/mem/largealloc.h | 22 +++---- src/mem/mediumslab.h | 32 +++++++---- src/mem/metaslab.h | 36 ++++++++---- src/mem/slab.h | 18 ++++-- src/mem/superslab.h | 30 +++++++--- src/test/func/sandbox/sandbox.cc | 8 +-- 10 files changed, 175 insertions(+), 126 deletions(-) diff --git a/src/mem/address_space.h b/src/mem/address_space.h index 81e740e..1553053 100644 --- a/src/mem/address_space.h +++ b/src/mem/address_space.h @@ -35,7 +35,7 @@ namespace snmalloc * bits::BITS is used for simplicity, we do not use below the pointer size, * and large entries will be unlikely to be supported by the platform. */ - std::array, 2>, bits::BITS> ranges = {}; + std::array, 2>, bits::BITS> ranges = {}; /** * This is infrequently used code, a spin lock simplifies the code @@ -46,7 +46,7 @@ namespace snmalloc /** * Checks a block satisfies its invariant. */ - inline void check_block(CapPtr base, size_t align_bits) + inline void check_block(CapPtr base, size_t align_bits) { SNMALLOC_ASSERT( base == pointer_align_up(base, bits::one_at_bit(align_bits))); @@ -59,7 +59,7 @@ namespace snmalloc /** * Adds a block to `ranges`. */ - void add_block(size_t align_bits, CapPtr base) + void add_block(size_t align_bits, CapPtr base) { check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); @@ -74,7 +74,7 @@ namespace snmalloc { // Add to linked list. commit_block(base, sizeof(void*)); - *(base.template as_static>().unsafe_capptr) = + *(base.template as_static>().unsafe_capptr) = ranges[align_bits][1]; check_block(ranges[align_bits][1], align_bits); } @@ -88,9 +88,9 @@ namespace snmalloc * Find a block of the correct size. May split larger blocks * to satisfy this request. */ - CapPtr remove_block(size_t align_bits) + CapPtr remove_block(size_t align_bits) { - CapPtr first = ranges[align_bits][0]; + CapPtr first = ranges[align_bits][0]; if (first == nullptr) { if (align_bits == (bits::BITS - 1)) @@ -100,7 +100,7 @@ namespace snmalloc } // Look for larger block and split up recursively - CapPtr bigger = remove_block(align_bits + 1); + CapPtr bigger = remove_block(align_bits + 1); if (bigger != nullptr) { auto left_over = pointer_offset(bigger, bits::one_at_bit(align_bits)); @@ -111,12 +111,12 @@ namespace snmalloc return bigger; } - CapPtr second = ranges[align_bits][1]; + CapPtr second = ranges[align_bits][1]; if (second != nullptr) { commit_block(second, sizeof(void*)); auto psecond = - second.template as_static>().unsafe_capptr; + second.template as_static>().unsafe_capptr; auto next = *psecond; ranges[align_bits][1] = next; // Zero memory. Client assumes memory contains only zeros. @@ -135,7 +135,7 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - void add_range(CapPtr base, size_t length) + void add_range(CapPtr base, size_t length) { // Find the minimum set of maximally aligned blocks in this range. // Each block's alignment and size are equal. @@ -157,7 +157,7 @@ namespace snmalloc /** * Commit a block of memory */ - void commit_block(CapPtr base, size_t size) + void commit_block(CapPtr base, size_t size) { // Rounding required for sub-page allocations. auto page_start = pointer_align_down(base); @@ -180,7 +180,7 @@ namespace snmalloc * arena_map for use in subsequent amplification. */ template - CapPtr reserve(size_t size, ArenaMap& arena_map) + CapPtr reserve(size_t size, ArenaMap& arena_map) { SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -194,18 +194,18 @@ namespace snmalloc pal_supports && !aal_supports) { if (size >= PAL::minimum_alloc_size) - return CapPtr( + return CapPtr( PAL::template reserve_aligned(size)); } - CapPtr res; + CapPtr res; { FlagLock lock(spin_lock); res = remove_block(bits::next_pow2_bits(size)); if (res == nullptr) { // Allocation failed ask OS for more memory - CapPtr block = nullptr; + CapPtr block = nullptr; size_t block_size = 0; if constexpr (pal_supports) { @@ -233,7 +233,10 @@ namespace snmalloc void* block_raw = PAL::template reserve_aligned(block_size); - block = CapPtr(block_raw); + // It's a bit of a lie to convert without applying bounds, but the + // platform will have bounded block for us and it's better that the + // rest of our internals expect CBChunk bounds. + block = CapPtr(block_raw); if constexpr (aal_supports) { @@ -254,7 +257,7 @@ namespace snmalloc // the PAL, and this could lead to suprious OOM. This is // particularly bad if the PAL gives all the memory on first call. auto block_and_size = PAL::reserve_at_least(size * 2); - block = CapPtr(block_and_size.first); + block = CapPtr(block_and_size.first); block_size = block_and_size.second; // Ensure block is pointer aligned. @@ -294,7 +297,7 @@ namespace snmalloc * used, by smaller objects. */ template - CapPtr + CapPtr reserve_with_left_over(size_t size, ArenaMap& arena_map) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -330,7 +333,7 @@ namespace snmalloc * Constructor that pre-initialises the address-space manager with a region * of memory. */ - AddressSpaceManager(CapPtr base, size_t length) + AddressSpaceManager(CapPtr base, size_t length) { add_range(base, length); } diff --git a/src/mem/alloc.h b/src/mem/alloc.h index d0bc925..5eaaab7 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -638,10 +638,10 @@ namespace snmalloc }; SlabList small_classes[NUM_SMALL_CLASSES]; - DLList medium_classes[NUM_MEDIUM_CLASSES]; + DLList medium_classes[NUM_MEDIUM_CLASSES]; - DLList super_available; - DLList super_only_short_available; + DLList super_available; + DLList super_only_short_available; RemoteCache remote; @@ -901,7 +901,7 @@ namespace snmalloc } SNMALLOC_FAST_PATH void dealloc_not_large_local( - CapPtr super, + CapPtr super, CapPtr p_auth_offseted, sizeclass_t sizeclass) { @@ -963,28 +963,27 @@ namespace snmalloc handle_message_queue_inner(); } - Superslab* get_superslab() + CapPtr get_superslab() { - Superslab* super = super_available.get_head(); + auto super = super_available.get_head(); if (super != nullptr) return super; super = large_allocator .template alloc(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE) - .template as_reinterpret() - .unsafe_capptr; + .template as_reinterpret(); if (super == nullptr) return super; super->init(public_state()); - chunkmap().set_slab(CapPtr(super)); + chunkmap().set_slab(super); super_available.insert(super); return super; } - void reposition_superslab(Superslab* super) + void reposition_superslab(CapPtr super) { switch (super->get_status()) { @@ -1025,11 +1024,12 @@ namespace snmalloc { // Pull a short slab from the list of superslabs that have only the // short slab available. - Superslab* super = super_only_short_available.pop(); + CapPtr super = super_only_short_available.pop(); if (super != nullptr) { - auto slab = Superslab::alloc_short_slab(super, sizeclass); + auto slab = + Superslab::alloc_short_slab(super.unsafe_capptr, sizeclass); SNMALLOC_ASSERT(super->is_full()); return CapPtr(slab); } @@ -1039,17 +1039,17 @@ namespace snmalloc if (super == nullptr) return nullptr; - auto slab = Superslab::alloc_short_slab(super, sizeclass); + auto slab = Superslab::alloc_short_slab(super.unsafe_capptr, sizeclass); reposition_superslab(super); return CapPtr(slab); } - Superslab* super = get_superslab(); + auto super = get_superslab(); if (super == nullptr) return nullptr; - auto slab = Superslab::alloc_slab(super, sizeclass); + auto slab = Superslab::alloc_slab(super.unsafe_capptr, sizeclass); reposition_superslab(super); return CapPtr(slab); } @@ -1233,7 +1233,7 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_unchecked( - CapPtr super, + CapPtr super, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1246,7 +1246,7 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_checked_chunkmap( - CapPtr super, + CapPtr super, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1260,8 +1260,8 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_checked_sizeclass( - CapPtr super, - CapPtr slab, + CapPtr super, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1274,8 +1274,8 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_start( - CapPtr super, - CapPtr slab, + CapPtr super, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1295,8 +1295,8 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_offseted( - CapPtr super, - CapPtr slab, + CapPtr super, + CapPtr slab, CapPtr p, sizeclass_t sizeclass) { @@ -1306,8 +1306,8 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( - CapPtr super, - CapPtr slab, + CapPtr super, + CapPtr slab, CapPtr p, sizeclass_t sizeclass) { @@ -1318,8 +1318,8 @@ namespace snmalloc } SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow( - CapPtr super, - CapPtr slab, + CapPtr super, + CapPtr slab, CapPtr p, sizeclass_t sizeclass) { @@ -1333,6 +1333,8 @@ namespace snmalloc if (a == Superslab::NoStatusChange) return; + auto super_slab = capptr_chunk_from_chunkd(super, SUPERSLAB_SIZE); + switch (super->get_status()) { case Superslab::Full: @@ -1345,29 +1347,29 @@ namespace snmalloc { if (was_full) { - super_available.insert(super.unsafe_capptr); + super_available.insert(super_slab); } else { - super_only_short_available.remove(super.unsafe_capptr); - super_available.insert(super.unsafe_capptr); + super_only_short_available.remove(super_slab); + super_available.insert(super_slab); } break; } case Superslab::OnlyShortSlabAvailable: { - super_only_short_available.insert(super.unsafe_capptr); + super_only_short_available.insert(super_slab); break; } case Superslab::Empty: { - super_available.remove(super.unsafe_capptr); + super_available.remove(super_slab); - chunkmap().clear_slab(super); + chunkmap().clear_slab(super_slab); large_allocator.dealloc( - super.template as_reinterpret(), 0); + super_slab.template as_reinterpret(), 0); stats().superslab_push(); break; } @@ -1381,7 +1383,7 @@ namespace snmalloc sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; auto sc = &medium_classes[medium_class]; - auto slab = sc->get_head(); + CapPtr slab = sc->get_head(); CapPtr p; if (slab != nullptr) @@ -1437,7 +1439,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void medium_dealloc_unchecked( - CapPtr slab, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1451,7 +1453,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void medium_dealloc_checked_chunkmap( - CapPtr slab, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1465,7 +1467,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void medium_dealloc_checked_sizeclass( - CapPtr slab, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1480,7 +1482,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void medium_dealloc_start( - CapPtr slab, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1498,31 +1500,34 @@ namespace snmalloc SNMALLOC_FAST_PATH void medium_dealloc_local( - CapPtr slab, + CapPtr slab, CapPtr p, sizeclass_t sizeclass) { stats().sizeclass_dealloc(sizeclass); bool was_full = Mediumslab::dealloc(slab, p); + auto slab_bounded = capptr_chunk_from_chunkd(slab, SUPERSLAB_SIZE); + if (Mediumslab::empty(slab)) { if (!was_full) { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; auto sc = &medium_classes[medium_class]; - sc->remove(slab); + sc->remove(slab_bounded); } - chunkmap().clear_slab(slab); - large_allocator.dealloc(slab.template as_reinterpret(), 0); + chunkmap().clear_slab(slab_bounded); + large_allocator.dealloc( + slab_bounded.template as_reinterpret(), 0); stats().superslab_push(); } else if (was_full) { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; auto sc = &medium_classes[medium_class]; - sc->insert(slab); + sc->insert(slab_bounded); } } @@ -1549,7 +1554,7 @@ namespace snmalloc if (large_class == 0) size = rsize; - auto p = + CapPtr p = large_allocator.template alloc(large_class, rsize, size); if (likely(p != nullptr)) { @@ -1615,13 +1620,13 @@ namespace snmalloc } size_t large_class = chunkmap_slab_kind - SUPERSLAB_BITS; + auto slab = Aal::capptr_bound(p_auth, size); - chunkmap().clear_large_size(p_auth, size); + chunkmap().clear_large_size(slab, size); stats().large_dealloc(large_class); // Initialise in order to set the correct SlabKind. - auto slab = p_auth.template as_static(); slab->init(); large_allocator.dealloc(slab, large_class); } diff --git a/src/mem/chunkmap.h b/src/mem/chunkmap.h index 4c037ff..9017558 100644 --- a/src/mem/chunkmap.h +++ b/src/mem/chunkmap.h @@ -118,21 +118,21 @@ namespace snmalloc * Set a pagemap entry indicating that there is a superslab at the * specified index. */ - static void set_slab(CapPtr slab) + static void set_slab(CapPtr slab) { set(address_cast(slab), static_cast(CMSuperslab)); } /** * Add a pagemap entry indicating that a medium slab has been allocated. */ - static void set_slab(CapPtr slab) + static void set_slab(CapPtr slab) { set(address_cast(slab), static_cast(CMMediumslab)); } /** * Remove an entry from the pagemap corresponding to a superslab. */ - static void clear_slab(CapPtr slab) + static void clear_slab(CapPtr slab) { SNMALLOC_ASSERT(get(address_cast(slab)) == CMSuperslab); set(address_cast(slab), static_cast(CMNotOurs)); @@ -140,7 +140,7 @@ namespace snmalloc /** * Remove an entry corresponding to a medium slab. */ - static void clear_slab(CapPtr slab) + static void clear_slab(CapPtr slab) { SNMALLOC_ASSERT(get(address_cast(slab)) == CMMediumslab); set(address_cast(slab), static_cast(CMNotOurs)); @@ -149,7 +149,7 @@ namespace snmalloc * Update the pagemap to reflect a large allocation, of `size` bytes from * address `p`. */ - static void set_large_size(CapPtr p, size_t size) + static void set_large_size(CapPtr p, size_t size) { size_t size_bits = bits::next_pow2_bits(size); set(address_cast(p), static_cast(size_bits)); @@ -167,7 +167,7 @@ namespace snmalloc * Update the pagemap to remove a large allocation, of `size` bytes from * address `p`. */ - static void clear_large_size(CapPtr vp, size_t size) + static void clear_large_size(CapPtr vp, size_t size) { auto p = address_cast(vp); size_t rounded_size = bits::next_pow2(size); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index d68d5d5..d7987df 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -325,7 +325,8 @@ namespace snmalloc * Start building a new free list. * Provide pointer to the slab to initialise the system. */ - void open(CapPtr p) + template // TODO: CBChunk-only + void open(CapPtr p) { SNMALLOC_ASSERT(empty()); for (size_t i = 0; i < LENGTH; i++) diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index daa7ea3..0d870fc 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -32,7 +32,7 @@ namespace snmalloc friend class MPMCStack; template friend class MemoryProviderStateMixin; - AtomicCapPtr next = nullptr; + AtomicCapPtr next = nullptr; public: void init() @@ -104,7 +104,7 @@ namespace snmalloc */ ModArray< NUM_LARGE_CLASSES, - MPMCStack> + MPMCStack> large_stack; public: @@ -115,7 +115,7 @@ namespace snmalloc * concurrently with other acceses. If there is no large allocation on a * particular stack then this will return `nullptr`. */ - SNMALLOC_FAST_PATH CapPtr + SNMALLOC_FAST_PATH CapPtr pop_large_stack(size_t large_class) { auto p = large_stack[large_class].pop(); @@ -132,7 +132,7 @@ namespace snmalloc * class specified by `large_class`. Always succeeds. */ SNMALLOC_FAST_PATH void - push_large_stack(CapPtr slab, size_t large_class) + push_large_stack(CapPtr slab, size_t large_class) { const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; available_large_chunks_in_bytes += rsize; @@ -150,7 +150,7 @@ namespace snmalloc * memory providers constructed in this way does not have to be able to * allocate memory, if the initial reservation is sufficient. */ - MemoryProviderStateMixin(CapPtr start, size_t len) + MemoryProviderStateMixin(CapPtr start, size_t len) : address_space(start, len) {} /** @@ -214,7 +214,7 @@ namespace snmalloc size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; size_t decommit_size = rsize - OS_PAGE_SIZE; // Grab all of the chunks of this size class. - CapPtr slab = large_stack[large_class].pop_all(); + CapPtr slab = large_stack[large_class].pop_all(); while (slab != nullptr) { // Decommit all except for the first page and then put it back on @@ -229,7 +229,7 @@ namespace snmalloc // happens-before relationship, so it's safe to use relaxed loads // here. auto next = slab->next.load(std::memory_order_relaxed); - large_stack[large_class].push(CapPtr( + large_stack[large_class].push(CapPtr( new (slab.unsafe_capptr) Decommittedslab())); slab = next; } @@ -279,7 +279,7 @@ namespace snmalloc } template - CapPtr reserve(size_t large_class) noexcept + CapPtr reserve(size_t large_class) noexcept { size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class; peak_memory_used_bytes += size; @@ -324,13 +324,13 @@ namespace snmalloc LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {} template - CapPtr + CapPtr alloc(size_t large_class, size_t rsize, size_t size) { SNMALLOC_ASSERT( (bits::one_at_bit(SUPERSLAB_BITS) << large_class) == rsize); - CapPtr p = + CapPtr p = memory_provider.pop_large_stack(large_class); if (p == nullptr) @@ -381,7 +381,7 @@ namespace snmalloc return p; } - void dealloc(CapPtr p, size_t large_class) + void dealloc(CapPtr p, size_t large_class) { if constexpr (decommit_strategy == DecommitSuperLazy) { diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index 6a11fe0..ec3e83c 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -12,12 +12,12 @@ namespace snmalloc // This is the view of a 16 mb area when it is being used to allocate // medium sized classes: 64 kb to 16 mb, non-inclusive. private: - friend DLList; + friend DLList; // Keep the allocator pointer on a separate cache line. It is read by // other threads, and does not change, so we avoid false sharing. - alignas(CACHELINE_SIZE) CapPtr next; - CapPtr prev; + alignas(CACHELINE_SIZE) CapPtr next; + CapPtr prev; uint16_t free; uint8_t head; @@ -44,16 +44,26 @@ namespace snmalloc return bits::align_up(sizeof(Mediumslab), min(OS_PAGE_SIZE, SLAB_SIZE)); } - template - static SNMALLOC_FAST_PATH CapPtr get(CapPtr p) + /** + * Given a highly-privileged pointer pointing to or within an object in + * this slab, return a pointer to the slab headers. + * + * In debug builds on StrictProvenance architectures, we will enforce the + * slab bounds on this returned pointer. In non-debug builds, we will + * return a highly-privileged pointer (i.e., CBArena) instead as these + * pointers are not exposed from the allocator. + */ + template + static SNMALLOC_FAST_PATH CapPtr + get(CapPtr p) { - static_assert(B == CBArena || B == CBChunk); - - return pointer_align_down(p); + return capptr_bound_chunkd( + pointer_align_down(p.as_void()), + SUPERSLAB_SIZE); } static void init( - CapPtr self, + CapPtr self, RemoteAllocator* alloc, sizeclass_t sc, size_t rsize) @@ -89,7 +99,7 @@ namespace snmalloc template static CapPtr - alloc(CapPtr self, size_t size) + alloc(CapPtr self, size_t size) { SNMALLOC_ASSERT(!full(self)); @@ -106,7 +116,7 @@ namespace snmalloc } static bool - dealloc(CapPtr self, CapPtr p) + dealloc(CapPtr self, CapPtr p) { SNMALLOC_ASSERT(self->head > 0); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index f967e6f..2173786 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -4,14 +4,15 @@ #include "../ds/dllist.h" #include "../ds/helpers.h" #include "freelist.h" +#include "ptrhelpers.h" #include "sizeclass.h" namespace snmalloc { class Slab; - using SlabList = CDLLNode; - using SlabLink = CDLLNode; + using SlabList = CDLLNode; + using SlabLink = CDLLNode; static_assert( sizeof(SlabLink) <= MIN_ALLOC_SIZE, @@ -71,7 +72,7 @@ namespace snmalloc return free_queue.s.next; } - void initialise(sizeclass_t sizeclass, CapPtr slab) + void initialise(sizeclass_t sizeclass, CapPtr slab) { free_queue.s.sizeclass = static_cast(sizeclass); free_queue.init(); @@ -120,8 +121,10 @@ namespace snmalloc return bits::min(threshold, max); } - SNMALLOC_FAST_PATH void set_full(CapPtr slab) + template + SNMALLOC_FAST_PATH void set_full(CapPtr slab) { + static_assert(B == CBChunkD || B == CBChunk); SNMALLOC_ASSERT(free_queue.empty()); // Prepare for the next free queue to be built. @@ -134,9 +137,13 @@ namespace snmalloc } template - static CapPtr get_slab(CapPtr p) + static SNMALLOC_FAST_PATH CapPtr()> + get_slab(CapPtr p) { - return pointer_align_down(p.as_void()); + static_assert(B == CBArena || B == CBChunkD || B == CBChunk); + + return capptr_bound_chunkd( + pointer_align_down(p.as_void()), SLAB_SIZE); } template @@ -145,8 +152,9 @@ namespace snmalloc return pointer_align_down(p.as_void()) == p; } - SNMALLOC_FAST_PATH - static bool is_start_of_object(CapPtr self, address_t p) + template + SNMALLOC_FAST_PATH static bool + is_start_of_object(CapPtr self, address_t p) { return is_multiple_of_sizeclass( self->sizeclass(), SLAB_SIZE - (p - address_align_down(p))); @@ -159,7 +167,7 @@ namespace snmalloc */ template static SNMALLOC_FAST_PATH CapPtr alloc( - CapPtr self, + CapPtr self, FreeListIter& fast_free_list, size_t rsize, LocalEntropy& entropy) @@ -169,17 +177,18 @@ namespace snmalloc self->free_queue.close(fast_free_list, entropy); auto n = fast_free_list.take(entropy); + auto n_slab = Aal::capptr_rebound(self.as_void(), n); entropy.refresh_bits(); // Treat stealing the free list as allocating it all. self->remove(); - self->set_full(Metaslab::get_slab(n)); + self->set_full(Metaslab::get_slab(n_slab)); auto p = remove_cache_friendly_offset(n, self->sizeclass()); SNMALLOC_ASSERT(is_start_of_object(self, address_cast(p))); - self->debug_slab_invariant(Metaslab::get_slab(n), entropy); + self->debug_slab_invariant(Metaslab::get_slab(n_slab), entropy); if constexpr (zero_mem == YesZero) { @@ -196,8 +205,11 @@ namespace snmalloc return capptr_export(Aal::capptr_bound(p, rsize)); } - void debug_slab_invariant(CapPtr slab, LocalEntropy& entropy) + template + void debug_slab_invariant(CapPtr slab, LocalEntropy& entropy) { + static_assert(B == CBChunkD || B == CBChunk); + #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) bool is_short = Metaslab::is_short(slab); diff --git a/src/mem/slab.h b/src/mem/slab.h index b051135..b398c74 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -1,6 +1,7 @@ #pragma once #include "freelist.h" +#include "ptrhelpers.h" #include "superslab.h" #include @@ -20,7 +21,7 @@ namespace snmalloc template static CapPtr get_meta(CapPtr self) { - static_assert(B == CBArena || B == CBChunk); + static_assert(B == CBArena || B == CBChunkD || B == CBChunk); auto super = Superslab::get(self); return super->get_meta(self); @@ -102,8 +103,8 @@ namespace snmalloc // bits. Note that this does remove the use from the meta slab, so it // doesn't need doing on the slow path. static SNMALLOC_FAST_PATH bool dealloc_fast( - CapPtr self, - CapPtr super, + CapPtr self, + CapPtr super, CapPtr p, LocalEntropy& entropy) { @@ -124,9 +125,9 @@ namespace snmalloc // Returns a complex return code for managing the superslab meta data. // i.e. This deallocation could make an entire superslab free. static SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow( - CapPtr self, + CapPtr self, SlabList* sl, - CapPtr super, + CapPtr super, CapPtr p, LocalEntropy& entropy) { @@ -154,7 +155,12 @@ namespace snmalloc allocated - meta->threshold_for_waking_slab(Metaslab::is_short(self)); // Push on the list of slabs for this sizeclass. - sl->insert_prev(meta.template as_static()); + // ChunkD-to-Chunk conversion might apply bounds, so we need to do so to + // the aligned object and then shift over to these bounds. + auto super_chunk = capptr_chunk_from_chunkd(super, SUPERSLAB_SIZE); + auto metalink = Aal::capptr_rebound( + super_chunk.as_void(), meta.template as_static()); + sl->insert_prev(metalink); meta->debug_slab_invariant(self, entropy); return Superslab::NoSlabReturn; } diff --git a/src/mem/superslab.h b/src/mem/superslab.h index b3f805f..8846096 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -30,15 +30,15 @@ namespace snmalloc class Superslab : public Allocslab { private: - friend DLList; + friend DLList; // Keep the allocator pointer on a separate cache line. It is read by // other threads, and does not change, so we avoid false sharing. alignas(CACHELINE_SIZE) // The superslab is kept on a doubly linked list of superslabs which // have some space. - Superslab* next; - Superslab* prev; + CapPtr next; + CapPtr prev; // This is a reference to the first unused slab in the free slab list // It is does not contain the short slab, which is handled using a bit @@ -83,12 +83,24 @@ namespace snmalloc StatusChange = 2 }; + /** + * Given a highly-privileged pointer pointing to or within an object in + * this slab, return a pointer to the slab headers. + * + * In debug builds on StrictProvenance architectures, we will enforce the + * slab bounds on this returned pointer. In non-debug builds, we will + * return a highly-privileged pointer (i.e., CBArena) instead as these + * pointers are not exposed from the allocator. + */ template - static SNMALLOC_FAST_PATH CapPtr get(CapPtr p) + static SNMALLOC_FAST_PATH CapPtr()> + get(CapPtr p) { - static_assert(B == CBArena || B == CBChunk); + static_assert(B == CBArena || B == CBChunkD || B == CBChunk); - return pointer_align_down(p.as_void()); + return capptr_bound_chunkd( + pointer_align_down(p.as_void()), + SUPERSLAB_SIZE); } static bool is_short_sizeclass(sizeclass_t sizeclass) @@ -199,7 +211,7 @@ namespace snmalloc Slab* slab = reinterpret_cast(self); auto& metaz = self->meta[0]; - metaz.initialise(sizeclass, CapPtr(slab)); + metaz.initialise(sizeclass, CapPtr(slab)); self->used++; return slab; @@ -216,7 +228,7 @@ namespace snmalloc auto& metah = self->meta[h]; uint8_t n = metah.next(); - metah.initialise(sizeclass, CapPtr(slab)); + metah.initialise(sizeclass, CapPtr(slab)); self->head = h + n + 1; self->used += 2; @@ -228,7 +240,7 @@ namespace snmalloc template Action dealloc_slab(CapPtr slab) { - static_assert(B == CBArena || B == CBChunk); + static_assert(B == CBArena || B == CBChunkD || B == CBChunk); // This is not the short slab. uint8_t index = static_cast(slab_to_index(slab)); diff --git a/src/test/func/sandbox/sandbox.cc b/src/test/func/sandbox/sandbox.cc index 1531133..1b4ffa3 100644 --- a/src/test/func/sandbox/sandbox.cc +++ b/src/test/func/sandbox/sandbox.cc @@ -113,7 +113,7 @@ namespace * * This method must be implemented for `LargeAlloc` to work. */ - CapPtr pop_large_stack(size_t large_class) + CapPtr pop_large_stack(size_t large_class) { return real_state->pop_large_stack(large_class); }; @@ -124,7 +124,7 @@ namespace * * This method must be implemented for `LargeAlloc` to work. */ - void push_large_stack(CapPtr slab, size_t large_class) + void push_large_stack(CapPtr slab, size_t large_class) { real_state->push_large_stack(slab, large_class); } @@ -136,7 +136,7 @@ namespace * This method must be implemented for `LargeAlloc` to work. */ template - CapPtr reserve(size_t large_class) noexcept + CapPtr reserve(size_t large_class) noexcept { return real_state->template reserve(large_class); } @@ -207,7 +207,7 @@ namespace top(pointer_offset(start, sb_size)), shared_state(new (start) SharedState()), state( - pointer_offset(CapPtr(start), sizeof(SharedState)), + pointer_offset(CapPtr(start), sizeof(SharedState)), sb_size - sizeof(SharedState)), alloc(state, SNMALLOC_DEFAULT_CHUNKMAP(), &shared_state->queue) {