diff --git a/src/mem/address_space.h b/src/mem/address_space.h index 81e740e..1553053 100644 --- a/src/mem/address_space.h +++ b/src/mem/address_space.h @@ -35,7 +35,7 @@ namespace snmalloc * bits::BITS is used for simplicity, we do not use below the pointer size, * and large entries will be unlikely to be supported by the platform. */ - std::array, 2>, bits::BITS> ranges = {}; + std::array, 2>, bits::BITS> ranges = {}; /** * This is infrequently used code, a spin lock simplifies the code @@ -46,7 +46,7 @@ namespace snmalloc /** * Checks a block satisfies its invariant. */ - inline void check_block(CapPtr base, size_t align_bits) + inline void check_block(CapPtr base, size_t align_bits) { SNMALLOC_ASSERT( base == pointer_align_up(base, bits::one_at_bit(align_bits))); @@ -59,7 +59,7 @@ namespace snmalloc /** * Adds a block to `ranges`. */ - void add_block(size_t align_bits, CapPtr base) + void add_block(size_t align_bits, CapPtr base) { check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); @@ -74,7 +74,7 @@ namespace snmalloc { // Add to linked list. commit_block(base, sizeof(void*)); - *(base.template as_static>().unsafe_capptr) = + *(base.template as_static>().unsafe_capptr) = ranges[align_bits][1]; check_block(ranges[align_bits][1], align_bits); } @@ -88,9 +88,9 @@ namespace snmalloc * Find a block of the correct size. May split larger blocks * to satisfy this request. */ - CapPtr remove_block(size_t align_bits) + CapPtr remove_block(size_t align_bits) { - CapPtr first = ranges[align_bits][0]; + CapPtr first = ranges[align_bits][0]; if (first == nullptr) { if (align_bits == (bits::BITS - 1)) @@ -100,7 +100,7 @@ namespace snmalloc } // Look for larger block and split up recursively - CapPtr bigger = remove_block(align_bits + 1); + CapPtr bigger = remove_block(align_bits + 1); if (bigger != nullptr) { auto left_over = pointer_offset(bigger, bits::one_at_bit(align_bits)); @@ -111,12 +111,12 @@ namespace snmalloc return bigger; } - CapPtr second = ranges[align_bits][1]; + CapPtr second = ranges[align_bits][1]; if (second != nullptr) { commit_block(second, sizeof(void*)); auto psecond = - second.template as_static>().unsafe_capptr; + second.template as_static>().unsafe_capptr; auto next = *psecond; ranges[align_bits][1] = next; // Zero memory. Client assumes memory contains only zeros. @@ -135,7 +135,7 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - void add_range(CapPtr base, size_t length) + void add_range(CapPtr base, size_t length) { // Find the minimum set of maximally aligned blocks in this range. // Each block's alignment and size are equal. @@ -157,7 +157,7 @@ namespace snmalloc /** * Commit a block of memory */ - void commit_block(CapPtr base, size_t size) + void commit_block(CapPtr base, size_t size) { // Rounding required for sub-page allocations. auto page_start = pointer_align_down(base); @@ -180,7 +180,7 @@ namespace snmalloc * arena_map for use in subsequent amplification. */ template - CapPtr reserve(size_t size, ArenaMap& arena_map) + CapPtr reserve(size_t size, ArenaMap& arena_map) { SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -194,18 +194,18 @@ namespace snmalloc pal_supports && !aal_supports) { if (size >= PAL::minimum_alloc_size) - return CapPtr( + return CapPtr( PAL::template reserve_aligned(size)); } - CapPtr res; + CapPtr res; { FlagLock lock(spin_lock); res = remove_block(bits::next_pow2_bits(size)); if (res == nullptr) { // Allocation failed ask OS for more memory - CapPtr block = nullptr; + CapPtr block = nullptr; size_t block_size = 0; if constexpr (pal_supports) { @@ -233,7 +233,10 @@ namespace snmalloc void* block_raw = PAL::template reserve_aligned(block_size); - block = CapPtr(block_raw); + // It's a bit of a lie to convert without applying bounds, but the + // platform will have bounded block for us and it's better that the + // rest of our internals expect CBChunk bounds. + block = CapPtr(block_raw); if constexpr (aal_supports) { @@ -254,7 +257,7 @@ namespace snmalloc // the PAL, and this could lead to suprious OOM. This is // particularly bad if the PAL gives all the memory on first call. auto block_and_size = PAL::reserve_at_least(size * 2); - block = CapPtr(block_and_size.first); + block = CapPtr(block_and_size.first); block_size = block_and_size.second; // Ensure block is pointer aligned. @@ -294,7 +297,7 @@ namespace snmalloc * used, by smaller objects. */ template - CapPtr + CapPtr reserve_with_left_over(size_t size, ArenaMap& arena_map) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -330,7 +333,7 @@ namespace snmalloc * Constructor that pre-initialises the address-space manager with a region * of memory. */ - AddressSpaceManager(CapPtr base, size_t length) + AddressSpaceManager(CapPtr base, size_t length) { add_range(base, length); } diff --git a/src/mem/alloc.h b/src/mem/alloc.h index d0bc925..5eaaab7 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -638,10 +638,10 @@ namespace snmalloc }; SlabList small_classes[NUM_SMALL_CLASSES]; - DLList medium_classes[NUM_MEDIUM_CLASSES]; + DLList medium_classes[NUM_MEDIUM_CLASSES]; - DLList super_available; - DLList super_only_short_available; + DLList super_available; + DLList super_only_short_available; RemoteCache remote; @@ -901,7 +901,7 @@ namespace snmalloc } SNMALLOC_FAST_PATH void dealloc_not_large_local( - CapPtr super, + CapPtr super, CapPtr p_auth_offseted, sizeclass_t sizeclass) { @@ -963,28 +963,27 @@ namespace snmalloc handle_message_queue_inner(); } - Superslab* get_superslab() + CapPtr get_superslab() { - Superslab* super = super_available.get_head(); + auto super = super_available.get_head(); if (super != nullptr) return super; super = large_allocator .template alloc(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE) - .template as_reinterpret() - .unsafe_capptr; + .template as_reinterpret(); if (super == nullptr) return super; super->init(public_state()); - chunkmap().set_slab(CapPtr(super)); + chunkmap().set_slab(super); super_available.insert(super); return super; } - void reposition_superslab(Superslab* super) + void reposition_superslab(CapPtr super) { switch (super->get_status()) { @@ -1025,11 +1024,12 @@ namespace snmalloc { // Pull a short slab from the list of superslabs that have only the // short slab available. - Superslab* super = super_only_short_available.pop(); + CapPtr super = super_only_short_available.pop(); if (super != nullptr) { - auto slab = Superslab::alloc_short_slab(super, sizeclass); + auto slab = + Superslab::alloc_short_slab(super.unsafe_capptr, sizeclass); SNMALLOC_ASSERT(super->is_full()); return CapPtr(slab); } @@ -1039,17 +1039,17 @@ namespace snmalloc if (super == nullptr) return nullptr; - auto slab = Superslab::alloc_short_slab(super, sizeclass); + auto slab = Superslab::alloc_short_slab(super.unsafe_capptr, sizeclass); reposition_superslab(super); return CapPtr(slab); } - Superslab* super = get_superslab(); + auto super = get_superslab(); if (super == nullptr) return nullptr; - auto slab = Superslab::alloc_slab(super, sizeclass); + auto slab = Superslab::alloc_slab(super.unsafe_capptr, sizeclass); reposition_superslab(super); return CapPtr(slab); } @@ -1233,7 +1233,7 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_unchecked( - CapPtr super, + CapPtr super, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1246,7 +1246,7 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_checked_chunkmap( - CapPtr super, + CapPtr super, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1260,8 +1260,8 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_checked_sizeclass( - CapPtr super, - CapPtr slab, + CapPtr super, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1274,8 +1274,8 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_start( - CapPtr super, - CapPtr slab, + CapPtr super, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1295,8 +1295,8 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_offseted( - CapPtr super, - CapPtr slab, + CapPtr super, + CapPtr slab, CapPtr p, sizeclass_t sizeclass) { @@ -1306,8 +1306,8 @@ namespace snmalloc } SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( - CapPtr super, - CapPtr slab, + CapPtr super, + CapPtr slab, CapPtr p, sizeclass_t sizeclass) { @@ -1318,8 +1318,8 @@ namespace snmalloc } SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow( - CapPtr super, - CapPtr slab, + CapPtr super, + CapPtr slab, CapPtr p, sizeclass_t sizeclass) { @@ -1333,6 +1333,8 @@ namespace snmalloc if (a == Superslab::NoStatusChange) return; + auto super_slab = capptr_chunk_from_chunkd(super, SUPERSLAB_SIZE); + switch (super->get_status()) { case Superslab::Full: @@ -1345,29 +1347,29 @@ namespace snmalloc { if (was_full) { - super_available.insert(super.unsafe_capptr); + super_available.insert(super_slab); } else { - super_only_short_available.remove(super.unsafe_capptr); - super_available.insert(super.unsafe_capptr); + super_only_short_available.remove(super_slab); + super_available.insert(super_slab); } break; } case Superslab::OnlyShortSlabAvailable: { - super_only_short_available.insert(super.unsafe_capptr); + super_only_short_available.insert(super_slab); break; } case Superslab::Empty: { - super_available.remove(super.unsafe_capptr); + super_available.remove(super_slab); - chunkmap().clear_slab(super); + chunkmap().clear_slab(super_slab); large_allocator.dealloc( - super.template as_reinterpret(), 0); + super_slab.template as_reinterpret(), 0); stats().superslab_push(); break; } @@ -1381,7 +1383,7 @@ namespace snmalloc sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; auto sc = &medium_classes[medium_class]; - auto slab = sc->get_head(); + CapPtr slab = sc->get_head(); CapPtr p; if (slab != nullptr) @@ -1437,7 +1439,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void medium_dealloc_unchecked( - CapPtr slab, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1451,7 +1453,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void medium_dealloc_checked_chunkmap( - CapPtr slab, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1465,7 +1467,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void medium_dealloc_checked_sizeclass( - CapPtr slab, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1480,7 +1482,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void medium_dealloc_start( - CapPtr slab, + CapPtr slab, CapPtr p_auth, CapPtr p_ret, sizeclass_t sizeclass) @@ -1498,31 +1500,34 @@ namespace snmalloc SNMALLOC_FAST_PATH void medium_dealloc_local( - CapPtr slab, + CapPtr slab, CapPtr p, sizeclass_t sizeclass) { stats().sizeclass_dealloc(sizeclass); bool was_full = Mediumslab::dealloc(slab, p); + auto slab_bounded = capptr_chunk_from_chunkd(slab, SUPERSLAB_SIZE); + if (Mediumslab::empty(slab)) { if (!was_full) { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; auto sc = &medium_classes[medium_class]; - sc->remove(slab); + sc->remove(slab_bounded); } - chunkmap().clear_slab(slab); - large_allocator.dealloc(slab.template as_reinterpret(), 0); + chunkmap().clear_slab(slab_bounded); + large_allocator.dealloc( + slab_bounded.template as_reinterpret(), 0); stats().superslab_push(); } else if (was_full) { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; auto sc = &medium_classes[medium_class]; - sc->insert(slab); + sc->insert(slab_bounded); } } @@ -1549,7 +1554,7 @@ namespace snmalloc if (large_class == 0) size = rsize; - auto p = + CapPtr p = large_allocator.template alloc(large_class, rsize, size); if (likely(p != nullptr)) { @@ -1615,13 +1620,13 @@ namespace snmalloc } size_t large_class = chunkmap_slab_kind - SUPERSLAB_BITS; + auto slab = Aal::capptr_bound(p_auth, size); - chunkmap().clear_large_size(p_auth, size); + chunkmap().clear_large_size(slab, size); stats().large_dealloc(large_class); // Initialise in order to set the correct SlabKind. - auto slab = p_auth.template as_static(); slab->init(); large_allocator.dealloc(slab, large_class); } diff --git a/src/mem/chunkmap.h b/src/mem/chunkmap.h index 4c037ff..9017558 100644 --- a/src/mem/chunkmap.h +++ b/src/mem/chunkmap.h @@ -118,21 +118,21 @@ namespace snmalloc * Set a pagemap entry indicating that there is a superslab at the * specified index. */ - static void set_slab(CapPtr slab) + static void set_slab(CapPtr slab) { set(address_cast(slab), static_cast(CMSuperslab)); } /** * Add a pagemap entry indicating that a medium slab has been allocated. */ - static void set_slab(CapPtr slab) + static void set_slab(CapPtr slab) { set(address_cast(slab), static_cast(CMMediumslab)); } /** * Remove an entry from the pagemap corresponding to a superslab. */ - static void clear_slab(CapPtr slab) + static void clear_slab(CapPtr slab) { SNMALLOC_ASSERT(get(address_cast(slab)) == CMSuperslab); set(address_cast(slab), static_cast(CMNotOurs)); @@ -140,7 +140,7 @@ namespace snmalloc /** * Remove an entry corresponding to a medium slab. */ - static void clear_slab(CapPtr slab) + static void clear_slab(CapPtr slab) { SNMALLOC_ASSERT(get(address_cast(slab)) == CMMediumslab); set(address_cast(slab), static_cast(CMNotOurs)); @@ -149,7 +149,7 @@ namespace snmalloc * Update the pagemap to reflect a large allocation, of `size` bytes from * address `p`. */ - static void set_large_size(CapPtr p, size_t size) + static void set_large_size(CapPtr p, size_t size) { size_t size_bits = bits::next_pow2_bits(size); set(address_cast(p), static_cast(size_bits)); @@ -167,7 +167,7 @@ namespace snmalloc * Update the pagemap to remove a large allocation, of `size` bytes from * address `p`. */ - static void clear_large_size(CapPtr vp, size_t size) + static void clear_large_size(CapPtr vp, size_t size) { auto p = address_cast(vp); size_t rounded_size = bits::next_pow2(size); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index d68d5d5..d7987df 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -325,7 +325,8 @@ namespace snmalloc * Start building a new free list. * Provide pointer to the slab to initialise the system. */ - void open(CapPtr p) + template // TODO: CBChunk-only + void open(CapPtr p) { SNMALLOC_ASSERT(empty()); for (size_t i = 0; i < LENGTH; i++) diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index daa7ea3..0d870fc 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -32,7 +32,7 @@ namespace snmalloc friend class MPMCStack; template friend class MemoryProviderStateMixin; - AtomicCapPtr next = nullptr; + AtomicCapPtr next = nullptr; public: void init() @@ -104,7 +104,7 @@ namespace snmalloc */ ModArray< NUM_LARGE_CLASSES, - MPMCStack> + MPMCStack> large_stack; public: @@ -115,7 +115,7 @@ namespace snmalloc * concurrently with other acceses. If there is no large allocation on a * particular stack then this will return `nullptr`. */ - SNMALLOC_FAST_PATH CapPtr + SNMALLOC_FAST_PATH CapPtr pop_large_stack(size_t large_class) { auto p = large_stack[large_class].pop(); @@ -132,7 +132,7 @@ namespace snmalloc * class specified by `large_class`. Always succeeds. */ SNMALLOC_FAST_PATH void - push_large_stack(CapPtr slab, size_t large_class) + push_large_stack(CapPtr slab, size_t large_class) { const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; available_large_chunks_in_bytes += rsize; @@ -150,7 +150,7 @@ namespace snmalloc * memory providers constructed in this way does not have to be able to * allocate memory, if the initial reservation is sufficient. */ - MemoryProviderStateMixin(CapPtr start, size_t len) + MemoryProviderStateMixin(CapPtr start, size_t len) : address_space(start, len) {} /** @@ -214,7 +214,7 @@ namespace snmalloc size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; size_t decommit_size = rsize - OS_PAGE_SIZE; // Grab all of the chunks of this size class. - CapPtr slab = large_stack[large_class].pop_all(); + CapPtr slab = large_stack[large_class].pop_all(); while (slab != nullptr) { // Decommit all except for the first page and then put it back on @@ -229,7 +229,7 @@ namespace snmalloc // happens-before relationship, so it's safe to use relaxed loads // here. auto next = slab->next.load(std::memory_order_relaxed); - large_stack[large_class].push(CapPtr( + large_stack[large_class].push(CapPtr( new (slab.unsafe_capptr) Decommittedslab())); slab = next; } @@ -279,7 +279,7 @@ namespace snmalloc } template - CapPtr reserve(size_t large_class) noexcept + CapPtr reserve(size_t large_class) noexcept { size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class; peak_memory_used_bytes += size; @@ -324,13 +324,13 @@ namespace snmalloc LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {} template - CapPtr + CapPtr alloc(size_t large_class, size_t rsize, size_t size) { SNMALLOC_ASSERT( (bits::one_at_bit(SUPERSLAB_BITS) << large_class) == rsize); - CapPtr p = + CapPtr p = memory_provider.pop_large_stack(large_class); if (p == nullptr) @@ -381,7 +381,7 @@ namespace snmalloc return p; } - void dealloc(CapPtr p, size_t large_class) + void dealloc(CapPtr p, size_t large_class) { if constexpr (decommit_strategy == DecommitSuperLazy) { diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index 6a11fe0..ec3e83c 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -12,12 +12,12 @@ namespace snmalloc // This is the view of a 16 mb area when it is being used to allocate // medium sized classes: 64 kb to 16 mb, non-inclusive. private: - friend DLList; + friend DLList; // Keep the allocator pointer on a separate cache line. It is read by // other threads, and does not change, so we avoid false sharing. - alignas(CACHELINE_SIZE) CapPtr next; - CapPtr prev; + alignas(CACHELINE_SIZE) CapPtr next; + CapPtr prev; uint16_t free; uint8_t head; @@ -44,16 +44,26 @@ namespace snmalloc return bits::align_up(sizeof(Mediumslab), min(OS_PAGE_SIZE, SLAB_SIZE)); } - template - static SNMALLOC_FAST_PATH CapPtr get(CapPtr p) + /** + * Given a highly-privileged pointer pointing to or within an object in + * this slab, return a pointer to the slab headers. + * + * In debug builds on StrictProvenance architectures, we will enforce the + * slab bounds on this returned pointer. In non-debug builds, we will + * return a highly-privileged pointer (i.e., CBArena) instead as these + * pointers are not exposed from the allocator. + */ + template + static SNMALLOC_FAST_PATH CapPtr + get(CapPtr p) { - static_assert(B == CBArena || B == CBChunk); - - return pointer_align_down(p); + return capptr_bound_chunkd( + pointer_align_down(p.as_void()), + SUPERSLAB_SIZE); } static void init( - CapPtr self, + CapPtr self, RemoteAllocator* alloc, sizeclass_t sc, size_t rsize) @@ -89,7 +99,7 @@ namespace snmalloc template static CapPtr - alloc(CapPtr self, size_t size) + alloc(CapPtr self, size_t size) { SNMALLOC_ASSERT(!full(self)); @@ -106,7 +116,7 @@ namespace snmalloc } static bool - dealloc(CapPtr self, CapPtr p) + dealloc(CapPtr self, CapPtr p) { SNMALLOC_ASSERT(self->head > 0); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index f967e6f..2173786 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -4,14 +4,15 @@ #include "../ds/dllist.h" #include "../ds/helpers.h" #include "freelist.h" +#include "ptrhelpers.h" #include "sizeclass.h" namespace snmalloc { class Slab; - using SlabList = CDLLNode; - using SlabLink = CDLLNode; + using SlabList = CDLLNode; + using SlabLink = CDLLNode; static_assert( sizeof(SlabLink) <= MIN_ALLOC_SIZE, @@ -71,7 +72,7 @@ namespace snmalloc return free_queue.s.next; } - void initialise(sizeclass_t sizeclass, CapPtr slab) + void initialise(sizeclass_t sizeclass, CapPtr slab) { free_queue.s.sizeclass = static_cast(sizeclass); free_queue.init(); @@ -120,8 +121,10 @@ namespace snmalloc return bits::min(threshold, max); } - SNMALLOC_FAST_PATH void set_full(CapPtr slab) + template + SNMALLOC_FAST_PATH void set_full(CapPtr slab) { + static_assert(B == CBChunkD || B == CBChunk); SNMALLOC_ASSERT(free_queue.empty()); // Prepare for the next free queue to be built. @@ -134,9 +137,13 @@ namespace snmalloc } template - static CapPtr get_slab(CapPtr p) + static SNMALLOC_FAST_PATH CapPtr()> + get_slab(CapPtr p) { - return pointer_align_down(p.as_void()); + static_assert(B == CBArena || B == CBChunkD || B == CBChunk); + + return capptr_bound_chunkd( + pointer_align_down(p.as_void()), SLAB_SIZE); } template @@ -145,8 +152,9 @@ namespace snmalloc return pointer_align_down(p.as_void()) == p; } - SNMALLOC_FAST_PATH - static bool is_start_of_object(CapPtr self, address_t p) + template + SNMALLOC_FAST_PATH static bool + is_start_of_object(CapPtr self, address_t p) { return is_multiple_of_sizeclass( self->sizeclass(), SLAB_SIZE - (p - address_align_down(p))); @@ -159,7 +167,7 @@ namespace snmalloc */ template static SNMALLOC_FAST_PATH CapPtr alloc( - CapPtr self, + CapPtr self, FreeListIter& fast_free_list, size_t rsize, LocalEntropy& entropy) @@ -169,17 +177,18 @@ namespace snmalloc self->free_queue.close(fast_free_list, entropy); auto n = fast_free_list.take(entropy); + auto n_slab = Aal::capptr_rebound(self.as_void(), n); entropy.refresh_bits(); // Treat stealing the free list as allocating it all. self->remove(); - self->set_full(Metaslab::get_slab(n)); + self->set_full(Metaslab::get_slab(n_slab)); auto p = remove_cache_friendly_offset(n, self->sizeclass()); SNMALLOC_ASSERT(is_start_of_object(self, address_cast(p))); - self->debug_slab_invariant(Metaslab::get_slab(n), entropy); + self->debug_slab_invariant(Metaslab::get_slab(n_slab), entropy); if constexpr (zero_mem == YesZero) { @@ -196,8 +205,11 @@ namespace snmalloc return capptr_export(Aal::capptr_bound(p, rsize)); } - void debug_slab_invariant(CapPtr slab, LocalEntropy& entropy) + template + void debug_slab_invariant(CapPtr slab, LocalEntropy& entropy) { + static_assert(B == CBChunkD || B == CBChunk); + #if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) bool is_short = Metaslab::is_short(slab); diff --git a/src/mem/slab.h b/src/mem/slab.h index b051135..b398c74 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -1,6 +1,7 @@ #pragma once #include "freelist.h" +#include "ptrhelpers.h" #include "superslab.h" #include @@ -20,7 +21,7 @@ namespace snmalloc template static CapPtr get_meta(CapPtr self) { - static_assert(B == CBArena || B == CBChunk); + static_assert(B == CBArena || B == CBChunkD || B == CBChunk); auto super = Superslab::get(self); return super->get_meta(self); @@ -102,8 +103,8 @@ namespace snmalloc // bits. Note that this does remove the use from the meta slab, so it // doesn't need doing on the slow path. static SNMALLOC_FAST_PATH bool dealloc_fast( - CapPtr self, - CapPtr super, + CapPtr self, + CapPtr super, CapPtr p, LocalEntropy& entropy) { @@ -124,9 +125,9 @@ namespace snmalloc // Returns a complex return code for managing the superslab meta data. // i.e. This deallocation could make an entire superslab free. static SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow( - CapPtr self, + CapPtr self, SlabList* sl, - CapPtr super, + CapPtr super, CapPtr p, LocalEntropy& entropy) { @@ -154,7 +155,12 @@ namespace snmalloc allocated - meta->threshold_for_waking_slab(Metaslab::is_short(self)); // Push on the list of slabs for this sizeclass. - sl->insert_prev(meta.template as_static()); + // ChunkD-to-Chunk conversion might apply bounds, so we need to do so to + // the aligned object and then shift over to these bounds. + auto super_chunk = capptr_chunk_from_chunkd(super, SUPERSLAB_SIZE); + auto metalink = Aal::capptr_rebound( + super_chunk.as_void(), meta.template as_static()); + sl->insert_prev(metalink); meta->debug_slab_invariant(self, entropy); return Superslab::NoSlabReturn; } diff --git a/src/mem/superslab.h b/src/mem/superslab.h index b3f805f..8846096 100644 --- a/src/mem/superslab.h +++ b/src/mem/superslab.h @@ -30,15 +30,15 @@ namespace snmalloc class Superslab : public Allocslab { private: - friend DLList; + friend DLList; // Keep the allocator pointer on a separate cache line. It is read by // other threads, and does not change, so we avoid false sharing. alignas(CACHELINE_SIZE) // The superslab is kept on a doubly linked list of superslabs which // have some space. - Superslab* next; - Superslab* prev; + CapPtr next; + CapPtr prev; // This is a reference to the first unused slab in the free slab list // It is does not contain the short slab, which is handled using a bit @@ -83,12 +83,24 @@ namespace snmalloc StatusChange = 2 }; + /** + * Given a highly-privileged pointer pointing to or within an object in + * this slab, return a pointer to the slab headers. + * + * In debug builds on StrictProvenance architectures, we will enforce the + * slab bounds on this returned pointer. In non-debug builds, we will + * return a highly-privileged pointer (i.e., CBArena) instead as these + * pointers are not exposed from the allocator. + */ template - static SNMALLOC_FAST_PATH CapPtr get(CapPtr p) + static SNMALLOC_FAST_PATH CapPtr()> + get(CapPtr p) { - static_assert(B == CBArena || B == CBChunk); + static_assert(B == CBArena || B == CBChunkD || B == CBChunk); - return pointer_align_down(p.as_void()); + return capptr_bound_chunkd( + pointer_align_down(p.as_void()), + SUPERSLAB_SIZE); } static bool is_short_sizeclass(sizeclass_t sizeclass) @@ -199,7 +211,7 @@ namespace snmalloc Slab* slab = reinterpret_cast(self); auto& metaz = self->meta[0]; - metaz.initialise(sizeclass, CapPtr(slab)); + metaz.initialise(sizeclass, CapPtr(slab)); self->used++; return slab; @@ -216,7 +228,7 @@ namespace snmalloc auto& metah = self->meta[h]; uint8_t n = metah.next(); - metah.initialise(sizeclass, CapPtr(slab)); + metah.initialise(sizeclass, CapPtr(slab)); self->head = h + n + 1; self->used += 2; @@ -228,7 +240,7 @@ namespace snmalloc template Action dealloc_slab(CapPtr slab) { - static_assert(B == CBArena || B == CBChunk); + static_assert(B == CBArena || B == CBChunkD || B == CBChunk); // This is not the short slab. uint8_t index = static_cast(slab_to_index(slab)); diff --git a/src/test/func/sandbox/sandbox.cc b/src/test/func/sandbox/sandbox.cc index 1531133..1b4ffa3 100644 --- a/src/test/func/sandbox/sandbox.cc +++ b/src/test/func/sandbox/sandbox.cc @@ -113,7 +113,7 @@ namespace * * This method must be implemented for `LargeAlloc` to work. */ - CapPtr pop_large_stack(size_t large_class) + CapPtr pop_large_stack(size_t large_class) { return real_state->pop_large_stack(large_class); }; @@ -124,7 +124,7 @@ namespace * * This method must be implemented for `LargeAlloc` to work. */ - void push_large_stack(CapPtr slab, size_t large_class) + void push_large_stack(CapPtr slab, size_t large_class) { real_state->push_large_stack(slab, large_class); } @@ -136,7 +136,7 @@ namespace * This method must be implemented for `LargeAlloc` to work. */ template - CapPtr reserve(size_t large_class) noexcept + CapPtr reserve(size_t large_class) noexcept { return real_state->template reserve(large_class); } @@ -207,7 +207,7 @@ namespace top(pointer_offset(start, sb_size)), shared_state(new (start) SharedState()), state( - pointer_offset(CapPtr(start), sizeof(SharedState)), + pointer_offset(CapPtr(start), sizeof(SharedState)), sb_size - sizeof(SharedState)), alloc(state, SNMALLOC_DEFAULT_CHUNKMAP(), &shared_state->queue) {