diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 684e257..d0bc925 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -177,14 +177,14 @@ namespace snmalloc { // Allocations smaller than the slab size are more likely. Improve // branch prediction by placing this case first. - return small_alloc(size).unsafe_capptr; + return capptr_reveal(small_alloc(size)); } - return alloc_not_small(size).unsafe_capptr; + return capptr_reveal(alloc_not_small(size)); } template - SNMALLOC_SLOW_PATH CapPtr alloc_not_small(size_t size) + SNMALLOC_SLOW_PATH CapPtr alloc_not_small(size_t size) { handle_message_queue(); @@ -1055,7 +1055,7 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH CapPtr small_alloc(size_t size) + SNMALLOC_FAST_PATH CapPtr small_alloc(size_t size) { SNMALLOC_ASSUME(size <= SLAB_SIZE); sizeclass_t sizeclass = size_to_sizeclass(size); @@ -1063,7 +1063,7 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH CapPtr + SNMALLOC_FAST_PATH CapPtr small_alloc_inner(sizeclass_t sizeclass, size_t size) { SNMALLOC_ASSUME(sizeclass < NUM_SMALL_CLASSES); @@ -1078,7 +1078,11 @@ namespace snmalloc pal_zero( p, sizeclass_to_size(sizeclass)); } - return p; + + // TODO: This goes away once our free lists are bounded + auto ret = Aal::capptr_bound(p, size); + + return capptr_export(ret); } if (likely(!has_messages())) @@ -1092,7 +1096,7 @@ namespace snmalloc * allocation request. */ template - SNMALLOC_SLOW_PATH CapPtr + SNMALLOC_SLOW_PATH CapPtr small_alloc_mq_slow(sizeclass_t sizeclass, size_t size) { handle_message_queue_inner(); @@ -1104,7 +1108,7 @@ namespace snmalloc * Attempt to find a new free list to allocate from */ template - SNMALLOC_SLOW_PATH CapPtr + SNMALLOC_SLOW_PATH CapPtr small_alloc_next_free_list(sizeclass_t sizeclass, size_t size) { size_t rsize = sizeclass_to_size(sizeclass); @@ -1129,7 +1133,7 @@ namespace snmalloc * new free list. */ template - SNMALLOC_SLOW_PATH CapPtr + SNMALLOC_SLOW_PATH CapPtr small_alloc_rare(sizeclass_t sizeclass, size_t size) { if (likely(!NeedsInitialisation(this))) @@ -1146,15 +1150,22 @@ namespace snmalloc * then directs the allocation request to the newly created allocator. */ template - SNMALLOC_SLOW_PATH CapPtr + SNMALLOC_SLOW_PATH CapPtr small_alloc_first_alloc(sizeclass_t sizeclass, size_t size) { + /* + * We have to convert through void* as part of the thread allocator + * initializer API. Be a little more verbose than strictly necessary to + * demonstrate that small_alloc_inner is giving us a CBAllocE-annotated + * pointer before we just go slapping that label on a void* later. + */ void* ret = InitThreadAllocator([sizeclass, size](void* alloc) { - return reinterpret_cast(alloc) - ->template small_alloc_inner(sizeclass, size) - .unsafe_capptr; + CapPtr ret = + reinterpret_cast(alloc) + ->template small_alloc_inner(sizeclass, size); + return ret.unsafe_capptr; }); - return CapPtr(ret); + return CapPtr(ret); } /** @@ -1162,7 +1173,7 @@ namespace snmalloc * list. */ template - SNMALLOC_FAST_PATH CapPtr + SNMALLOC_FAST_PATH CapPtr small_alloc_new_free_list(sizeclass_t sizeclass) { auto& bp = bump_ptrs[sizeclass]; @@ -1179,7 +1190,7 @@ namespace snmalloc * the request from that new list. */ template - SNMALLOC_FAST_PATH CapPtr + SNMALLOC_FAST_PATH CapPtr small_alloc_build_free_list(sizeclass_t sizeclass) { auto& bp = bump_ptrs[sizeclass]; @@ -1194,7 +1205,11 @@ namespace snmalloc { pal_zero(p, sizeclass_to_size(sizeclass)); } - return CapPtr(p); + + // TODO: This goes away once our free lists are bounded + auto p_bounded = Aal::capptr_bound(p, rsize); + + return capptr_export(p_bounded); } /** @@ -1203,7 +1218,7 @@ namespace snmalloc * local bump allocator and service the request from that new list. */ template - SNMALLOC_SLOW_PATH CapPtr + SNMALLOC_SLOW_PATH CapPtr small_alloc_new_slab(sizeclass_t sizeclass) { auto& bp = bump_ptrs[sizeclass]; @@ -1360,14 +1375,14 @@ namespace snmalloc } template - CapPtr + CapPtr medium_alloc(sizeclass_t sizeclass, size_t rsize, size_t size) { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; auto sc = &medium_classes[medium_class]; auto slab = sc->get_head(); - CapPtr p; + CapPtr p; if (slab != nullptr) { @@ -1389,31 +1404,34 @@ namespace snmalloc */ void* ret = InitThreadAllocator([size, rsize, sizeclass](void* alloc) { - CapPtr ret = + CapPtr ret = reinterpret_cast(alloc)->medium_alloc( sizeclass, rsize, size); return ret.unsafe_capptr; }); - return CapPtr(ret); + return CapPtr(ret); } - slab = large_allocator - .template alloc(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE) - .template as_reinterpret(); - if (slab == nullptr) + auto newslab = + large_allocator + .template alloc(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE) + .template as_reinterpret(); + + if (newslab == nullptr) return nullptr; - Mediumslab::init(slab, public_state(), sizeclass, rsize); - chunkmap().set_slab(slab); + Mediumslab::init(newslab, public_state(), sizeclass, rsize); + chunkmap().set_slab(newslab); p = Mediumslab::alloc( - slab, rsize); + newslab, rsize); - if (!Mediumslab::full(slab)) - sc->insert(slab); + if (!Mediumslab::full(newslab)) + sc->insert(newslab); } stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); + return p; } @@ -1509,17 +1527,17 @@ namespace snmalloc } template - CapPtr large_alloc(size_t size) + CapPtr large_alloc(size_t size) { if (NeedsInitialisation(this)) { // MSVC-vs-CapPtr triggering; xref CapPtr's constructor void* ret = InitThreadAllocator([size](void* alloc) { - CapPtr ret = + CapPtr ret = reinterpret_cast(alloc)->large_alloc(size); return ret.unsafe_capptr; }); - return CapPtr(ret); + return CapPtr(ret); } size_t size_bits = bits::next_pow2_bits(size); @@ -1540,7 +1558,7 @@ namespace snmalloc stats().alloc_request(size); stats().large_alloc(large_class); } - return p.as_void(); + return capptr_export(Aal::capptr_bound(p, rsize)); } void large_dealloc_unchecked( diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index 02028b7..6a11fe0 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -88,7 +88,7 @@ namespace snmalloc } template - static CapPtr + static CapPtr alloc(CapPtr self, size_t size) { SNMALLOC_ASSERT(!full(self)); @@ -102,7 +102,7 @@ namespace snmalloc else UNUSED(size); - return p; + return capptr_export(Aal::capptr_bound(p, size)); } static bool diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index c7f5054..f967e6f 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -158,7 +158,7 @@ namespace snmalloc * `fast_free_list` for further allocations. */ template - static SNMALLOC_FAST_PATH CapPtr alloc( + static SNMALLOC_FAST_PATH CapPtr alloc( CapPtr self, FreeListIter& fast_free_list, size_t rsize, @@ -193,7 +193,7 @@ namespace snmalloc UNUSED(rsize); } - return p; + return capptr_export(Aal::capptr_bound(p, rsize)); } void debug_slab_invariant(CapPtr slab, LocalEntropy& entropy)