diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 9dabcc4..0a7187f 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -177,14 +177,14 @@ namespace snmalloc { // Allocations smaller than the slab size are more likely. Improve // branch prediction by placing this case first. - return small_alloc(size); + return small_alloc(size).unsafe_capptr; } - return alloc_not_small(size); + return alloc_not_small(size).unsafe_capptr; } template - SNMALLOC_SLOW_PATH ALLOCATOR void* alloc_not_small(size_t size) + SNMALLOC_SLOW_PATH CapPtr alloc_not_small(size_t size) { handle_message_queue(); @@ -200,8 +200,7 @@ namespace snmalloc return medium_alloc(sizeclass, rsize, size); } - return large_alloc(size).unsafe_capptr; - + return large_alloc(size); #endif } @@ -1040,7 +1039,7 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH void* small_alloc(size_t size) + SNMALLOC_FAST_PATH CapPtr small_alloc(size_t size) { SNMALLOC_ASSUME(size <= SLAB_SIZE); sizeclass_t sizeclass = size_to_sizeclass(size); @@ -1048,7 +1047,7 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH void* + SNMALLOC_FAST_PATH CapPtr small_alloc_inner(sizeclass_t sizeclass, size_t size) { SNMALLOC_ASSUME(sizeclass < NUM_SMALL_CLASSES); @@ -1064,7 +1063,7 @@ namespace snmalloc pal_zero( p, sizeclass_to_size(sizeclass)); } - return p; + return CapPtr(p); } if (likely(!has_messages())) @@ -1078,7 +1077,7 @@ namespace snmalloc * allocation request. */ template - SNMALLOC_SLOW_PATH void* + SNMALLOC_SLOW_PATH CapPtr small_alloc_mq_slow(sizeclass_t sizeclass, size_t size) { handle_message_queue_inner(); @@ -1090,7 +1089,7 @@ namespace snmalloc * Attempt to find a new free list to allocate from */ template - SNMALLOC_SLOW_PATH void* + SNMALLOC_SLOW_PATH CapPtr small_alloc_next_free_list(sizeclass_t sizeclass, size_t size) { size_t rsize = sizeclass_to_size(sizeclass); @@ -1115,7 +1114,7 @@ namespace snmalloc * new free list. */ template - SNMALLOC_SLOW_PATH void* + SNMALLOC_SLOW_PATH CapPtr small_alloc_rare(sizeclass_t sizeclass, size_t size) { if (likely(!NeedsInitialisation(this))) @@ -1132,13 +1131,15 @@ namespace snmalloc * then directs the allocation request to the newly created allocator. */ template - SNMALLOC_SLOW_PATH void* + SNMALLOC_SLOW_PATH CapPtr small_alloc_first_alloc(sizeclass_t sizeclass, size_t size) { - return InitThreadAllocator([sizeclass, size](void* alloc) { + void* ret = InitThreadAllocator([sizeclass, size](void* alloc) { return reinterpret_cast(alloc) - ->template small_alloc_inner(sizeclass, size); + ->template small_alloc_inner(sizeclass, size) + .unsafe_capptr; }); + return CapPtr(ret); } /** @@ -1146,7 +1147,8 @@ namespace snmalloc * list. */ template - SNMALLOC_FAST_PATH void* small_alloc_new_free_list(sizeclass_t sizeclass) + SNMALLOC_FAST_PATH CapPtr + small_alloc_new_free_list(sizeclass_t sizeclass) { auto& bp = bump_ptrs[sizeclass]; if (likely(pointer_align_up(bp, SLAB_SIZE) != bp)) @@ -1162,7 +1164,8 @@ namespace snmalloc * the request from that new list. */ template - SNMALLOC_FAST_PATH void* small_alloc_build_free_list(sizeclass_t sizeclass) + SNMALLOC_FAST_PATH CapPtr + small_alloc_build_free_list(sizeclass_t sizeclass) { auto& bp = bump_ptrs[sizeclass]; auto rsize = sizeclass_to_size(sizeclass); @@ -1177,7 +1180,7 @@ namespace snmalloc { pal_zero(p, sizeclass_to_size(sizeclass)); } - return p; + return CapPtr(p); } /** @@ -1186,7 +1189,8 @@ namespace snmalloc * local bump allocator and service the request from that new list. */ template - SNMALLOC_SLOW_PATH void* small_alloc_new_slab(sizeclass_t sizeclass) + SNMALLOC_SLOW_PATH CapPtr + small_alloc_new_slab(sizeclass_t sizeclass) { auto& bp = bump_ptrs[sizeclass]; // Fetch new slab @@ -1347,7 +1351,8 @@ namespace snmalloc } template - void* medium_alloc(sizeclass_t sizeclass, size_t rsize, size_t size) + CapPtr + medium_alloc(sizeclass_t sizeclass, size_t rsize, size_t size) { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; @@ -1367,10 +1372,20 @@ namespace snmalloc { if (NeedsInitialisation(this)) { - return InitThreadAllocator([size, rsize, sizeclass](void* alloc) { - return reinterpret_cast(alloc)->medium_alloc( - sizeclass, rsize, size); - }); + /* + * We have to convert through void* as part of the thread allocator + * initializer API. Be a little more verbose than strictly necessary + * to demonstrate that small_alloc_inner is giving us an annotated + * pointer before we just go slapping that label on a void* later. + */ + void* ret = + InitThreadAllocator([size, rsize, sizeclass](void* alloc) { + CapPtr ret = + reinterpret_cast(alloc)->medium_alloc( + sizeclass, rsize, size); + return ret.unsafe_capptr; + }); + return CapPtr(ret); } slab = large_allocator .template alloc(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE) @@ -1390,7 +1405,7 @@ namespace snmalloc stats().alloc_request(size); stats().sizeclass_alloc(sizeclass); - return p.unsafe_capptr; + return p; } SNMALLOC_FAST_PATH diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index fb9d993..43404a4 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -158,7 +158,7 @@ namespace snmalloc * `fast_free_list` for further allocations. */ template - static SNMALLOC_FAST_PATH void* alloc( + static SNMALLOC_FAST_PATH CapPtr alloc( CapPtr self, FreeListIter& fast_free_list, size_t rsize, @@ -194,7 +194,7 @@ namespace snmalloc UNUSED(rsize); } - return p; + return CapPtr(p); } void debug_slab_invariant(CapPtr slab, LocalEntropy& entropy)