diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h index 7ccfd1b..9abc35a 100644 --- a/src/mem/largealloc.h +++ b/src/mem/largealloc.h @@ -63,7 +63,8 @@ namespace snmalloc void new_block() { - void* r = reserve(SUPERSLAB_SIZE, SUPERSLAB_SIZE); + // Reserve the smallest large_class which is SUPERSLAB_SIZE + void* r = reserve(0); PAL::template notify_using(r, OS_PAGE_SIZE); bump = r; @@ -121,6 +122,24 @@ namespace snmalloc lazy_decommit_guard.clear(); } + void push_space(address_t start, size_t large_class) + { + void * p = pointer_cast(start); + if (large_class > 0) + PAL::template notify_using(p, OS_PAGE_SIZE); + else + { + if (decommit_strategy == DecommitSuperLazy) + { + PAL::template notify_using(p, OS_PAGE_SIZE); + p = new (p) Decommittedslab(); + } + else + PAL::template notify_using(p, SUPERSLAB_SIZE); + } + large_stack[large_class].push(reinterpret_cast(p)); + } + public: /** * Stack of large allocations that have been returned for reuse. @@ -197,33 +216,76 @@ namespace snmalloc } template - void* reserve(size_t size, size_t align) noexcept + void* reserve(size_t large_class) noexcept { + size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class; + size_t align = size; + if constexpr (pal_supports) { return PAL::template reserve(size, align); } else { - size_t request = size; - // Add align, so we can guarantee to provide at least size. - request += align; - // Alignment must be a power of 2. - assert(align == bits::next_pow2(align)); - + // Reserve 4 times the amount, and put aligned leftovers into the + // large_stack + size_t request = bits::max(size * 4, SUPERSLAB_SIZE * 8); void* p = PAL::template reserve(request); - auto p0 = address_cast(p); - auto start = bits::align_up(p0, align); + address_t p0 = address_cast(p); + address_t start = bits::align_up(p0, align); + address_t p1 = p0 + request; + address_t end = start + size; - if (start > p0) + // Turn off pages for extra allocations. + // Will turn pages back on for stack entries. + if (committed) { - uintptr_t end = bits::align_down(p0 + request, align); - PAL::notify_not_using(p, start - p0); - PAL::notify_not_using(pointer_cast(end), (p0 + request) - end); - p = pointer_cast(start); + if (start - p0 > 0) + { + PAL::notify_not_using(p, start - p0); + } + assert(p1 - end > 0); + PAL::notify_not_using(pointer_cast(end), p1 - end); } - return p; + + for (; end < bits::align_down(p1,align); end += size) + { + push_space(end, large_class); + } + + // Put offcuts before alignment into the large stack + address_t offcut_end = start; + address_t offcut_start; + for (size_t i = large_class; i > 0;) + { + i--; + size_t offcut_align = bits::one_at_bit(SUPERSLAB_BITS) << i; + offcut_start = bits::align_up(p0, offcut_align); + if (offcut_start != offcut_end) + { + push_space(offcut_start, i); + offcut_end = offcut_start; + } + } + + // Put offcuts after returned block into the large stack + offcut_start = end; + for (size_t i = large_class; i > 0;) + { + i--; + auto offcut_align = bits::one_at_bit(SUPERSLAB_BITS) << i; + offcut_end = bits::align_down(p1, offcut_align); + if (offcut_start != offcut_end) + { + push_space(offcut_start, i); + offcut_start = offcut_end; + } + } + + // printf("Alloc %zx (size = %zx)\n", start, size); + + return pointer_cast(start); } } @@ -295,7 +357,7 @@ namespace snmalloc if (p == nullptr) { - p = memory_provider.template reserve(rsize, rsize); + p = memory_provider.template reserve(large_class); memory_provider.template notify_using(p, size); } else diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 101177e..176c5b7 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -32,8 +32,9 @@ int main() { MemoryProviderStateMixin mp; - size_t size = 1ULL << 28; - oe_base = mp.reserve(size, 1); + size_t large_class = 28 - SUPERSLAB_BITS; + size_t size = 1ULL << (SUPERSLAB_BITS + large_class); + oe_base = mp.reserve(large_class); oe_end = (uint8_t*)oe_base + size; std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl; diff --git a/src/test/func/two_alloc_types/main.cc b/src/test/func/two_alloc_types/main.cc index ef9c1b9..39d2b2b 100644 --- a/src/test/func/two_alloc_types/main.cc +++ b/src/test/func/two_alloc_types/main.cc @@ -45,8 +45,9 @@ int main() MemoryProviderStateMixin mp; - size_t size = 1ULL << 26; - oe_base = mp.reserve(size, 1); + size_t large_class = 26 - SUPERSLAB_BITS; + size_t size = 1ULL << (SUPERSLAB_BITS + large_class); + oe_base = mp.reserve(large_class); oe_end = (uint8_t*)oe_base + size; std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;