diff --git a/src/backend/address_space.h b/src/backend/address_space.h index e8d1483..abccd83 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -71,7 +71,7 @@ namespace snmalloc CapPtr res; { FlagLock lock(spin_lock); - res = core.template reserve(size); + res = core.template reserve(size, pagemap); if (res == nullptr) { // Allocation failed ask OS for more memory @@ -130,10 +130,10 @@ namespace snmalloc pagemap.register_range(address_cast(block), block_size); - core.template add_range(block, block_size); + core.template add_range(block, block_size, pagemap); // still holding lock so guaranteed to succeed. - res = core.template reserve(size); + res = core.template reserve(size, pagemap); } } @@ -167,7 +167,8 @@ namespace snmalloc if (rsize > size) { FlagLock lock(spin_lock); - core.template add_range(pointer_offset(res, size), rsize - size); + core.template add_range( + pointer_offset(res, size), rsize - size, pagemap); } if constexpr (committed) @@ -187,10 +188,11 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - void add_range(CapPtr base, size_t length) + template + void add_range(CapPtr base, size_t length, Pagemap& pagemap) { FlagLock lock(spin_lock); - core.add_range(base, length); + core.add_range(base, length, pagemap); } }; } // namespace snmalloc diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 6f0ccc5..549c317 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -11,9 +11,6 @@ namespace snmalloc { /** - * TODO all comment in this file need revisiting. Core versus locking global - * version. - * * Implements a power of two allocator, where all blocks are aligned to the * same power of two as their size. This is what snmalloc uses to get * alignment of very large sizeclasses. @@ -23,33 +20,36 @@ namespace snmalloc */ class AddressSpaceManagerCore { + struct FreeChunk + { + CapPtr next; + }; + /** * Stores the blocks of address space * - * The first level of array indexes based on power of two size. + * The array indexes based on power of two size. * - * The first entry ranges[n][0] is just a pointer to an address range - * of size 2^n. - * - * The second entry ranges[n][1] is a pointer to a linked list of blocks - * of this size. The final block in the list is not committed, so we commit - * on pop for this corner case. - * - * Invariants - * ranges[n][1] != nullptr => ranges[n][0] != nullptr + * The entries for each size form a linked list. For sizes below + * MIN_CHUNK_SIZE they are linked through the first location in the + * block of memory. For sizes of, and above, MIN_CHUNK_SIZE they are + * linked using the pagemap. We only use the smaller than MIN_CHUNK_SIZE + * allocations for meta-data, so we can be sure that the next pointers + * never occur in a blocks that are ultimately used for object allocations. * * bits::BITS is used for simplicity, we do not use below the pointer size, * and large entries will be unlikely to be supported by the platform. */ - std::array, 2>, bits::BITS> ranges = {}; + std::array, bits::BITS> ranges = {}; /** * Checks a block satisfies its invariant. */ - inline void check_block(CapPtr base, size_t align_bits) + inline void check_block(CapPtr base, size_t align_bits) { SNMALLOC_ASSERT( - base == pointer_align_up(base, bits::one_at_bit(align_bits))); + address_cast(base) == + bits::align_up(address_cast(base), bits::one_at_bit(align_bits))); // All blocks need to be bigger than a pointer. SNMALLOC_ASSERT(bits::one_at_bit(align_bits) >= sizeof(void*)); UNUSED(base); @@ -57,45 +57,83 @@ namespace snmalloc } /** - * Adds a block to `ranges`. + * Set next pointer for a power of two address range. + * + * This abstracts the use of either + * - the pagemap; or + * - the first pointer word of the block + * to store the next pointer for the list of unused address space of a + * particular size. */ - template - void add_block(size_t align_bits, CapPtr base) + template + void set_next( + size_t align_bits, + CapPtr base, + CapPtr next, + Pagemap& pagemap) { - check_block(base, align_bits); - SNMALLOC_ASSERT(align_bits < 64); - if (ranges[align_bits][0] == nullptr) + if (align_bits >= MIN_CHUNK_BITS) { - // Prefer first slot if available. - ranges[align_bits][0] = base; + // The pagemap stores MetaEntrys, abuse the metaslab field to be the + // next block in the stack of blocks. + // + // The pagemap entries here have nullptr (i.e., fake_large_remote) as + // their remote, and so other accesses to the pagemap (by + // external_pointer, for example) will not attempt to follow this + // "Metaslab" pointer. + MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr, 0); + pagemap.set(address_cast(base), t); return; } - if (ranges[align_bits][1] != nullptr) + base->next = next; + } + + /** + * Get next pointer for a power of two address range. + * + * This abstracts the use of either + * - the pagemap; or + * - the first pointer word of the block + * to store the next pointer for the list of unused address space of a + * particular size. + */ + template + CapPtr get_next( + size_t align_bits, CapPtr base, Pagemap& pagemap) + { + if (align_bits >= MIN_CHUNK_BITS) { -#ifdef SNMALLOC_TRACING - std::cout << "Add range linking." << std::endl; -#endif - // Add to linked list. - commit_block(base, sizeof(void*)); - *(base.template as_static>().unsafe_ptr()) = - ranges[align_bits][1]; - check_block(ranges[align_bits][1], align_bits); + const MetaEntry& t = pagemap.template get(address_cast(base)); + return CapPtr( + reinterpret_cast(t.get_metaslab())); } - // Update head of list - ranges[align_bits][1] = base; - check_block(ranges[align_bits][1], align_bits); + return base->next; + } + + /** + * Adds a block to `ranges`. + */ + template + void add_block( + size_t align_bits, CapPtr base, Pagemap& pagemap) + { + check_block(base, align_bits); + SNMALLOC_ASSERT(align_bits < 64); + + set_next(align_bits, base, ranges[align_bits], pagemap); + ranges[align_bits] = base.as_static(); } /** * Find a block of the correct size. May split larger blocks * to satisfy this request. */ - template - CapPtr remove_block(size_t align_bits) + template + CapPtr remove_block(size_t align_bits, Pagemap& pagemap) { - CapPtr first = ranges[align_bits][0]; + CapPtr first = ranges[align_bits]; if (first == nullptr) { if (align_bits == (bits::BITS - 1)) @@ -105,37 +143,34 @@ namespace snmalloc } // Look for larger block and split up recursively - CapPtr bigger = remove_block(align_bits + 1); + CapPtr bigger = + remove_block(align_bits + 1, pagemap); if (bigger != nullptr) { + // This block is going to be broken up into sub CHUNK_SIZE blocks + // so we need to commit it to enable the next pointers to be used + // inside the block. + if ((align_bits + 1) == MIN_CHUNK_BITS) + { + commit_block(bigger, MIN_CHUNK_SIZE); + } + size_t left_over_size = bits::one_at_bit(align_bits); auto left_over = pointer_offset(bigger, left_over_size); - ranges[align_bits][0] = - Aal::capptr_bound(left_over, left_over_size); - check_block(left_over, align_bits); + + add_block( + align_bits, + Aal::capptr_bound(left_over, left_over_size), + pagemap); + check_block(left_over.as_static(), align_bits); } - check_block(bigger, align_bits + 1); + check_block(bigger.as_static(), align_bits + 1); return bigger; } - CapPtr second = ranges[align_bits][1]; - if (second != nullptr) - { - commit_block(second, sizeof(void*)); - auto psecond = - second.template as_static>().unsafe_ptr(); - auto next = *psecond; - ranges[align_bits][1] = next; - // Zero memory. Client assumes memory contains only zeros. - *psecond = nullptr; - check_block(second, align_bits); - check_block(next, align_bits); - return second; - } - check_block(first, align_bits); - ranges[align_bits][0] = nullptr; - return first; + ranges[align_bits] = get_next(align_bits, first, pagemap); + return first.as_void(); } public: @@ -143,9 +178,21 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - template - void add_range(CapPtr base, size_t length) + template + void add_range(CapPtr base, size_t length, Pagemap& pagemap) { + // For start and end that are not chunk sized, we need to + // commit the pages to track the allocations. + auto base_chunk = pointer_align_up(base, MIN_CHUNK_SIZE); + auto end = pointer_offset(base, length); + auto end_chunk = pointer_align_down(end, MIN_CHUNK_SIZE); + auto start_length = pointer_diff(base, base_chunk); + auto end_length = pointer_diff(end_chunk, end); + if (start_length != 0) + commit_block(base, start_length); + if (end_length != 0) + commit_block(end_chunk, end_length); + // Find the minimum set of maximally aligned blocks in this range. // Each block's alignment and size are equal. while (length >= sizeof(void*)) @@ -154,9 +201,10 @@ namespace snmalloc size_t length_align_bits = (bits::BITS - 1) - bits::clz(length); size_t align_bits = bits::min(base_align_bits, length_align_bits); size_t align = bits::one_at_bit(align_bits); + auto b = base.as_static(); - check_block(base, align_bits); - add_block(align_bits, base); + check_block(b, align_bits); + add_block(align_bits, b, pagemap); base = pointer_offset(base, align); length -= align; @@ -188,8 +236,8 @@ namespace snmalloc * part of satisfying the request will be registered with the provided * arena_map for use in subsequent amplification. */ - template - CapPtr reserve(size_t size) + template + CapPtr reserve(size_t size, Pagemap& pagemap) { #ifdef SNMALLOC_TRACING std::cout << "ASM Core reserve request:" << size << std::endl; @@ -198,7 +246,7 @@ namespace snmalloc SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); - return remove_block(bits::next_pow2_bits(size)); + return remove_block(bits::next_pow2_bits(size), pagemap); } /** @@ -208,8 +256,8 @@ namespace snmalloc * This is useful for allowing the space required for alignment to be * used, by smaller objects. */ - template - CapPtr reserve_with_left_over(size_t size) + template + CapPtr reserve_with_left_over(size_t size, Pagemap& pagemap) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -217,13 +265,13 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(rsize); + auto res = reserve(rsize, pagemap); if (res != nullptr) { if (rsize > size) { - add_range(pointer_offset(res, size), rsize - size); + add_range(pointer_offset(res, size), rsize - size, pagemap); } } return res; diff --git a/src/backend/backend.h b/src/backend/backend.h index e17b044..cab9944 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -90,7 +90,8 @@ namespace snmalloc fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); auto [heap_base, heap_length] = pagemap.init(base, length); - address_space.add_range(CapPtr(heap_base), heap_length); + address_space.add_range( + CapPtr(heap_base), heap_length, pagemap); if constexpr (!fixed_range) { @@ -155,7 +156,7 @@ namespace snmalloc auto& local = local_state->local_address_space; #endif - p = local.template reserve_with_left_over(size); + p = local.template reserve_with_left_over(size, h.pagemap); if (p != nullptr) { return p; @@ -174,10 +175,10 @@ namespace snmalloc } #endif PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range(refill, refill_size); + local.template add_range(refill, refill_size, h.pagemap); // This should succeed - return local.template reserve_with_left_over(size); + return local.template reserve_with_left_over(size, h.pagemap); } #ifdef SNMALLOC_CHECK_CLIENT