From 79486b63315753272afa0d0998da89cda6ca9d1f Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 3 Feb 2022 17:17:58 +0000 Subject: [PATCH] Rearrange templates for buddy allocator. --- src/backend/address_space.h | 25 +++++----- src/backend/address_space_core.h | 42 ++++++---------- src/backend/backend.h | 85 ++++++++++++++++---------------- src/test/func/pagemap/pagemap.cc | 2 - 4 files changed, 71 insertions(+), 83 deletions(-) diff --git a/src/backend/address_space.h b/src/backend/address_space.h index d2e29f2..1b3ce40 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -19,10 +19,12 @@ namespace snmalloc * It cannot unreserve memory, so this does not require the * usual complexity of a buddy allocator. */ - template + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL, + SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap> class AddressSpaceManager { - AddressSpaceManagerCore core; + AddressSpaceManagerCore core; /** * This is infrequently used code, a spin lock simplifies the code @@ -42,7 +44,7 @@ namespace snmalloc * part of satisfying the request will be registered with the provided * arena_map for use in subsequent amplification. */ - template + template capptr::Chunk reserve(typename Pagemap::LocalState* local_state, size_t size) { @@ -70,7 +72,7 @@ namespace snmalloc capptr::Chunk res; { FlagLock lock(spin_lock); - res = core.template reserve(local_state, size); + res = core.template reserve(local_state, size); if (res == nullptr) { // Allocation failed ask OS for more memory @@ -134,10 +136,10 @@ namespace snmalloc Pagemap::register_range(local_state, address_cast(block), block_size); - core.template add_range(local_state, block, block_size); + core.template add_range(local_state, block, block_size); // still holding lock so guaranteed to succeed. - res = core.template reserve(local_state, size); + res = core.template reserve(local_state, size); } } @@ -155,7 +157,7 @@ namespace snmalloc * This is useful for allowing the space required for alignment to be * used, by smaller objects. */ - template + template capptr::Chunk reserve_with_left_over( typename Pagemap::LocalState* local_state, size_t size) { @@ -165,19 +167,19 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(local_state, rsize); + auto res = reserve(local_state, rsize); if (res != nullptr) { if (rsize > size) { FlagLock lock(spin_lock); - core.template add_range( + core.template add_range( local_state, pointer_offset(res, size), rsize - size); } if constexpr (committed) - core.commit_block(res, size); + core.template commit_block(res, size); } return res; } @@ -193,14 +195,13 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - template void add_range( typename Pagemap::LocalState* local_state, capptr::Chunk base, size_t length) { FlagLock lock(spin_lock); - core.add_range(local_state, base, length); + core.template add_range(local_state, base, length); } }; } // namespace snmalloc diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index b2396cb..df26b39 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -31,6 +31,7 @@ namespace snmalloc * to Metaslab objects in perpetuity; it could also make {set,get}_next less * scary. */ + template class AddressSpaceManagerCore { struct FreeChunk @@ -77,7 +78,6 @@ namespace snmalloc * to store the next pointer for the list of unused address space of a * particular size. */ - template void set_next( typename Pagemap::LocalState* local_state, size_t align_bits, @@ -112,7 +112,6 @@ namespace snmalloc * to store the next pointer for the list of unused address space of a * particular size. */ - template capptr::Chunk get_next( typename Pagemap::LocalState* local_state, size_t align_bits, @@ -132,9 +131,7 @@ namespace snmalloc /** * Adds a block to `ranges`. */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + template void add_block( typename Pagemap::LocalState* local_state, size_t align_bits, @@ -143,17 +140,15 @@ namespace snmalloc check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); - set_next(local_state, align_bits, base, ranges[align_bits]); - ranges[align_bits] = base.as_static(); + set_next(local_state, align_bits, base, ranges[align_bits]); + ranges[align_bits] = base.template as_static(); } /** * Find a block of the correct size. May split larger blocks * to satisfy this request. */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + template capptr::Chunk remove_block(typename Pagemap::LocalState* local_state, size_t align_bits) { @@ -168,7 +163,7 @@ namespace snmalloc // Look for larger block and split up recursively capptr::Chunk bigger = - remove_block(local_state, align_bits + 1); + remove_block(local_state, align_bits + 1); if (SNMALLOC_UNLIKELY(bigger == nullptr)) return nullptr; @@ -184,7 +179,7 @@ namespace snmalloc size_t half_bigger_size = bits::one_at_bit(align_bits); auto left_over = pointer_offset(bigger, half_bigger_size); - add_block( + add_block( local_state, align_bits, Aal::capptr_bound( @@ -196,7 +191,7 @@ namespace snmalloc } check_block(first, align_bits); - ranges[align_bits] = get_next(local_state, align_bits, first); + ranges[align_bits] = get_next(local_state, align_bits, first); return first.as_void(); } @@ -205,9 +200,7 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + template void add_range( typename Pagemap::LocalState* local_state, capptr::Chunk base, @@ -242,7 +235,7 @@ namespace snmalloc Aal::capptr_bound(base, align); check_block(b, align_bits); - add_block(local_state, align_bits, b); + add_block(local_state, align_bits, b); base = pointer_offset(base, align); length -= align; @@ -274,9 +267,7 @@ namespace snmalloc * part of satisfying the request will be registered with the provided * arena_map for use in subsequent amplification. */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + template capptr::Chunk reserve(typename Pagemap::LocalState* local_state, size_t size) { @@ -287,8 +278,7 @@ namespace snmalloc SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); - return remove_block( - local_state, bits::next_pow2_bits(size)); + return remove_block(local_state, bits::next_pow2_bits(size)); } /** @@ -298,9 +288,7 @@ namespace snmalloc * This is useful for allowing the space required for alignment to be * used by smaller objects. */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + template capptr::Chunk reserve_with_left_over( typename Pagemap::LocalState* local_state, size_t size) { @@ -310,7 +298,7 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(local_state, rsize); + auto res = reserve(local_state, rsize); if (res != nullptr) { @@ -323,7 +311,7 @@ namespace snmalloc size_t residual_size = rsize - size; auto residual = pointer_offset(res, size); res = Aal::capptr_bound(res, size); - add_range(local_state, residual, residual_size); + add_range(local_state, residual, residual_size); } } return res; diff --git a/src/backend/backend.h b/src/backend/backend.h index 42bb760..232445e 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -62,7 +62,9 @@ namespace snmalloc * address space to protect this from corruption. */ static capptr::Chunk alloc_meta_data( - AddressSpaceManager& global, LocalState* local_state, size_t size) + AddressSpaceManager& global, + LocalState* local_state, + size_t size) { return reserve(global, local_state, size); } @@ -77,7 +79,7 @@ namespace snmalloc * where metaslab, is the second element of the pair return. */ static std::pair, Metaslab*> alloc_chunk( - AddressSpaceManager& global, + AddressSpaceManager& global, LocalState* local_state, size_t size, RemoteAllocator* remote, @@ -123,7 +125,9 @@ namespace snmalloc */ template static capptr::Chunk reserve( - AddressSpaceManager& global, LocalState* local_state, size_t size) + AddressSpaceManager& global, + LocalState* local_state, + size_t size) { #ifdef SNMALLOC_META_PROTECTED constexpr auto MAX_CACHED_SIZE = @@ -142,16 +146,14 @@ namespace snmalloc auto& local = local_state->local_address_space; #endif - p = local.template reserve_with_left_over( - local_state, size); + p = local.template reserve_with_left_over(local_state, size); if (p != nullptr) { return p; } auto refill_size = LOCAL_CACHE_BLOCK; - auto refill = - global.template reserve(local_state, refill_size); + auto refill = global.template reserve(local_state, refill_size); if (refill == nullptr) return nullptr; @@ -163,12 +165,10 @@ namespace snmalloc } #endif PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range( - local_state, refill, refill_size); + local.template add_range(local_state, refill, refill_size); // This should succeed - return local.template reserve_with_left_over( - local_state, size); + return local.template reserve_with_left_over(local_state, size); } #ifdef SNMALLOC_META_PROTECTED @@ -179,7 +179,7 @@ namespace snmalloc size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); size_t size_request = rsize * 64; - p = global.template reserve(local_state, size_request); + p = global.template reserve(local_state, size_request); if (p == nullptr) return nullptr; @@ -195,8 +195,7 @@ namespace snmalloc SNMALLOC_ASSERT(!is_meta); #endif - p = global.template reserve_with_left_over( - local_state, size); + p = global.template reserve_with_left_over(local_state, size); return p; } @@ -242,33 +241,7 @@ namespace snmalloc public: using Pal = PAL; - /** - * Local state for the backend allocator. - * - * This class contains thread local structures to make the implementation - * of the backend allocator more efficient. - */ - class LocalState - { - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL2, - typename LocalState, - SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap> - friend class AddressSpaceAllocatorCommon; - - AddressSpaceManagerCore local_address_space; - -#ifdef SNMALLOC_META_PROTECTED - /** - * Secondary local address space, so we can apply some randomisation - * and guard pages to protect the meta-data. - */ - AddressSpaceManagerCore local_meta_address_space; -#endif - }; - - SNMALLOC_REQUIRE_CONSTINIT - static inline AddressSpaceManager address_space; + class LocalState; class Pagemap { @@ -343,6 +316,34 @@ namespace snmalloc } }; + /** + * Local state for the backend allocator. + * + * This class contains thread local structures to make the implementation + * of the backend allocator more efficient. + */ + class LocalState + { + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL2, + typename LocalState, + SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap2> + friend class AddressSpaceAllocatorCommon; + + AddressSpaceManagerCore local_address_space; + +#ifdef SNMALLOC_META_PROTECTED + /** + * Secondary local address space, so we can apply some randomisation + * and guard pages to protect the meta-data. + */ + AddressSpaceManagerCore local_meta_address_space; +#endif + }; + + SNMALLOC_REQUIRE_CONSTINIT + static inline AddressSpaceManager address_space; + private: using AddressSpaceAllocator = AddressSpaceAllocatorCommon; @@ -364,7 +365,7 @@ namespace snmalloc auto [heap_base, heap_length] = Pagemap::concretePagemap.init(base, length); - address_space.template add_range( + address_space.add_range( local_state, capptr::Chunk(heap_base), heap_length); } diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index 410ba8c..74f14c4 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -21,8 +21,6 @@ struct T T() {} }; -AddressSpaceManager address_space; - FlatPagemap pagemap_test_unbound; FlatPagemap pagemap_test_bound;