diff --git a/src/backend/address_space.h b/src/backend/address_space.h deleted file mode 100644 index 15c1ae3..0000000 --- a/src/backend/address_space.h +++ /dev/null @@ -1,201 +0,0 @@ -#pragma once -#include "../ds/address.h" -#include "../ds/flaglock.h" -#include "../pal/pal.h" -#include "address_space_core.h" - -#include -#ifdef SNMALLOC_TRACING -# include -#endif - -namespace snmalloc -{ - /** - * Implements a power of two allocator, where all blocks are aligned to the - * same power of two as their size. This is what snmalloc uses to get - * alignment of very large sizeclasses. - * - * It cannot unreserve memory, so this does not require the - * usual complexity of a buddy allocator. - */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap> - class AddressSpaceManager - { - AddressSpaceManagerCore core; - - /** - * This is infrequently used code, a spin lock simplifies the code - * considerably, and should never be on the fast path. - */ - FlagWord spin_lock{}; - - public: - /** - * Returns a pointer to a block of memory of the supplied size. - * The block will be committed, if specified by the template parameter. - * The returned block is guaranteed to be aligened to the size. - * - * Only request 2^n sizes, and not less than a pointer. - * - * On StrictProvenance architectures, any underlying allocations made as - * part of satisfying the request will be registered with the provided - * arena_map for use in subsequent amplification. - */ - template - capptr::Chunk reserve(size_t size) - { -#ifdef SNMALLOC_TRACING - std::cout << "ASM reserve request:" << size << std::endl; -#endif - SNMALLOC_ASSERT(bits::is_pow2(size)); - SNMALLOC_ASSERT(size >= sizeof(void*)); - - /* - * For sufficiently large allocations with platforms that support aligned - * allocations, try asking the platform directly. - */ - if constexpr (pal_supports) - { - if (size >= PAL::minimum_alloc_size) - { - auto base = - capptr::Chunk(PAL::template reserve_aligned(size)); - Pagemap::register_range(address_cast(base), size); - return base; - } - } - - capptr::Chunk res; - { - FlagLock lock(spin_lock); - res = core.template reserve(size); - if (res == nullptr) - { - // Allocation failed ask OS for more memory - capptr::Chunk block = nullptr; - size_t block_size = 0; - if constexpr (pal_supports) - { - /* - * We will have handled the case where size >= - * minimum_alloc_size above, so we are left to handle only small - * things here. - */ - block_size = PAL::minimum_alloc_size; - - void* block_raw = PAL::template reserve_aligned(block_size); - - // It's a bit of a lie to convert without applying bounds, but the - // platform will have bounded block for us and it's better that - // the rest of our internals expect CBChunk bounds. - block = capptr::Chunk(block_raw); - } - else if constexpr (!pal_supports) - { - // Need at least 2 times the space to guarantee alignment. - bool overflow; - size_t needed_size = bits::umul(size, 2, overflow); - if (overflow) - { - return nullptr; - } - // Magic number (27) for over-allocating a block of memory - // These should be further refined based on experiments. - constexpr size_t min_size = bits::one_at_bit(27); - for (size_t size_request = bits::max(needed_size, min_size); - size_request >= needed_size; - size_request = size_request / 2) - { - block = capptr::Chunk(PAL::reserve(size_request)); - if (block != nullptr) - { - block_size = size_request; - break; - } - } - - // Ensure block is pointer aligned. - if ( - pointer_align_up(block, sizeof(void*)) != block || - bits::align_up(block_size, sizeof(void*)) > block_size) - { - auto diff = - pointer_diff(block, pointer_align_up(block, sizeof(void*))); - block_size = block_size - diff; - block_size = bits::align_down(block_size, sizeof(void*)); - } - } - if (block == nullptr) - { - return nullptr; - } - - Pagemap::register_range(address_cast(block), block_size); - - core.template add_range(block, block_size); - - // still holding lock so guaranteed to succeed. - res = core.template reserve(size); - } - } - - // Don't need lock while committing pages. - if constexpr (committed) - core.template commit_block(res, size); - - return res; - } - - /** - * Aligns block to next power of 2 above size, and unused space at the end - * of the block is retained by the address space manager. - * - * This is useful for allowing the space required for alignment to be - * used, by smaller objects. - */ - template - capptr::Chunk reserve_with_left_over(size_t size) - { - SNMALLOC_ASSERT(size >= sizeof(void*)); - - size = bits::align_up(size, sizeof(void*)); - - size_t rsize = bits::next_pow2(size); - - auto res = reserve(rsize); - - if (res != nullptr) - { - if (rsize > size) - { - FlagLock lock(spin_lock); - core.template add_range(pointer_offset(res, size), rsize - size); - } - - if constexpr (committed) - core.template commit_block(res, size); - } - return res; - } - - /** - * Default constructor. An address-space manager constructed in this way - * does not own any memory at the start and will request any that it needs - * from the PAL. - */ - AddressSpaceManager() = default; - - /** - * Add a range of memory to the address space. - * Divides blocks into power of two sizes with natural alignment - */ - void add_range(capptr::Chunk base, size_t length) - { - FlagLock lock(spin_lock); - core.template add_range(base, length); - } - }; -} // namespace snmalloc diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h deleted file mode 100644 index 533b173..0000000 --- a/src/backend/address_space_core.h +++ /dev/null @@ -1,314 +0,0 @@ -#pragma once -#include "../ds/address.h" -#include "../ds/flaglock.h" -#include "../mem/allocconfig.h" -#include "../mem/metaslab.h" -#include "../pal/pal.h" -#include "backend_concept.h" - -#include -#ifdef SNMALLOC_TRACING -# include -#endif - -namespace snmalloc -{ - /** - * Implements a power of two allocator, where all blocks are aligned to the - * same power of two as their size. This is what snmalloc uses to get - * alignment of very large sizeclasses. - * - * It cannot unreserve memory, so this does not require the - * usual complexity of a buddy allocator. - * - * TODO: This manages pieces of memory smaller than (1U << MIN_CHUNK_BITS) to - * source Metaslab and LocalCache objects. On CHERI, where ASLR and guard - * pages are not needed, it may be worth switching to a design where we - * bootstrap allocators with at least two embedded Metaslab-s that can be used - * to construct slabs for LocalCache and, of course, additional Metaslab - * objects. That would let us stop splitting memory below that threshold - * here, and may reduce address space fragmentation or address space committed - * to Metaslab objects in perpetuity; it could also make {set,get}_next less - * scary. - */ - template - class AddressSpaceManagerCore - { - struct FreeChunk - { - capptr::Chunk next; - }; - - /** - * Stores the blocks of address space - * - * The array indexes based on power of two size. - * - * The entries for each size form a linked list. For sizes below - * MIN_CHUNK_SIZE they are linked through the first location in the - * block of memory. For sizes of, and above, MIN_CHUNK_SIZE they are - * linked using the pagemap. We only use the smaller than MIN_CHUNK_SIZE - * allocations for meta-data, so we can be sure that the next pointers - * never occur in a blocks that are ultimately used for object allocations. - * - * bits::BITS is used for simplicity, we do not use below the pointer size, - * and large entries will be unlikely to be supported by the platform. - */ - std::array, bits::BITS> ranges = {}; - - /** - * Checks a block satisfies its invariant. - */ - inline void check_block(capptr::Chunk base, size_t align_bits) - { - SNMALLOC_ASSERT( - address_cast(base) == - bits::align_up(address_cast(base), bits::one_at_bit(align_bits))); - // All blocks need to be bigger than a pointer. - SNMALLOC_ASSERT(bits::one_at_bit(align_bits) >= sizeof(void*)); - UNUSED(base, align_bits); - } - - /** - * Set next pointer for a power of two address range. - * - * This abstracts the use of either - * - the pagemap; or - * - the first pointer word of the block - * to store the next pointer for the list of unused address space of a - * particular size. - */ - void set_next( - size_t align_bits, - capptr::Chunk base, - capptr::Chunk next) - { - if (align_bits >= MIN_CHUNK_BITS) - { - // The pagemap stores `MetaEntry`s; abuse the metaslab field to be the - // next block in the stack of blocks. - // - // The pagemap entries here have nullptr as their remote, and so other - // accesses to the pagemap (by external_pointer, for example) will not - // attempt to follow this "Metaslab" pointer. - // - // dealloc() can reject attempts to free such MetaEntry-s due to the - // zero sizeclass. - MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr); - Pagemap::set_metaentry(address_cast(base), 1, t); - return; - } - - base->next = next; - } - - /** - * Get next pointer for a power of two address range. - * - * This abstracts the use of either - * - the pagemap; or - * - the first pointer word of the block - * to store the next pointer for the list of unused address space of a - * particular size. - */ - capptr::Chunk - get_next(size_t align_bits, capptr::Chunk base) - { - if (align_bits >= MIN_CHUNK_BITS) - { - const MetaEntry& t = - Pagemap::template get_metaentry(address_cast(base)); - return capptr::Chunk( - reinterpret_cast(t.get_metaslab_no_remote())); - } - - return base->next; - } - - /** - * Adds a block to `ranges`. - */ - template - void add_block(size_t align_bits, capptr::Chunk base) - { - check_block(base, align_bits); - SNMALLOC_ASSERT(align_bits < 64); - - set_next(align_bits, base, ranges[align_bits]); - ranges[align_bits] = base.template as_static(); - } - - /** - * Find a block of the correct size. May split larger blocks - * to satisfy this request. - */ - template - capptr::Chunk remove_block(size_t align_bits) - { - capptr::Chunk first = ranges[align_bits]; - if (first == nullptr) - { - if (align_bits == (bits::BITS - 1)) - { - // Out of memory - errno = ENOMEM; - return nullptr; - } - - // Look for larger block and split up recursively - capptr::Chunk bigger = remove_block(align_bits + 1); - - if (SNMALLOC_UNLIKELY(bigger == nullptr)) - return nullptr; - - // This block is going to be broken up into sub CHUNK_SIZE blocks - // so we need to commit it to enable the next pointers to be used - // inside the block. - if ((align_bits + 1) == MIN_CHUNK_BITS) - { - commit_block(bigger, MIN_CHUNK_SIZE); - } - - size_t half_bigger_size = bits::one_at_bit(align_bits); - auto left_over = pointer_offset(bigger, half_bigger_size); - - add_block( - align_bits, - Aal::capptr_bound( - left_over, half_bigger_size)); - check_block(left_over.as_static(), align_bits); - check_block(bigger.as_static(), align_bits); - return Aal::capptr_bound( - bigger, half_bigger_size); - } - - check_block(first, align_bits); - ranges[align_bits] = get_next(align_bits, first); - return first.as_void(); - } - - public: - /** - * Add a range of memory to the address space. - * Divides blocks into power of two sizes with natural alignment - */ - template - void add_range(capptr::Chunk base, size_t length) - { - // For start and end that are not chunk sized, we need to - // commit the pages to track the allocations. - auto base_chunk = pointer_align_up(base, MIN_CHUNK_SIZE); - auto end = pointer_offset(base, length); - auto end_chunk = pointer_align_down(end, MIN_CHUNK_SIZE); - auto start_length = pointer_diff(base, base_chunk); - auto end_length = pointer_diff(end_chunk, end); - if (start_length != 0) - commit_block(base, start_length); - if (end_length != 0) - commit_block(end_chunk, end_length); - - // Find the minimum set of maximally aligned blocks in this range. - // Each block's alignment and size are equal. - while (length >= sizeof(void*)) - { - size_t base_align_bits = bits::ctz(address_cast(base)); - size_t length_align_bits = (bits::BITS - 1) - bits::clz(length); - size_t align_bits = bits::min(base_align_bits, length_align_bits); - size_t align = bits::one_at_bit(align_bits); - - /* - * Now that we have found a maximally-aligned block, we can set bounds - * and be certain that we won't hit representation imprecision. - */ - auto b = - Aal::capptr_bound(base, align); - - check_block(b, align_bits); - add_block(align_bits, b); - - base = pointer_offset(base, align); - length -= align; - } - } - - /** - * Commit a block of memory - */ - template - void commit_block(capptr::Chunk base, size_t size) - { - // Rounding required for sub-page allocations. - auto page_start = pointer_align_down(base); - auto page_end = - pointer_align_up(pointer_offset(base, size)); - size_t using_size = pointer_diff(page_start, page_end); - PAL::template notify_using(page_start.unsafe_ptr(), using_size); - } - - /** - * Returns a pointer to a block of memory of the supplied size. - * The block will be committed, if specified by the template parameter. - * The returned block is guaranteed to be aligened to the size. - * - * Only request 2^n sizes, and not less than a pointer. - * - * On StrictProvenance architectures, any underlying allocations made as - * part of satisfying the request will be registered with the provided - * arena_map for use in subsequent amplification. - */ - template - capptr::Chunk reserve(size_t size) - { -#ifdef SNMALLOC_TRACING - std::cout << "ASM Core reserve request:" << size << std::endl; -#endif - - SNMALLOC_ASSERT(bits::is_pow2(size)); - SNMALLOC_ASSERT(size >= sizeof(void*)); - - return remove_block(bits::next_pow2_bits(size)); - } - - /** - * Aligns block to next power of 2 above size, and unused space at the end - * of the block is retained by the address space manager. - * - * This is useful for allowing the space required for alignment to be - * used by smaller objects. - */ - template - capptr::Chunk reserve_with_left_over(size_t size) - { - SNMALLOC_ASSERT(size >= sizeof(void*)); - - size = bits::align_up(size, sizeof(void*)); - - size_t rsize = bits::next_pow2(size); - - auto res = reserve(rsize); - - if (res != nullptr) - { - if (rsize > size) - { - /* - * Set bounds on the allocation requested but leave the residual with - * wider bounds for the moment; we'll fix that above in add_range. - */ - size_t residual_size = rsize - size; - auto residual = pointer_offset(res, size); - res = Aal::capptr_bound(res, size); - add_range(residual, residual_size); - } - } - return res; - } - - /** - * Default constructor. An address-space manager constructed in this way - * does not own any memory at the start and will request any that it needs - * from the PAL. - */ - AddressSpaceManagerCore() = default; - }; -} // namespace snmalloc diff --git a/src/backend/backend.h b/src/backend/backend.h index e7cff28..c84e00c 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -2,9 +2,19 @@ #include "../mem/allocconfig.h" #include "../mem/metaslab.h" #include "../pal/pal.h" -#include "address_space.h" +#include "chunkallocator.h" +#include "commitrange.h" #include "commonconfig.h" +#include "empty_range.h" +#include "globalrange.h" +#include "largebuddyrange.h" #include "pagemap.h" +#include "pagemapregisterrange.h" +#include "palrange.h" +#include "range_helpers.h" +#include "smallbuddyrange.h" +#include "statsrange.h" +#include "subrange.h" #if defined(SNMALLOC_CHECK_CLIENT) && !defined(OPEN_ENCLAVE) /** @@ -21,213 +31,6 @@ namespace snmalloc { - /** - * This helper class implements the core functionality to allocate from an - * address space and pagemap. Any backend implementation can use this class to - * help with basic address space managment. - */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - typename LocalState, - SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap> - class AddressSpaceAllocatorCommon - { - // Size of local address space requests. Currently aimed at 2MiB large - // pages but should make this configurable (i.e. for OE, so we don't need as - // much space). -#ifdef OPEN_ENCLAVE - // Don't prefetch address space on OE, as it is limited. - // This could cause perf issues during warm-up phases. - constexpr static size_t LOCAL_CACHE_BLOCK = 0; -#else - constexpr static size_t LOCAL_CACHE_BLOCK = bits::one_at_bit(21); -#endif - -#ifdef SNMALLOC_META_PROTECTED - // When protecting the meta-data, we use a smaller block for the meta-data - // that is randomised inside a larger block. This needs to be at least a - // page so that we can use guard pages. - constexpr static size_t LOCAL_CACHE_META_BLOCK = - bits::max(MIN_CHUNK_SIZE * 2, OS_PAGE_SIZE); - static_assert( - LOCAL_CACHE_META_BLOCK <= LOCAL_CACHE_BLOCK, - "LOCAL_CACHE_META_BLOCK must be smaller than LOCAL_CACHE_BLOCK"); -#endif - - public: - /** - * Provide a block of meta-data with size and align. - * - * Backend allocator may use guard pages and separate area of - * address space to protect this from corruption. - */ - static capptr::Chunk alloc_meta_data( - AddressSpaceManager& global, - LocalState* local_state, - size_t size) - { - return reserve(global, local_state, size); - } - - /** - * Returns a chunk of memory with alignment and size of `size`, and a - * metaslab block. - * - * It additionally set the meta-data for this chunk of memory to - * be - * (remote, sizeclass, metaslab) - * where metaslab, is the second element of the pair return. - */ - static std::pair, Metaslab*> alloc_chunk( - AddressSpaceManager& global, - LocalState* local_state, - size_t size, - RemoteAllocator* remote, - sizeclass_t sizeclass) - { - SNMALLOC_ASSERT(bits::is_pow2(size)); - SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); - - auto meta = reinterpret_cast( - reserve(global, local_state, sizeof(Metaslab)).unsafe_ptr()); - - if (meta == nullptr) - return {nullptr, nullptr}; - - capptr::Chunk p = reserve(global, local_state, size); - -#ifdef SNMALLOC_TRACING - std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" - << std::endl; -#endif - if (p == nullptr) - { - // TODO: This is leaking `meta`. Currently there is no facility for - // meta-data reuse, so will leave until we develop more expressive - // meta-data management. -#ifdef SNMALLOC_TRACING - std::cout << "Out of memory" << std::endl; -#endif - return {p, nullptr}; - } - - meta->meta_common.chunk = p; - - MetaEntry t(meta, remote, sizeclass); - Pagemap::set_metaentry(address_cast(p), size, t); - return {p, meta}; - } - - private: - /** - * Internal method for acquiring state from the local and global address - * space managers. - */ - template - static capptr::Chunk reserve( - AddressSpaceManager& global, - LocalState* local_state, - size_t size) - { -#ifdef SNMALLOC_META_PROTECTED - constexpr auto MAX_CACHED_SIZE = - is_meta ? LOCAL_CACHE_META_BLOCK : LOCAL_CACHE_BLOCK; -#else - constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK; -#endif - - capptr::Chunk p; - if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) - { -#ifdef SNMALLOC_META_PROTECTED - auto& local = is_meta ? local_state->local_meta_address_space : - local_state->local_address_space; -#else - auto& local = local_state->local_address_space; -#endif - - p = local.template reserve_with_left_over(size); - if (p != nullptr) - { - return p; - } - - auto refill_size = LOCAL_CACHE_BLOCK; - auto refill = global.template reserve(refill_size); - if (refill == nullptr) - return nullptr; - -#ifdef SNMALLOC_META_PROTECTED - if (is_meta) - { - refill = sub_range(refill, LOCAL_CACHE_BLOCK, LOCAL_CACHE_META_BLOCK); - refill_size = LOCAL_CACHE_META_BLOCK; - } -#endif - PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range(refill, refill_size); - - // This should succeed - return local.template reserve_with_left_over(size); - } - -#ifdef SNMALLOC_META_PROTECTED - // During start up we need meta-data before we have a local allocator - // This code protects that meta-data with randomisation, and guard pages. - if (local_state == nullptr && is_meta) - { - size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); - size_t size_request = rsize * 64; - - p = global.template reserve(size_request); - if (p == nullptr) - return nullptr; - - p = sub_range(p, size_request, rsize); - - PAL::template notify_using(p.unsafe_ptr(), rsize); - return p; - } - - // This path does not apply any guard pages to very large - // meta data requests. There are currently no meta data-requests - // this large. This assert checks for this assumption breaking. - SNMALLOC_ASSERT(!is_meta); -#endif - - p = global.template reserve_with_left_over(size); - return p; - } - -#ifdef SNMALLOC_META_PROTECTED - /** - * Returns a sub-range of [return, return+sub_size] that is contained in - * the range [base, base+full_size]. The first and last slot are not used - * so that the edges can be used for guard pages. - */ - static capptr::Chunk - sub_range(capptr::Chunk base, size_t full_size, size_t sub_size) - { - SNMALLOC_ASSERT(bits::is_pow2(full_size)); - SNMALLOC_ASSERT(bits::is_pow2(sub_size)); - SNMALLOC_ASSERT(full_size % sub_size == 0); - SNMALLOC_ASSERT(full_size / sub_size >= 4); - - size_t offset_mask = full_size - sub_size; - - // Don't use first or last block in the larger reservation - // Loop required to get uniform distribution. - size_t offset; - do - { - offset = get_entropy64() & offset_mask; - } while ((offset == 0) || (offset == offset_mask)); - - return pointer_offset(base, offset); - } -#endif - }; - /** * This class implements the standard backend for handling allocations. * It abstracts page table management and address space management. @@ -278,7 +81,7 @@ namespace snmalloc * Set the metadata associated with a chunk. */ SNMALLOC_FAST_PATH - static void set_metaentry(address_t p, size_t size, MetaEntry t) + static void set_metaentry(address_t p, size_t size, const MetaEntry& t) { for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) { @@ -313,38 +116,70 @@ namespace snmalloc } }; - /** - * Local state for the backend allocator. - * - * This class contains thread local structures to make the implementation - * of the backend allocator more efficient. - */ - class LocalState - { - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL2, - typename LocalState, - SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap2> - friend class AddressSpaceAllocatorCommon; +#if defined(_WIN32) || defined(__CHERI_PURE_CAPABILITY__) + static constexpr bool CONSOLIDATE_PAL_ALLOCS = false; +#else + static constexpr bool CONSOLIDATE_PAL_ALLOCS = true; +#endif - AddressSpaceManagerCore local_address_space; +#if defined(OPEN_ENCLAVE) + // Single global buddy allocator is used on open enclave due to + // the limited address space. + using GlobalR = GlobalRange>>>; + using ObjectRange = GlobalR; + using GlobalMetaRange = ObjectRange; +#else + // Set up source of memory + using P = PalRange; + using Base = std:: + conditional_t>; + // Global range of memory + using StatsR = StatsRange>; + using GlobalR = GlobalRange; + +# ifdef SNMALLOC_META_PROTECTED + // Source for object allocations + using ObjectRange = + LargeBuddyRange, 21, 21, Pagemap>; + // Set up protected range for metadata + using SubR = CommitRange, DefaultPal>; + using MetaRange = + SmallBuddyRange>; + using GlobalMetaRange = GlobalRange; +# else + // Source for object allocations and metadata + // No separation between the two + using ObjectRange = SmallBuddyRange< + LargeBuddyRange, 21, 21, Pagemap>>; + using GlobalMetaRange = GlobalRange; +# endif +#endif + + struct LocalState + { + typename ObjectRange::State object_range; #ifdef SNMALLOC_META_PROTECTED - /** - * Secondary local address space, so we can apply some randomisation - * and guard pages to protect the meta-data. - */ - AddressSpaceManagerCore local_meta_address_space; + typename MetaRange::State meta_range; + + typename MetaRange::State& get_meta_range() + { + return meta_range; + } +#else + typename ObjectRange::State& get_meta_range() + { + return object_range; + } #endif }; - SNMALLOC_REQUIRE_CONSTINIT - static inline AddressSpaceManager address_space; - - private: - using AddressSpaceAllocator = - AddressSpaceAllocatorCommon; - public: template static std::enable_if_t init() @@ -361,7 +196,17 @@ namespace snmalloc auto [heap_base, heap_length] = Pagemap::concretePagemap.init(base, length); - address_space.add_range(capptr::Chunk(heap_base), heap_length); + + Pagemap::register_range(address_cast(heap_base), heap_length); + + // Push memory into the global range. + range_to_pow_2_blocks( + capptr::Chunk(heap_base), + heap_length, + [&](capptr::Chunk p, size_t sz, bool) { + typename GlobalR::State g; + g->dealloc_range(p, sz); + }); } /** @@ -378,8 +223,21 @@ namespace snmalloc static capptr::Chunk alloc_meta_data(LocalState* local_state, size_t size) { - return AddressSpaceAllocator::alloc_meta_data( - address_space, local_state, size); + capptr::Chunk p; + if (local_state != nullptr) + { + p = local_state->get_meta_range()->alloc_range_with_leftover(size); + } + else + { + typename GlobalMetaRange::State global_state; + p = global_state->alloc_range(bits::next_pow2(size)); + } + + if (p == nullptr) + errno = ENOMEM; + + return p; } /** @@ -392,13 +250,73 @@ namespace snmalloc * where metaslab, is the second element of the pair return. */ static std::pair, Metaslab*> alloc_chunk( - LocalState* local_state, + LocalState& local_state, size_t size, RemoteAllocator* remote, sizeclass_t sizeclass) { - return AddressSpaceAllocator::alloc_chunk( - address_space, local_state, size, remote, sizeclass); + SNMALLOC_ASSERT(bits::is_pow2(size)); + SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); + + auto meta_cap = + local_state.get_meta_range()->alloc_range(sizeof(Metaslab)); + + auto meta = meta_cap.template as_reinterpret().unsafe_ptr(); + + if (meta == nullptr) + { + errno = ENOMEM; + return {nullptr, nullptr}; + } + + auto p = local_state.object_range->alloc_range(size); + +#ifdef SNMALLOC_TRACING + std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" + << std::endl; +#endif + if (p == nullptr) + { + local_state.get_meta_range()->dealloc_range(meta_cap, sizeof(Metaslab)); + errno = ENOMEM; +#ifdef SNMALLOC_TRACING + std::cout << "Out of memory" << std::endl; +#endif + return {p, nullptr}; + } + + meta->meta_common.chunk = p; + + MetaEntry t(meta, remote, sizeclass); + Pagemap::set_metaentry(address_cast(p), size, t); + + p = Aal::capptr_bound(p, size); + return {p, meta}; + } + + static void dealloc_chunk( + LocalState& local_state, ChunkRecord* chunk_record, size_t size) + { + auto chunk = chunk_record->meta_common.chunk; + + local_state.get_meta_range()->dealloc_range( + capptr::Chunk(chunk_record), sizeof(Metaslab)); + + // TODO, should we set the sizeclass to something specific here? + + local_state.object_range->dealloc_range(chunk, size); + } + + static size_t get_current_usage() + { + typename StatsR::State stats_state; + return stats_state->get_current_usage(); + } + + static size_t get_peak_usage() + { + typename StatsR::State stats_state; + return stats_state->get_peak_usage(); } }; } // namespace snmalloc diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h index 9aac9ff..895e19a 100644 --- a/src/backend/backend_concept.h +++ b/src/backend/backend_concept.h @@ -4,7 +4,7 @@ # include # include "../ds/concept.h" # include "../pal/pal_concept.h" - +# include "../ds/address.h" namespace snmalloc { class MetaEntry; @@ -20,7 +20,7 @@ namespace snmalloc requires( address_t addr, size_t sz, - MetaEntry t) + const MetaEntry& t) { { Meta::set_metaentry(addr, sz, t) } -> ConceptSame; diff --git a/src/backend/buddy.h b/src/backend/buddy.h new file mode 100644 index 0000000..7e94c84 --- /dev/null +++ b/src/backend/buddy.h @@ -0,0 +1,122 @@ +#pragma once + +#include "../ds/address.h" +#include "../ds/bits.h" +#include "../ds/redblacktree.h" + +namespace snmalloc +{ + /** + * Class representing a buddy allocator + * + * Underlying node `Rep` representation is passed in. + * + * The allocator can handle blocks between inclusive MIN_SIZE_BITS and + * exclusive MAX_SIZE_BITS. + */ + template + class Buddy + { + std::array, MAX_SIZE_BITS - MIN_SIZE_BITS> trees; + + size_t to_index(size_t size) + { + auto log = snmalloc::bits::next_pow2_bits(size); + SNMALLOC_ASSERT(log >= MIN_SIZE_BITS); + SNMALLOC_ASSERT(log < MAX_SIZE_BITS); + + return log - MIN_SIZE_BITS; + } + + void validate_block(typename Rep::Contents addr, size_t size) + { + SNMALLOC_ASSERT(bits::is_pow2(size)); + SNMALLOC_ASSERT(addr == Rep::align_down(addr, size)); + UNUSED(addr, size); + } + + public: + constexpr Buddy() = default; + /** + * Add a block to the buddy allocator. + * + * Blocks needs to be power of two size and aligned to the same power of + * two. + * + * Returns null, if the block is successfully added. Otherwise, returns the + * consolidated block that is MAX_SIZE_BITS big, and hence too large for + * this allocator. + */ + typename Rep::Contents add_block(typename Rep::Contents addr, size_t size) + { + auto idx = to_index(size); + + validate_block(addr, size); + + auto buddy = Rep::buddy(addr, size); + + auto path = trees[idx].get_root_path(); + bool contains_buddy = trees[idx].find(path, buddy); + + if (contains_buddy) + { + // Only check if we can consolidate after we know the buddy is in + // the buddy allocator. This is required to prevent possible segfaults + // from looking at the buddies meta-data, which we only know exists + // once we have found it in the red-black tree. + if (Rep::can_consolidate(addr, size)) + { + trees[idx].remove_path(path); + + // Add to next level cache + size *= 2; + addr = Rep::align_down(addr, size); + if (size == bits::one_at_bit(MAX_SIZE_BITS)) + // Too big for this buddy allocator. + return addr; + return add_block(addr, size); + } + + // Re-traverse as the path was to the buddy, + // but the representation says we cannot combine. + // We must find the correct place for this element. + // Something clever could be done here, but it's not worth it. + // path = trees[idx].get_root_path(); + trees[idx].find(path, addr); + } + trees[idx].insert_path(path, addr); + return Rep::null; + } + + /** + * Removes a block of size from the buddy allocator. + * + * Return Rep::null if this cannot be satisfied. + */ + typename Rep::Contents remove_block(size_t size) + { + auto idx = to_index(size); + + auto addr = trees[idx].remove_min(); + if (addr != Rep::null) + { + validate_block(addr, size); + return addr; + } + + if (size * 2 == bits::one_at_bit(MAX_SIZE_BITS)) + // Too big for this buddy allocator + return Rep::null; + + auto bigger = remove_block(size * 2); + if (bigger == Rep::null) + return Rep::null; + + auto second = Rep::offset(bigger, size); + + // Split large block + add_block(second, size); + return bigger; + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/mem/chunkallocator.h b/src/backend/chunkallocator.h similarity index 97% rename from src/mem/chunkallocator.h rename to src/backend/chunkallocator.h index bd21034..09e4fff 100644 --- a/src/mem/chunkallocator.h +++ b/src/backend/chunkallocator.h @@ -1,5 +1,11 @@ #pragma once +/*** + * WARNING: This file is not currently in use. The functionality has not + * be transistioned to the new backend. It does not seem to be required + * but further testing is required before we delete it. + */ + #include "../backend/backend_concept.h" #include "../ds/mpmcstack.h" #include "../ds/spmcstack.h" @@ -316,7 +322,10 @@ namespace snmalloc SharedStateHandle::template alloc_meta_data(local_state, size); if (p == nullptr) + { + errno = ENOMEM; return nullptr; + } return new (p.unsafe_ptr()) U(std::forward(args)...); } diff --git a/src/backend/commitrange.h b/src/backend/commitrange.h new file mode 100644 index 0000000..a0032d6 --- /dev/null +++ b/src/backend/commitrange.h @@ -0,0 +1,44 @@ +#pragma once + +#include "../ds/ptrwrap.h" + +namespace snmalloc +{ + template + class CommitRange + { + typename ParentRange::State parent{}; + + public: + class State + { + CommitRange commit_range{}; + + public: + constexpr State() = default; + + CommitRange* operator->() + { + return &commit_range; + } + }; + + static constexpr bool Aligned = ParentRange::Aligned; + + constexpr CommitRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + auto range = parent->alloc_range(size); + if (range != nullptr) + PAL::template notify_using(range.unsafe_ptr(), size); + return range; + } + + void dealloc_range(capptr::Chunk base, size_t size) + { + PAL::notify_not_using(base.unsafe_ptr(), size); + parent->dealloc_range(base, size); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/commonconfig.h b/src/backend/commonconfig.h index 74f949d..66314db 100644 --- a/src/backend/commonconfig.h +++ b/src/backend/commonconfig.h @@ -1,6 +1,8 @@ #pragma once +#include "../backend/backend_concept.h" #include "../ds/defines.h" +#include "../mem/remoteallocator.h" namespace snmalloc { diff --git a/src/backend/empty_range.h b/src/backend/empty_range.h new file mode 100644 index 0000000..5feabd9 --- /dev/null +++ b/src/backend/empty_range.h @@ -0,0 +1,29 @@ +#include "../ds/ptrwrap.h" + +namespace snmalloc +{ + class EmptyRange + { + public: + class State + { + public: + EmptyRange* operator->() + { + static EmptyRange range{}; + return ⦥ + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = true; + + constexpr EmptyRange() = default; + + capptr::Chunk alloc_range(size_t) + { + return nullptr; + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h index a0fe819..9bb33ae 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -1,7 +1,7 @@ #pragma once #include "../backend/backend.h" -#include "../mem/chunkallocator.h" +#include "../backend/chunkallocator.h" #include "../mem/corealloc.h" #include "../mem/pool.h" #include "commonconfig.h" @@ -19,17 +19,10 @@ namespace snmalloc private: using Backend = BackendAllocator; - inline static ChunkAllocatorState chunk_allocator_state; inline static GlobalPoolState alloc_pool; public: - static ChunkAllocatorState& - get_chunk_allocator_state(typename Backend::LocalState* = nullptr) - { - return chunk_allocator_state; - } - static GlobalPoolState& pool() { return alloc_pool; diff --git a/src/backend/globalconfig.h b/src/backend/globalconfig.h index b1243a5..3b46856 100644 --- a/src/backend/globalconfig.h +++ b/src/backend/globalconfig.h @@ -1,7 +1,7 @@ #pragma once #include "../backend/backend.h" -#include "../mem/chunkallocator.h" +#include "../backend/chunkallocator.h" #include "../mem/corealloc.h" #include "../mem/pool.h" #include "commonconfig.h" @@ -36,8 +36,6 @@ namespace snmalloc private: using Backend = BackendAllocator; - SNMALLOC_REQUIRE_CONSTINIT - inline static ChunkAllocatorState chunk_allocator_state; SNMALLOC_REQUIRE_CONSTINIT inline static GlobalPoolState alloc_pool; @@ -49,12 +47,6 @@ namespace snmalloc inline static FlagWord initialisation_lock{}; public: - static ChunkAllocatorState& - get_chunk_allocator_state(Backend::LocalState* = nullptr) - { - return chunk_allocator_state; - } - static GlobalPoolState& pool() { return alloc_pool; diff --git a/src/backend/globalrange.h b/src/backend/globalrange.h new file mode 100644 index 0000000..7b4bb0b --- /dev/null +++ b/src/backend/globalrange.h @@ -0,0 +1,54 @@ +#pragma once + +#include "../ds/defines.h" +#include "../ds/helpers.h" +#include "../ds/ptrwrap.h" + +namespace snmalloc +{ + /** + * Makes the supplied ParentRange into a global variable, + * and protects access with a lock. + */ + template + class GlobalRange + { + typename ParentRange::State parent{}; + + /** + * This is infrequently used code, a spin lock simplifies the code + * considerably, and should never be on the fast path. + */ + FlagWord spin_lock{}; + + public: + class State + { + SNMALLOC_REQUIRE_CONSTINIT static inline GlobalRange global_range{}; + + public: + constexpr GlobalRange* operator->() + { + return &global_range; + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = ParentRange::Aligned; + + constexpr GlobalRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + FlagLock lock(spin_lock); + return parent->alloc_range(size); + } + + void dealloc_range(capptr::Chunk base, size_t size) + { + FlagLock lock(spin_lock); + parent->dealloc_range(base, size); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/largebuddyrange.h b/src/backend/largebuddyrange.h new file mode 100644 index 0000000..1d15dfe --- /dev/null +++ b/src/backend/largebuddyrange.h @@ -0,0 +1,289 @@ +#pragma once + +#include "../ds/address.h" +#include "../ds/bits.h" +#include "../mem/allocconfig.h" +#include "../mem/metaslab.h" +#include "../pal/pal.h" +#include "buddy.h" +#include "range_helpers.h" + +#include + +namespace snmalloc +{ + /** + * Class for using the pagemap entries for the buddy allocator. + */ + template + class BuddyChunkRep + { + public: + using Holder = uintptr_t; + using Contents = uintptr_t; + + static constexpr address_t RED_BIT = 2; + + static constexpr Contents null = 0; + + static void set(Holder* ptr, Contents r) + { + SNMALLOC_ASSERT((r & (MIN_CHUNK_SIZE - 1)) == 0); + // Preserve lower bits. + *ptr = r | address_cast(*ptr & (MIN_CHUNK_SIZE - 1)) | BACKEND_MARKER; + } + + static Contents get(const Holder* ptr) + { + return *ptr & ~(MIN_CHUNK_SIZE - 1); + } + + static Holder& ref(bool direction, Contents k) + { + MetaEntry& entry = + Pagemap::template get_metaentry_mut(address_cast(k)); + if (direction) + return *reinterpret_cast(&entry.meta); + + return *reinterpret_cast(&entry.remote_and_sizeclass); + } + + static bool is_red(Contents k) + { + return (ref(true, k) & RED_BIT) == RED_BIT; + } + + static void set_red(Contents k, bool new_is_red) + { + if (new_is_red != is_red(k)) + ref(true, k) ^= RED_BIT; + } + + static Contents offset(Contents k, size_t size) + { + return k + size; + } + + static Contents buddy(Contents k, size_t size) + { + return k ^ size; + } + + static Contents align_down(Contents k, size_t size) + { + return k & ~(size - 1); + } + + static bool compare(Contents k1, Contents k2) + { + return k1 > k2; + } + + static bool equal(Contents k1, Contents k2) + { + return k1 == k2; + } + + static uintptr_t printable(Contents k) + { + return k; + } + + static bool can_consolidate(Contents k, size_t size) + { + // Need to know both entries exist in the pagemap. + // This must only be called if that has already been + // ascertained. + // The buddy could be in a part of the pagemap that has + // not been registered and thus could segfault on access. + auto larger = bits::max(k, buddy(k, size)); + MetaEntry& entry = + Pagemap::template get_metaentry_mut(address_cast(larger)); + return !entry.is_boundary(); + } + }; + + template< + typename ParentRange, + size_t REFILL_SIZE_BITS, + size_t MAX_SIZE_BITS, + SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap, + bool Consolidate = true> + class LargeBuddyRange + { + typename ParentRange::State parent{}; + + static constexpr size_t REFILL_SIZE = bits::one_at_bit(REFILL_SIZE_BITS); + + /** + * + */ + Buddy, MIN_CHUNK_BITS, MAX_SIZE_BITS> buddy_large; + + /** + * The parent might not support deallocation if this buddy allocator covers + * the whole range. Uses template insanity to make this work. + */ + template + std::enable_if_t + parent_dealloc_range(capptr::Chunk base, size_t size) + { + static_assert( + MAX_SIZE_BITS != (bits::BITS - 1), "Don't set SFINAE parameter"); + parent->dealloc_range(base, size); + } + + void dealloc_overflow(capptr::Chunk overflow) + { + if constexpr (MAX_SIZE_BITS != (bits::BITS - 1)) + { + if (overflow != nullptr) + { + parent->dealloc_range(overflow, bits::one_at_bit(MAX_SIZE_BITS)); + } + } + else + { + if (overflow != nullptr) + abort(); + } + } + + /** + * Add a range of memory to the address space. + * Divides blocks into power of two sizes with natural alignment + */ + void add_range(capptr::Chunk base, size_t length) + { + range_to_pow_2_blocks( + base, + length, + [this](capptr::Chunk base, size_t align, bool first) { + if constexpr (!Consolidate) + { + // Tag first entry so we don't consolidate it. + if (first) + { + Pagemap::get_metaentry_mut(address_cast(base)).set_boundary(); + } + } + else + { + UNUSED(first); + } + + auto overflow = capptr::Chunk(reinterpret_cast( + buddy_large.add_block(base.unsafe_uintptr(), align))); + + dealloc_overflow(overflow); + }); + } + + capptr::Chunk refill(size_t size) + { + if (ParentRange::Aligned) + { + // TODO have to add consolidation blocker for these cases. + if (size >= REFILL_SIZE) + { + return parent->alloc_range(size); + } + + auto refill_range = parent->alloc_range(REFILL_SIZE); + if (refill_range != nullptr) + add_range(pointer_offset(refill_range, size), REFILL_SIZE - size); + return refill_range; + } + + // Need to overallocate to get the alignment right. + bool overflow = false; + size_t needed_size = bits::umul(size, 2, overflow); + if (overflow) + { + return nullptr; + } + + auto refill_size = bits::max(needed_size, REFILL_SIZE); + while (needed_size <= refill_size) + { + auto refill = parent->alloc_range(refill_size); + + if (refill != nullptr) + { + add_range(refill, refill_size); + + SNMALLOC_ASSERT(refill_size < bits::one_at_bit(MAX_SIZE_BITS)); + static_assert( + (REFILL_SIZE < bits::one_at_bit(MAX_SIZE_BITS)) || + ParentRange::Aligned, + "Required to prevent overflow."); + + return alloc_range(size); + } + + refill_size >>= 1; + } + + return nullptr; + } + + public: + class State + { + LargeBuddyRange buddy_range; + + public: + LargeBuddyRange* operator->() + { + return &buddy_range; + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = true; + + constexpr LargeBuddyRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); + SNMALLOC_ASSERT(bits::is_pow2(size)); + + if (size >= (bits::one_at_bit(MAX_SIZE_BITS) - 1)) + { + if (ParentRange::Aligned) + return parent->alloc_range(size); + + return nullptr; + } + + auto result = capptr::Chunk( + reinterpret_cast(buddy_large.remove_block(size))); + + if (result != nullptr) + return result; + + return refill(size); + } + + void dealloc_range(capptr::Chunk base, size_t size) + { + SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); + SNMALLOC_ASSERT(bits::is_pow2(size)); + + if constexpr (MAX_SIZE_BITS != (bits::BITS - 1)) + { + if (size >= (bits::one_at_bit(MAX_SIZE_BITS) - 1)) + { + parent_dealloc_range(base, size); + return; + } + } + + auto overflow = capptr::Chunk(reinterpret_cast( + buddy_large.add_block(base.unsafe_uintptr(), size))); + dealloc_overflow(overflow); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 18463b9..85bb113 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -115,7 +115,7 @@ namespace snmalloc static_assert( has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); #ifdef SNMALLOC_TRACING - std::cout << "Pagemap.init " << b << " (" << s << ")" << std::endl; + message<1024>("Pagemap.init {} ({})", b, s); #endif SNMALLOC_ASSERT(s != 0); // TODO take account of pagemap size in the calculation of how big it @@ -311,10 +311,10 @@ namespace snmalloc return base + (entry_index << GRANULARITY_BITS); } - void set(address_t p, T t) + void set(address_t p, const T& t) { #ifdef SNMALLOC_TRACING - std::cout << "Pagemap.Set " << (void*)(uintptr_t)p << std::endl; + message<1024>("Pagemap.Set {}", p); #endif if constexpr (has_bounds) { diff --git a/src/backend/pagemapregisterrange.h b/src/backend/pagemapregisterrange.h new file mode 100644 index 0000000..de55b13 --- /dev/null +++ b/src/backend/pagemapregisterrange.h @@ -0,0 +1,43 @@ +#pragma once + +#include "../ds/address.h" +#include "../pal/pal.h" + +namespace snmalloc +{ + template< + SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap, + typename ParentRange> + class PagemapRegisterRange + { + typename ParentRange::State state{}; + + public: + class State + { + PagemapRegisterRange range; + + public: + PagemapRegisterRange* operator->() + { + return ⦥ + } + + constexpr State() = default; + }; + + constexpr PagemapRegisterRange() = default; + + static constexpr bool Aligned = ParentRange::Aligned; + + capptr::Chunk alloc_range(size_t size) + { + auto base = state->alloc_range(size); + + if (base != nullptr) + Pagemap::register_range(address_cast(base), size); + + return base; + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/palrange.h b/src/backend/palrange.h new file mode 100644 index 0000000..82d36ce --- /dev/null +++ b/src/backend/palrange.h @@ -0,0 +1,59 @@ +#pragma once +#include "../ds/address.h" +#include "../pal/pal.h" + +namespace snmalloc +{ + template + class PalRange + { + public: + class State + { + public: + PalRange* operator->() + { + // There is no state required for the PalRange + // using a global just to satisfy the typing. + static PalRange range{}; + return ⦥ + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = pal_supports; + + constexpr PalRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + if (bits::next_pow2_bits(size) >= bits::BITS - 1) + { + return nullptr; + } + + if constexpr (pal_supports) + { + SNMALLOC_ASSERT(size >= PAL::minimum_alloc_size); + auto result = + capptr::Chunk(PAL::template reserve_aligned(size)); + +#ifdef SNMALLOC_TRACING + message<1024>("Pal range alloc: {} ({})", result.unsafe_ptr(), size); +#endif + return result; + } + else + { + auto result = capptr::Chunk(PAL::reserve(size)); + +#ifdef SNMALLOC_TRACING + message<1024>("Pal range alloc: {} ({})", result.unsafe_ptr(), size); +#endif + + return result; + } + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/range_helpers.h b/src/backend/range_helpers.h new file mode 100644 index 0000000..1dd6ea4 --- /dev/null +++ b/src/backend/range_helpers.h @@ -0,0 +1,37 @@ +#pragma once + +#include "../ds/ptrwrap.h" + +namespace snmalloc +{ + template + void range_to_pow_2_blocks(capptr::Chunk base, size_t length, F f) + { + auto end = pointer_offset(base, length); + base = pointer_align_up(base, bits::one_at_bit(MIN_BITS)); + end = pointer_align_down(end, bits::one_at_bit(MIN_BITS)); + length = pointer_diff(base, end); + + bool first = true; + + // Find the minimum set of maximally aligned blocks in this range. + // Each block's alignment and size are equal. + while (length >= sizeof(void*)) + { + size_t base_align_bits = bits::ctz(address_cast(base)); + size_t length_align_bits = (bits::BITS - 1) - bits::clz(length); + size_t align_bits = bits::min(base_align_bits, length_align_bits); + size_t align = bits::one_at_bit(align_bits); + + /* + * Now that we have found a maximally-aligned block, we can set bounds + * and be certain that we won't hit representation imprecision. + */ + f(base, align, first); + first = false; + + base = pointer_offset(base, align); + length -= align; + } + } +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/smallbuddyrange.h b/src/backend/smallbuddyrange.h new file mode 100644 index 0000000..b813b67 --- /dev/null +++ b/src/backend/smallbuddyrange.h @@ -0,0 +1,226 @@ +#pragma once + +#include "../ds/address.h" +#include "../pal/pal.h" +#include "range_helpers.h" + +namespace snmalloc +{ + /** + * struct for representing the redblack nodes + * directly inside the meta data. + */ + struct FreeChunk + { + capptr::Chunk left; + capptr::Chunk right; + }; + + /** + * Class for using the allocations own space to store in the RBTree. + */ + class BuddyInplaceRep + { + public: + using Holder = capptr::Chunk; + using Contents = capptr::Chunk; + + static constexpr Contents null = nullptr; + + static constexpr address_t MASK = 1; + static void set(Holder* ptr, Contents r) + { + SNMALLOC_ASSERT((address_cast(r) & MASK) == 0); + if (r == nullptr) + *ptr = capptr::Chunk( + reinterpret_cast((*ptr).unsafe_uintptr() & MASK)); + else + // Preserve lower bit. + *ptr = pointer_offset(r, (address_cast(*ptr) & MASK)) + .template as_static(); + } + + static Contents get(Holder* ptr) + { + return pointer_align_down<2, FreeChunk>((*ptr).as_void()); + } + + static Holder& ref(bool direction, Contents r) + { + if (direction) + return r->left; + + return r->right; + } + + static bool is_red(Contents k) + { + if (k == nullptr) + return false; + return (address_cast(ref(false, k)) & MASK) == MASK; + } + + static void set_red(Contents k, bool new_is_red) + { + if (new_is_red != is_red(k)) + { + auto& r = ref(false, k); + auto old_addr = pointer_align_down<2, FreeChunk>(r.as_void()); + + if (new_is_red) + { + if (old_addr == nullptr) + r = capptr::Chunk(reinterpret_cast(MASK)); + else + r = pointer_offset(old_addr, MASK).template as_static(); + } + else + { + r = old_addr; + } + } + } + + static Contents offset(Contents k, size_t size) + { + return pointer_offset(k, size).template as_static(); + } + + static Contents buddy(Contents k, size_t size) + { + // This is just doing xor size, but with what API + // exists on capptr. + auto base = pointer_align_down(k.as_void(), size * 2); + auto offset = (address_cast(k) & size) ^ size; + return pointer_offset(base, offset).template as_static(); + } + + static Contents align_down(Contents k, size_t size) + { + return pointer_align_down(k.as_void(), size); + } + + static bool compare(Contents k1, Contents k2) + { + return address_cast(k1) > address_cast(k2); + } + + static bool equal(Contents k1, Contents k2) + { + return address_cast(k1) == address_cast(k2); + } + + static address_t printable(Contents k) + { + return address_cast(k); + } + + static bool can_consolidate(Contents k, size_t size) + { + UNUSED(k, size); + return true; + } + }; + + template + class SmallBuddyRange + { + typename ParentRange::State parent{}; + + static constexpr size_t MIN_BITS = + bits::next_pow2_bits_const(sizeof(FreeChunk)); + + Buddy buddy_small; + + /** + * Add a range of memory to the address space. + * Divides blocks into power of two sizes with natural alignment + */ + void add_range(capptr::Chunk base, size_t length) + { + range_to_pow_2_blocks( + base, length, [this](capptr::Chunk base, size_t align, bool) { + capptr::Chunk overflow = + buddy_small.add_block(base.as_reinterpret(), align) + .template as_reinterpret(); + if (overflow != nullptr) + parent->dealloc_range(overflow, bits::one_at_bit(MIN_CHUNK_BITS)); + }); + } + + capptr::Chunk refill(size_t size) + { + auto refill = parent->alloc_range(MIN_CHUNK_SIZE); + + if (refill != nullptr) + add_range(pointer_offset(refill, size), MIN_CHUNK_SIZE - size); + + return refill; + } + + public: + class State + { + SmallBuddyRange buddy_range; + + public: + SmallBuddyRange* operator->() + { + return &buddy_range; + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = true; + static_assert(ParentRange::Aligned, "ParentRange must be aligned"); + + constexpr SmallBuddyRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + if (size >= MIN_CHUNK_SIZE) + { + return parent->alloc_range(size); + } + + auto result = buddy_small.remove_block(size); + if (result != nullptr) + { + result->left = nullptr; + result->right = nullptr; + return result.template as_reinterpret(); + } + return refill(size); + } + + capptr::Chunk alloc_range_with_leftover(size_t size) + { + SNMALLOC_ASSERT(size <= MIN_CHUNK_SIZE); + + auto rsize = bits::next_pow2(size); + + auto result = alloc_range(rsize); + + if (result == nullptr) + return nullptr; + + auto remnant = pointer_offset(result, size); + + add_range(remnant, rsize - size); + + return result.template as_reinterpret(); + } + + void dealloc_range(capptr::Chunk base, size_t size) + { + if (size >= MIN_CHUNK_SIZE) + { + parent->dealloc_range(base, size); + return; + } + + add_range(base, size); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/statsrange.h b/src/backend/statsrange.h new file mode 100644 index 0000000..0a61fef --- /dev/null +++ b/src/backend/statsrange.h @@ -0,0 +1,68 @@ +#pragma once + +#include + +namespace snmalloc +{ + /** + * Used to measure memory usage. + */ + template + class StatsRange + { + typename ParentRange::State parent{}; + + static inline std::atomic current_usage{}; + static inline std::atomic peak_usage{}; + + public: + class State + { + StatsRange stats_range{}; + + public: + constexpr StatsRange* operator->() + { + return &stats_range; + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = ParentRange::Aligned; + + constexpr StatsRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + auto result = parent->alloc_range(size); + if (result != nullptr) + { + auto prev = current_usage.fetch_add(size); + auto curr = peak_usage.load(); + while (curr < prev + size) + { + if (peak_usage.compare_exchange_weak(curr, prev + size)) + break; + } + } + return result; + } + + void dealloc_range(capptr::Chunk base, size_t size) + { + current_usage -= size; + parent->dealloc_range(base, size); + } + + size_t get_current_usage() + { + return current_usage.load(); + } + + size_t get_peak_usage() + { + return peak_usage.load(); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/subrange.h b/src/backend/subrange.h new file mode 100644 index 0000000..c83bdab --- /dev/null +++ b/src/backend/subrange.h @@ -0,0 +1,55 @@ +#pragma once +#include "../mem/entropy.h" + +namespace snmalloc +{ + /** + * Creates an area inside a large allocation that is larger by + * 2^RATIO_BITS. Will not return a the block at the start or + * the end of the large allocation. + */ + template + class SubRange + { + typename ParentRange::State parent{}; + + public: + class State + { + SubRange sub_range{}; + + public: + constexpr State() = default; + + SubRange* operator->() + { + return &sub_range; + } + }; + + constexpr SubRange() = default; + + static constexpr bool Aligned = ParentRange::Aligned; + + capptr::Chunk alloc_range(size_t sub_size) + { + SNMALLOC_ASSERT(bits::is_pow2(sub_size)); + + auto full_size = sub_size << RATIO_BITS; + auto overblock = parent->alloc_range(full_size); + if (overblock == nullptr) + return nullptr; + + size_t offset_mask = full_size - sub_size; + // Don't use first or last block in the larger reservation + // Loop required to get uniform distribution. + size_t offset; + do + { + offset = get_entropy64() & offset_mask; + } while ((offset == 0) || (offset == offset_mask)); + + return pointer_offset(overblock, offset); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/ds/address.h b/src/ds/address.h index 13c6ed8..0c3cdc7 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -72,13 +72,17 @@ namespace snmalloc * as per above, and uses the wrapper types in its own definition, e.g., of * capptr_bound. */ - template inline SNMALLOC_FAST_PATH address_t address_cast(CapPtr a) { return address_cast(a.unsafe_ptr()); } + inline SNMALLOC_FAST_PATH address_t address_cast(uintptr_t a) + { + return static_cast(a); + } + /** * Test if a pointer is aligned to a given size, which must be a power of * two. diff --git a/src/ds/redblacktree.h b/src/ds/redblacktree.h index 80325b3..10eb55f 100644 --- a/src/ds/redblacktree.h +++ b/src/ds/redblacktree.h @@ -1,44 +1,15 @@ #pragma once +#include "../pal/pal.h" #include "concept.h" #include "defines.h" #include #include #include +#include namespace snmalloc { - template - class debug_out - { - public: - template - static void msg(A a, Args... args) - { - if constexpr (TRACE) - { -#ifdef SNMALLOC_TRACING - std::cout << a; -#else - UNUSED(a); -#endif - - msg(args...); - } - } - - template - static void msg() - { - if constexpr (TRACE && new_line) - { -#ifdef SNMALLOC_TRACING - std::cout << std::endl; -#endif - } - } - }; - #ifdef __cpp_concepts template concept RBRepTypes = requires() @@ -118,20 +89,20 @@ namespace snmalloc return Rep::get(ptr); } - K operator=(K t) + ChildRef& operator=(const K& t) { // Use representations assigment, so we update the correct bits // color and other things way also be stored in the Holder. Rep::set(ptr, t); - return t; + return *this; } - bool operator==(ChildRef& t) + bool operator==(const ChildRef t) const { return ptr == t.ptr; } - bool operator!=(ChildRef& t) + bool operator!=(const ChildRef t) const { return ptr != t.ptr; } @@ -140,6 +111,11 @@ namespace snmalloc { return ptr; } + + bool is_null() + { + return Rep::get(ptr) == Rep::null; + } }; // Root field of the tree @@ -171,60 +147,47 @@ namespace snmalloc UNUSED(curr, lower, upper); return 0; } - if (curr == Rep::null) - return 1; - - if ( - ((lower != Rep::null) && curr < lower) || - ((upper != Rep::null) && curr > upper)) - { - if constexpr (TRACE) - { - debug_out::msg( - "Invariant failed: ", - curr, - " is out of bounds ", - lower, - ", ", - upper); - print(); - } - snmalloc::error("Invariant failed"); - } - - if ( - Rep::is_red(curr) && - (Rep::is_red(get_dir(true, curr)) || Rep::is_red(get_dir(false, curr)))) - { - if constexpr (TRACE) - { - debug_out::msg( - "Red invariant failed: ", curr, " is red and has red children"); - print(); - } - snmalloc::error("Invariant failed"); - } - - int left_inv = invariant(get_dir(true, curr), lower, curr); - int right_inv = invariant(get_dir(false, curr), curr, upper); - - if (left_inv != right_inv) - { - if constexpr (TRACE) - { - debug_out::msg( - "Balance failed: ", - curr, - " has different black depths on left and right"); - print(); - } - snmalloc::error("Invariant failed"); - } - - if (Rep::is_red(curr)) - return left_inv; else + { + if (curr == Rep::null) + return 1; + + if ( + ((lower != Rep::null) && Rep::compare(lower, curr)) || + ((upper != Rep::null) && Rep::compare(curr, upper))) + { + report_fatal_error( + "Invariant failed: {} is out of bounds {}..{}", + Rep::printable(curr), + Rep::printable(lower), + Rep::printable(upper)); + } + + if ( + Rep::is_red(curr) && + (Rep::is_red(get_dir(true, curr)) || + Rep::is_red(get_dir(false, curr)))) + { + report_fatal_error( + "Invariant failed: {} is red and has red child", + Rep::printable(curr)); + } + + int left_inv = invariant(get_dir(true, curr), lower, curr); + int right_inv = invariant(get_dir(false, curr), curr, upper); + + if (left_inv != right_inv) + { + report_fatal_error( + "Invariant failed: {} has different black depths", + Rep::printable(curr)); + } + + if (Rep::is_red(curr)) + return left_inv; + return left_inv + 1; + } } struct RBStep @@ -293,7 +256,7 @@ namespace snmalloc bool move(bool direction) { auto next = get_dir(direction, curr()); - if (next == Rep::null) + if (next.is_null()) return false; path[length] = {next, direction}; length++; @@ -308,7 +271,7 @@ namespace snmalloc auto next = get_dir(direction, curr()); path[length] = {next, direction}; length++; - return next != Rep::null; + return !(next.is_null()); } // Remove top element from the path. @@ -354,16 +317,12 @@ namespace snmalloc { for (size_t i = 0; i < length; i++) { - debug_out::msg( - "->", + message<1024>( + " -> {} @ {} ({})", K(path[i].node), - "@", path[i].node.addr(), - " (", - path[i].dir, - ") "); + path[i].dir); } - debug_out::msg(); } } }; @@ -378,8 +337,8 @@ namespace snmalloc { if constexpr (TRACE) { - debug_out::msg("-------"); - debug_out::msg(msg); + message<100>("-------"); + message<1024>(msg); path.print(); print(base); } @@ -390,7 +349,7 @@ namespace snmalloc } public: - RBTree() {} + constexpr RBTree() = default; void print() { @@ -401,9 +360,9 @@ namespace snmalloc { if constexpr (TRACE) { - if (curr == Rep::null) + if (curr.is_null()) { - debug_out::msg(indent, "\\_", "null"); + message<1024>("{}\\_null", indent); return; } @@ -415,17 +374,14 @@ namespace snmalloc auto reset = "\e[0m"; #endif - debug_out::msg( + message<1024>( + "{}\\_{}{}{}@{} ({})", indent, - "\\_", colour, curr, reset, - "@", curr.addr(), - " (", - depth, - ")"); + depth); if ((get_dir(true, curr) != 0) || (get_dir(false, curr) != 0)) { auto s_indent = std::string(indent); @@ -439,14 +395,14 @@ namespace snmalloc { bool dir; - if (path.curr() == Rep::null) + if (path.curr().is_null()) return false; do { - if (path.curr() == value) + if (Rep::equal(path.curr(), value)) return true; - dir = path.curr() > value; + dir = Rep::compare(path.curr(), value); } while (path.move_inc_null(dir)); return false; @@ -455,7 +411,7 @@ namespace snmalloc bool remove_path(RBPath& path) { ChildRef splice = path.curr(); - SNMALLOC_ASSERT(splice != Rep::null); + SNMALLOC_ASSERT(!(splice.is_null())); debug_log("Removing", path); @@ -494,6 +450,8 @@ namespace snmalloc debug_log("Splice done", path); + // TODO: Clear node contents? + // Red leaf removal requires no rebalancing. if (leaf_red) return true; @@ -618,7 +576,7 @@ namespace snmalloc // Insert an element at the given path. void insert_path(RBPath path, K value) { - SNMALLOC_ASSERT(path.curr() == Rep::null); + SNMALLOC_ASSERT(path.curr().is_null()); path.curr() = value; get_dir(true, path.curr()) = Rep::null; get_dir(false, path.curr()) = Rep::null; @@ -706,7 +664,7 @@ namespace snmalloc K remove_min() { - if (get_root() == Rep::null) + if (get_root().is_null()) return Rep::null; auto path = get_root_path(); @@ -722,7 +680,7 @@ namespace snmalloc bool remove_elem(K value) { - if (get_root() == Rep::null) + if (get_root().is_null()) return false; auto path = get_root_path(); @@ -749,4 +707,4 @@ namespace snmalloc return RBPath(root); } }; -} +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index f621e0c..48d3263 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -1,8 +1,8 @@ #pragma once +#include "../backend/chunkallocator.h" #include "../ds/defines.h" #include "allocconfig.h" -#include "chunkallocator.h" #include "localcache.h" #include "metaslab.h" #include "pool.h" @@ -58,11 +58,6 @@ namespace snmalloc */ MetaslabCache alloc_classes[NUM_SMALL_SIZECLASSES]; - /** - * Local cache for the Chunk allocator. - */ - ChunkAllocatorLocalState chunk_local_state; - /** * Local entropy source and current version of keys for * this thread @@ -392,11 +387,11 @@ namespace snmalloc // TODO delay the clear to the next user of the slab, or teardown so // don't touch the cache lines at this point in snmalloc_check_client. auto chunk_record = clear_slab(meta, sizeclass); - ChunkAllocator::dealloc( + + SharedStateHandle::dealloc_chunk( get_backend_local_state(), - chunk_local_state, chunk_record, - sizeclass_to_slab_sizeclass(sizeclass)); + sizeclass_to_slab_size(sizeclass)); return true; }); @@ -419,23 +414,17 @@ namespace snmalloc // Handle large deallocation here. size_t entry_sizeclass = entry.get_sizeclass().as_large(); size_t size = bits::one_at_bit(entry_sizeclass); - size_t slab_sizeclass = - metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass); #ifdef SNMALLOC_TRACING - std::cout << "Large deallocation: " << size - << " chunk sizeclass: " << slab_sizeclass << std::endl; + std::cout << "Large deallocation: " << size << std::endl; #else UNUSED(size); #endif auto slab_record = reinterpret_cast(meta); - ChunkAllocator::dealloc( - get_backend_local_state(), - chunk_local_state, - slab_record, - slab_sizeclass); + SharedStateHandle::dealloc_chunk( + get_backend_local_state(), slab_record, size); return; } @@ -594,9 +583,6 @@ namespace snmalloc message_queue().invariant(); } - ChunkAllocator::register_local_state( - get_backend_local_state(), chunk_local_state); - if constexpr (DEBUG) { for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) @@ -686,7 +672,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(CapPtr p) { - auto entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p)); if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy))) return; @@ -791,21 +777,17 @@ namespace snmalloc // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); - size_t slab_sizeclass = sizeclass_to_slab_sizeclass(sizeclass); #ifdef SNMALLOC_TRACING std::cout << "rsize " << rsize << std::endl; std::cout << "slab size " << slab_size << std::endl; #endif - auto [slab, meta] = - snmalloc::ChunkAllocator::alloc_chunk( - get_backend_local_state(), - chunk_local_state, - sizeclass_t::from_small_class(sizeclass), - slab_sizeclass, - slab_size, - public_state()); + auto [slab, meta] = SharedStateHandle::alloc_chunk( + get_backend_local_state(), + slab_size, + public_state(), + sizeclass_t::from_small_class(sizeclass)); if (slab == nullptr) { diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 6fb378e..d9bf184 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -180,13 +180,11 @@ namespace snmalloc return check_init([&](CoreAlloc* core_alloc) { // Grab slab of correct size // Set remote as large allocator remote. - auto [chunk, meta] = ChunkAllocator::alloc_chunk( + auto [chunk, meta] = SharedStateHandle::alloc_chunk( core_alloc->get_backend_local_state(), - core_alloc->chunk_local_state, - size_to_sizeclass_full(size), - large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), - core_alloc->public_state()); + core_alloc->public_state(), + size_to_sizeclass_full(size)); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING @@ -201,7 +199,7 @@ namespace snmalloc if (zero_mem == YesZero && chunk.unsafe_ptr() != nullptr) { SharedStateHandle::Pal::template zero( - chunk.unsafe_ptr(), size); + chunk.unsafe_ptr(), bits::next_pow2(size)); } return capptr_chunk_is_alloc(capptr_to_user_address_control(chunk)); @@ -268,7 +266,7 @@ namespace snmalloc std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size " << alloc_size(p.unsafe_ptr()) << std::endl; #endif - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), p, key_global); @@ -716,7 +714,7 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a // large object, so we can pull the check for null off the fast path. - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw)); return sizeclass_full_to_size(entry.get_sizeclass()); @@ -761,7 +759,7 @@ namespace snmalloc size_t remaining_bytes(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::template get_metaentry( address_cast(p)); @@ -790,7 +788,7 @@ namespace snmalloc size_t index_in_object(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::template get_metaentry( address_cast(p)); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index bad2d3c..a046e09 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -227,7 +227,26 @@ namespace snmalloc */ class MetaEntry { - Metaslab* meta{nullptr}; // may also be ChunkRecord* + template + friend class BuddyChunkRep; + + /** + * The pointer to the metaslab, the bottom bit is used to indicate if this + * is the first chunk in a PAL allocation, that cannot be combined with + * the preceeding chunk. + */ + uintptr_t meta{0}; + + /** + * Bit used to indicate this should not be considered part of the previous + * PAL allocation. + * + * Some platforms cannot treat different PalAllocs as a single allocation. + * This is true on CHERI as the combined permission might not be + * representable. It is also true on Windows as you cannot Commit across + * multiple continuous VirtualAllocs. + */ + static constexpr address_t BOUNDARY_BIT = 1; /** * A bit-packed pointer to the owning allocator (if any), and the sizeclass @@ -253,7 +272,8 @@ namespace snmalloc */ SNMALLOC_FAST_PATH MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) - : meta(meta), remote_and_sizeclass(remote_and_sizeclass) + : meta(reinterpret_cast(meta)), + remote_and_sizeclass(remote_and_sizeclass) {} SNMALLOC_FAST_PATH @@ -261,23 +281,13 @@ namespace snmalloc Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass = sizeclass_t()) - : meta(meta) + : meta(reinterpret_cast(meta)) { /* remote might be nullptr; cast to uintptr_t before offsetting */ remote_and_sizeclass = pointer_offset(reinterpret_cast(remote), sizeclass.raw()); } - /** - * Return the Metaslab field as a void*, guarded by an assert that there is - * no remote that owns this chunk. - */ - [[nodiscard]] SNMALLOC_FAST_PATH void* get_metaslab_no_remote() const - { - SNMALLOC_ASSERT(get_remote() == nullptr); - return static_cast(meta); - } - /** * Return the Metaslab metadata associated with this chunk, guarded by an * assert that this chunk is being used as a slab (i.e., has an associated @@ -286,7 +296,7 @@ namespace snmalloc [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const { SNMALLOC_ASSERT(get_remote() != nullptr); - return meta; + return reinterpret_cast(meta & ~BOUNDARY_BIT); } /** @@ -295,7 +305,7 @@ namespace snmalloc * only safe use for this is to pass it to the two-argument constructor of * this class. */ - [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() + [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const { return remote_and_sizeclass; } @@ -303,7 +313,8 @@ namespace snmalloc [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const { return reinterpret_cast( - pointer_align_down(remote_and_sizeclass)); + pointer_align_down( + remote_and_sizeclass)); } [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const @@ -312,7 +323,32 @@ namespace snmalloc // https://github.com/CTSRD-CHERI/llvm-project/issues/588 return sizeclass_t::from_raw( static_cast(remote_and_sizeclass) & - (alignof(RemoteAllocator) - 1)); + (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); + } + + MetaEntry(const MetaEntry&) = delete; + + MetaEntry& operator=(const MetaEntry& other) + { + // Don't overwrite the boundary bit with the other's + meta = (other.meta & ~BOUNDARY_BIT) | address_cast(meta & BOUNDARY_BIT); + remote_and_sizeclass = other.remote_and_sizeclass; + return *this; + } + + void set_boundary() + { + meta |= BOUNDARY_BIT; + } + + [[nodiscard]] bool is_boundary() const + { + return meta & BOUNDARY_BIT; + } + + bool clear_boundary_bit() + { + return meta &= ~BOUNDARY_BIT; } }; @@ -328,5 +364,4 @@ namespace snmalloc uint16_t unused = 0; uint16_t length = 0; }; - } // namespace snmalloc diff --git a/src/mem/pool.h b/src/mem/pool.h index ffb9572..c7924c6 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -1,9 +1,9 @@ #pragma once +#include "../backend/chunkallocator.h" #include "../ds/flaglock.h" #include "../ds/mpmcstack.h" #include "../pal/pal_concept.h" -#include "chunkallocator.h" #include "pooled.h" namespace snmalloc diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 3c060e6..f8b309e 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -2,7 +2,6 @@ #include "../mem/allocconfig.h" #include "../mem/freelist.h" -#include "../mem/metaslab.h" #include "../mem/sizeclasstable.h" #include @@ -11,9 +10,19 @@ namespace snmalloc { // Remotes need to be aligned enough that the bottom bits have enough room for - // all the size classes, both large and small. + // all the size classes, both large and small. An additional bit is required + // to separate backend uses. static constexpr size_t REMOTE_MIN_ALIGN = - bits::max(CACHELINE_SIZE, SIZECLASS_REP_SIZE); + bits::max(CACHELINE_SIZE, SIZECLASS_REP_SIZE) << 1; + + // This bit is set on the RemoteAllocator* to indicate it is + // actually being used by the backend for some other use. + static constexpr size_t BACKEND_MARKER = REMOTE_MIN_ALIGN >> 1; + + // The bit above the sizeclass is always zero unless this is used + // by the backend to represent another datastructure such as the buddy + // allocator entries. + constexpr size_t REMOTE_WITH_BACKEND_MARKER_ALIGN = BACKEND_MARKER; /** * Global key for all remote lists. diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index cafe158..5e75599 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -101,18 +101,25 @@ namespace snmalloc if (!list[i].empty()) { auto [first, last] = list[i].extract_segment(key); - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(address_cast(first)); + auto remote = entry.get_remote(); + // If the allocator is not correctly aligned, then the bit that is + // set implies this is used by the backend, and we should not be + // deallocating memory here. + snmalloc_check_client( + (address_cast(remote) & BACKEND_MARKER) == 0, + "Delayed detection of attempt to free internal structure."); if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) { auto domesticate_nop = [](freelist::QueuePtr p) { return freelist::HeadPtr(p.unsafe_ptr()); }; - entry.get_remote()->enqueue(first, last, key, domesticate_nop); + remote->enqueue(first, last, key, domesticate_nop); } else { - entry.get_remote()->enqueue(first, last, key, domesticate); + remote->enqueue(first, last, key, domesticate); } sent_something = true; } @@ -134,7 +141,7 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key, domesticate); - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(address_cast(r)); auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); diff --git a/src/override/malloc-extensions.cc b/src/override/malloc-extensions.cc index a760401..1e0810b 100644 --- a/src/override/malloc-extensions.cc +++ b/src/override/malloc-extensions.cc @@ -6,8 +6,8 @@ using namespace snmalloc; void get_malloc_info_v1(malloc_info_v1* stats) { - auto unused_chunks = Globals::get_chunk_allocator_state().unused_memory(); - auto peak = Globals::get_chunk_allocator_state().peak_memory_usage(); - stats->current_memory_usage = peak - unused_chunks; + auto curr = Globals::get_current_usage(); + auto peak = Globals::get_peak_usage(); + stats->current_memory_usage = curr; stats->peak_memory_usage = peak; } diff --git a/src/override/rust.cc b/src/override/rust.cc index ebce39b..8892323 100644 --- a/src/override/rust.cc +++ b/src/override/rust.cc @@ -48,8 +48,6 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_realloc)( extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(rust_statistics)( size_t* current_memory_usage, size_t* peak_memory_usage) { - auto unused_chunks = Globals::get_chunk_allocator_state().unused_memory(); - auto peak = Globals::get_chunk_allocator_state().peak_memory_usage(); - *current_memory_usage = peak - unused_chunks; - *peak_memory_usage = peak; + *current_memory_usage = Globals::get_current_usage(); + *peak_memory_usage = Globals::get_peak_usage(); } \ No newline at end of file diff --git a/src/pal/pal.h b/src/pal/pal.h index c4dfd0c..b893914 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -168,4 +168,10 @@ namespace snmalloc Pal::error(msg.get_message()); } + template + inline void message(Args... args) + { + MessageBuilder msg{std::forward(args)...}; + Pal::message(msg.get_message()); + } } // namespace snmalloc diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index dc1a1f6..6d64a25 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -150,7 +150,8 @@ namespace snmalloc void* r = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE); if (r == nullptr) - error("out of memory"); + report_fatal_error( + "out of memory: {} ({}) could not be committed", p, size); } /// OS specific function for zeroing memory diff --git a/src/snmalloc_core.h b/src/snmalloc_core.h index d8b14aa..7ae0a33 100644 --- a/src/snmalloc_core.h +++ b/src/snmalloc_core.h @@ -1,6 +1,5 @@ #pragma once -#include "backend/address_space.h" #include "backend/commonconfig.h" #include "backend/pagemap.h" #include "mem/globalalloc.h" diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 1ee9067..08172b3 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -135,10 +135,11 @@ void test_realloc(void* p, size_t size, int err, bool null) START_TEST("realloc({}({}), {})", p, old_size, size); errno = SUCCESS; auto new_p = our_realloc(p, size); - // Realloc failure case, deallocate original block + check_result(size, 1, new_p, err, null); + // Realloc failure case, deallocate original block as not + // handled by check_result. if (new_p == nullptr && size != 0) our_free(p); - check_result(size, 1, new_p, err, null); } void test_posix_memalign(size_t size, size_t align, int err, bool null) diff --git a/src/test/func/redblack/redblack.cc b/src/test/func/redblack/redblack.cc index 2c96541..5901331 100644 --- a/src/test/func/redblack/redblack.cc +++ b/src/test/func/redblack/redblack.cc @@ -73,6 +73,21 @@ public: if (new_is_red != is_red(k)) array[k].left.value ^= 1; } + + static bool compare(key k1, key k2) + { + return k1 > k2; + } + + static bool equal(key k1, key k2) + { + return k1 == k2; + } + + static size_t printable(key k) + { + return k; + } }; template