Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Buddy (#468)

Browse Source 
# Small changes before rewrite

* Additional bit in remote allocator to prevent type confusion with the backend.
* Move Chunk allocator to backend.
* Improvements to RedBlack tree
* Expose message from Pal

# Complete backend rewrite

This provides two key changes:

* We use buddy allocators to allow memory to reconsolidated
* The backend is factored into a series of small operations that
    allocate and deallocate memory.

The backend now uses "Ranges", there are two ranges that don't require a
parent range:
* EmptyRange - Never returns any memory
* PalRange - Returns memory from the platform.

All other ranges require a parent range to supply memory to them.  Some
ranges support both allocation and deallocation, and some just
deallocation.  For instance,  CommitRange supports both, and maps
requests to the parent range, but will Commit and Decommit the memory.

As the ranges perform only a single task, they are generally small and
easy to follow.  The two exceptions to this are the two BuddyRanges
(Large and Small).  Large is for CHUNK_SIZE and above blocks, while
Small is for below CHUNK_SIZE blocks.  Both are implemented with a buddy
allocator, but the SmallBuddyRange uses in place meta-data, while the
LargeBuddyRange uses the pagemap for its meta-data.  This means the
LargeBuddyRange can keep the majority of memory it is managing
decommitted.

The Backend glues together the various ranges to support the appropriate
way to manage memory on the platform.
This commit is contained in:
Matthew Parkinson
2022-03-11 18:16:06 +00:00
committed by GitHub
parent a602643fd2
commit 5287000453
35 changed files with 1410 additions and 972 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
src/backend/address_space.h
View File

@@ -1,201 +0,0 @@
#pragma once
#include "../ds/address.h"
#include "../ds/flaglock.h"
#include "../pal/pal.h"
#include "address_space_core.h"
#include <array>
#ifdef SNMALLOC_TRACING
# include <iostream>
#endif
namespace snmalloc
{
/**
* Implements a power of two allocator, where all blocks are aligned to the
* same power of two as their size. This is what snmalloc uses to get
* alignment of very large sizeclasses.
*
* It cannot unreserve memory, so this does not require the
* usual complexity of a buddy allocator.
*/
template<
SNMALLOC_CONCEPT(ConceptPAL) PAL,
SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap>
class AddressSpaceManager
{
AddressSpaceManagerCore<Pagemap> core;
/**
* This is infrequently used code, a spin lock simplifies the code
* considerably, and should never be on the fast path.
*/
FlagWord spin_lock{};
public:
/**
* Returns a pointer to a block of memory of the supplied size.
* The block will be committed, if specified by the template parameter.
* The returned block is guaranteed to be aligened to the size.
*
* Only request 2^n sizes, and not less than a pointer.
*
* On StrictProvenance architectures, any underlying allocations made as
* part of satisfying the request will be registered with the provided
* arena_map for use in subsequent amplification.
*/
template<bool committed>
capptr::Chunk<void> reserve(size_t size)
{
#ifdef SNMALLOC_TRACING
std::cout << "ASM reserve request:" << size << std::endl;
#endif
SNMALLOC_ASSERT(bits::is_pow2(size));
SNMALLOC_ASSERT(size >= sizeof(void*));
/*
* For sufficiently large allocations with platforms that support aligned
* allocations, try asking the platform directly.
*/
if constexpr (pal_supports<AlignedAllocation, PAL>)
{
if (size >= PAL::minimum_alloc_size)
{
auto base =
capptr::Chunk<void>(PAL::template reserve_aligned<committed>(size));
Pagemap::register_range(address_cast(base), size);
return base;
}
}
capptr::Chunk<void> res;
{
FlagLock lock(spin_lock);
res = core.template reserve<PAL>(size);
if (res == nullptr)
{
// Allocation failed ask OS for more memory
capptr::Chunk<void> block = nullptr;
size_t block_size = 0;
if constexpr (pal_supports<AlignedAllocation, PAL>)
{
/*
* We will have handled the case where size >=
* minimum_alloc_size above, so we are left to handle only small
* things here.
*/
block_size = PAL::minimum_alloc_size;
void* block_raw = PAL::template reserve_aligned<false>(block_size);
// It's a bit of a lie to convert without applying bounds, but the
// platform will have bounded block for us and it's better that
// the rest of our internals expect CBChunk bounds.
block = capptr::Chunk<void>(block_raw);
}
else if constexpr (!pal_supports<NoAllocation, PAL>)
{
// Need at least 2 times the space to guarantee alignment.
bool overflow;
size_t needed_size = bits::umul(size, 2, overflow);
if (overflow)
{
return nullptr;
}
// Magic number (27) for over-allocating a block of memory
// These should be further refined based on experiments.
constexpr size_t min_size = bits::one_at_bit(27);
for (size_t size_request = bits::max(needed_size, min_size);
size_request >= needed_size;
size_request = size_request / 2)
{
block = capptr::Chunk<void>(PAL::reserve(size_request));
if (block != nullptr)
{
block_size = size_request;
break;
}
}
// Ensure block is pointer aligned.
if (
pointer_align_up(block, sizeof(void*)) != block ||
bits::align_up(block_size, sizeof(void*)) > block_size)
{
auto diff =
pointer_diff(block, pointer_align_up(block, sizeof(void*)));
block_size = block_size - diff;
block_size = bits::align_down(block_size, sizeof(void*));
}
}
if (block == nullptr)
{
return nullptr;
}
Pagemap::register_range(address_cast(block), block_size);
core.template add_range<PAL>(block, block_size);
// still holding lock so guaranteed to succeed.
res = core.template reserve<PAL>(size);
}
}
// Don't need lock while committing pages.
if constexpr (committed)
core.template commit_block<PAL>(res, size);
return res;
}
/**
* Aligns block to next power of 2 above size, and unused space at the end
* of the block is retained by the address space manager.
*
* This is useful for allowing the space required for alignment to be
* used, by smaller objects.
*/
template<bool committed>
capptr::Chunk<void> reserve_with_left_over(size_t size)
{
SNMALLOC_ASSERT(size >= sizeof(void*));
size = bits::align_up(size, sizeof(void*));
size_t rsize = bits::next_pow2(size);
auto res = reserve<false>(rsize);
if (res != nullptr)
{
if (rsize > size)
{
FlagLock lock(spin_lock);
core.template add_range<PAL>(pointer_offset(res, size), rsize - size);
}
if constexpr (committed)
core.template commit_block<PAL>(res, size);
}
return res;
}
/**
* Default constructor. An address-space manager constructed in this way
* does not own any memory at the start and will request any that it needs
* from the PAL.
*/
AddressSpaceManager() = default;
/**
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(capptr::Chunk<void> base, size_t length)
{
FlagLock lock(spin_lock);
core.template add_range<PAL>(base, length);
}
};
} // namespace snmalloc

314
src/backend/address_space_core.h
View File

@@ -1,314 +0,0 @@
#pragma once
#include "../ds/address.h"
#include "../ds/flaglock.h"
#include "../mem/allocconfig.h"
#include "../mem/metaslab.h"
#include "../pal/pal.h"
#include "backend_concept.h"
#include <array>
#ifdef SNMALLOC_TRACING
# include <iostream>
#endif
namespace snmalloc
{
/**
* Implements a power of two allocator, where all blocks are aligned to the
* same power of two as their size. This is what snmalloc uses to get
* alignment of very large sizeclasses.
*
* It cannot unreserve memory, so this does not require the
* usual complexity of a buddy allocator.
*
* TODO: This manages pieces of memory smaller than (1U << MIN_CHUNK_BITS) to
* source Metaslab and LocalCache objects. On CHERI, where ASLR and guard
* pages are not needed, it may be worth switching to a design where we
* bootstrap allocators with at least two embedded Metaslab-s that can be used
* to construct slabs for LocalCache and, of course, additional Metaslab
* objects. That would let us stop splitting memory below that threshold
* here, and may reduce address space fragmentation or address space committed
* to Metaslab objects in perpetuity; it could also make {set,get}_next less
* scary.
*/
template<SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap>
class AddressSpaceManagerCore
{
struct FreeChunk
{
capptr::Chunk<FreeChunk> next;
};
/**
* Stores the blocks of address space
*
* The array indexes based on power of two size.
*
* The entries for each size form a linked list. For sizes below
* MIN_CHUNK_SIZE they are linked through the first location in the
* block of memory. For sizes of, and above, MIN_CHUNK_SIZE they are
* linked using the pagemap. We only use the smaller than MIN_CHUNK_SIZE
* allocations for meta-data, so we can be sure that the next pointers
* never occur in a blocks that are ultimately used for object allocations.
*
* bits::BITS is used for simplicity, we do not use below the pointer size,
* and large entries will be unlikely to be supported by the platform.
*/
std::array<capptr::Chunk<FreeChunk>, bits::BITS> ranges = {};
/**
* Checks a block satisfies its invariant.
*/
inline void check_block(capptr::Chunk<FreeChunk> base, size_t align_bits)
{
SNMALLOC_ASSERT(
address_cast(base) ==
bits::align_up(address_cast(base), bits::one_at_bit(align_bits)));
// All blocks need to be bigger than a pointer.
SNMALLOC_ASSERT(bits::one_at_bit(align_bits) >= sizeof(void*));
UNUSED(base, align_bits);
}
/**
* Set next pointer for a power of two address range.
*
* This abstracts the use of either
* - the pagemap; or
* - the first pointer word of the block
* to store the next pointer for the list of unused address space of a
* particular size.
*/
void set_next(
size_t align_bits,
capptr::Chunk<FreeChunk> base,
capptr::Chunk<FreeChunk> next)
{
if (align_bits >= MIN_CHUNK_BITS)
{
// The pagemap stores `MetaEntry`s; abuse the metaslab field to be the
// next block in the stack of blocks.
//
// The pagemap entries here have nullptr as their remote, and so other
// accesses to the pagemap (by external_pointer, for example) will not
// attempt to follow this "Metaslab" pointer.
//
// dealloc() can reject attempts to free such MetaEntry-s due to the
// zero sizeclass.
MetaEntry t(reinterpret_cast<Metaslab*>(next.unsafe_ptr()), nullptr);
Pagemap::set_metaentry(address_cast(base), 1, t);
return;
}
base->next = next;
}
/**
* Get next pointer for a power of two address range.
*
* This abstracts the use of either
* - the pagemap; or
* - the first pointer word of the block
* to store the next pointer for the list of unused address space of a
* particular size.
*/
capptr::Chunk<FreeChunk>
get_next(size_t align_bits, capptr::Chunk<FreeChunk> base)
{
if (align_bits >= MIN_CHUNK_BITS)
{
const MetaEntry& t =
Pagemap::template get_metaentry<false>(address_cast(base));
return capptr::Chunk<FreeChunk>(
reinterpret_cast<FreeChunk*>(t.get_metaslab_no_remote()));
}
return base->next;
}
/**
* Adds a block to `ranges`.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
void add_block(size_t align_bits, capptr::Chunk<FreeChunk> base)
{
check_block(base, align_bits);
SNMALLOC_ASSERT(align_bits < 64);
set_next(align_bits, base, ranges[align_bits]);
ranges[align_bits] = base.template as_static<FreeChunk>();
}
/**
* Find a block of the correct size. May split larger blocks
* to satisfy this request.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
capptr::Chunk<void> remove_block(size_t align_bits)
{
capptr::Chunk<FreeChunk> first = ranges[align_bits];
if (first == nullptr)
{
if (align_bits == (bits::BITS - 1))
{
// Out of memory
errno = ENOMEM;
return nullptr;
}
// Look for larger block and split up recursively
capptr::Chunk<void> bigger = remove_block<PAL>(align_bits + 1);
if (SNMALLOC_UNLIKELY(bigger == nullptr))
return nullptr;
// This block is going to be broken up into sub CHUNK_SIZE blocks
// so we need to commit it to enable the next pointers to be used
// inside the block.
if ((align_bits + 1) == MIN_CHUNK_BITS)
{
commit_block<PAL>(bigger, MIN_CHUNK_SIZE);
}
size_t half_bigger_size = bits::one_at_bit(align_bits);
auto left_over = pointer_offset(bigger, half_bigger_size);
add_block<PAL>(
align_bits,
Aal::capptr_bound<FreeChunk, capptr::bounds::Chunk>(
left_over, half_bigger_size));
check_block(left_over.as_static<FreeChunk>(), align_bits);
check_block(bigger.as_static<FreeChunk>(), align_bits);
return Aal::capptr_bound<void, capptr::bounds::Chunk>(
bigger, half_bigger_size);
}
check_block(first, align_bits);
ranges[align_bits] = get_next(align_bits, first);
return first.as_void();
}
public:
/**
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
void add_range(capptr::Chunk<void> base, size_t length)
{
// For start and end that are not chunk sized, we need to
// commit the pages to track the allocations.
auto base_chunk = pointer_align_up(base, MIN_CHUNK_SIZE);
auto end = pointer_offset(base, length);
auto end_chunk = pointer_align_down(end, MIN_CHUNK_SIZE);
auto start_length = pointer_diff(base, base_chunk);
auto end_length = pointer_diff(end_chunk, end);
if (start_length != 0)
commit_block<PAL>(base, start_length);
if (end_length != 0)
commit_block<PAL>(end_chunk, end_length);
// Find the minimum set of maximally aligned blocks in this range.
// Each block's alignment and size are equal.
while (length >= sizeof(void*))
{
size_t base_align_bits = bits::ctz(address_cast(base));
size_t length_align_bits = (bits::BITS - 1) - bits::clz(length);
size_t align_bits = bits::min(base_align_bits, length_align_bits);
size_t align = bits::one_at_bit(align_bits);
/*
* Now that we have found a maximally-aligned block, we can set bounds
* and be certain that we won't hit representation imprecision.
*/
auto b =
Aal::capptr_bound<FreeChunk, capptr::bounds::Chunk>(base, align);
check_block(b, align_bits);
add_block<PAL>(align_bits, b);
base = pointer_offset(base, align);
length -= align;
}
}
/**
* Commit a block of memory
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
void commit_block(capptr::Chunk<void> base, size_t size)
{
// Rounding required for sub-page allocations.
auto page_start = pointer_align_down<OS_PAGE_SIZE, char>(base);
auto page_end =
pointer_align_up<OS_PAGE_SIZE, char>(pointer_offset(base, size));
size_t using_size = pointer_diff(page_start, page_end);
PAL::template notify_using<NoZero>(page_start.unsafe_ptr(), using_size);
}
/**
* Returns a pointer to a block of memory of the supplied size.
* The block will be committed, if specified by the template parameter.
* The returned block is guaranteed to be aligened to the size.
*
* Only request 2^n sizes, and not less than a pointer.
*
* On StrictProvenance architectures, any underlying allocations made as
* part of satisfying the request will be registered with the provided
* arena_map for use in subsequent amplification.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
capptr::Chunk<void> reserve(size_t size)
{
#ifdef SNMALLOC_TRACING
std::cout << "ASM Core reserve request:" << size << std::endl;
#endif
SNMALLOC_ASSERT(bits::is_pow2(size));
SNMALLOC_ASSERT(size >= sizeof(void*));
return remove_block<PAL>(bits::next_pow2_bits(size));
}
/**
* Aligns block to next power of 2 above size, and unused space at the end
* of the block is retained by the address space manager.
*
* This is useful for allowing the space required for alignment to be
* used by smaller objects.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
capptr::Chunk<void> reserve_with_left_over(size_t size)
{
SNMALLOC_ASSERT(size >= sizeof(void*));
size = bits::align_up(size, sizeof(void*));
size_t rsize = bits::next_pow2(size);
auto res = reserve<PAL>(rsize);
if (res != nullptr)
{
if (rsize > size)
{
/*
* Set bounds on the allocation requested but leave the residual with
* wider bounds for the moment; we'll fix that above in add_range.
*/
size_t residual_size = rsize - size;
auto residual = pointer_offset(res, size);
res = Aal::capptr_bound<void, capptr::bounds::Chunk>(res, size);
add_range<PAL>(residual, residual_size);
}
}
return res;
}
/**
* Default constructor. An address-space manager constructed in this way
* does not own any memory at the start and will request any that it needs
* from the PAL.
*/
AddressSpaceManagerCore() = default;
};
} // namespace snmalloc

400
src/backend/backend.h
View File

@@ -2,9 +2,19 @@
#include "../mem/allocconfig.h"
#include "../mem/metaslab.h"
#include "../pal/pal.h"
#include "address_space.h"
#include "chunkallocator.h"
#include "commitrange.h"
#include "commonconfig.h"
#include "empty_range.h"
#include "globalrange.h"
#include "largebuddyrange.h"
#include "pagemap.h"
#include "pagemapregisterrange.h"
#include "palrange.h"
#include "range_helpers.h"
#include "smallbuddyrange.h"
#include "statsrange.h"
#include "subrange.h"
#if defined(SNMALLOC_CHECK_CLIENT) && !defined(OPEN_ENCLAVE)
/**
@@ -21,213 +31,6 @@
namespace snmalloc
{
/**
* This helper class implements the core functionality to allocate from an
* address space and pagemap. Any backend implementation can use this class to
* help with basic address space managment.
*/
template<
SNMALLOC_CONCEPT(ConceptPAL) PAL,
typename LocalState,
SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap>
class AddressSpaceAllocatorCommon
{
// Size of local address space requests. Currently aimed at 2MiB large
// pages but should make this configurable (i.e. for OE, so we don't need as
// much space).
#ifdef OPEN_ENCLAVE
// Don't prefetch address space on OE, as it is limited.
// This could cause perf issues during warm-up phases.
constexpr static size_t LOCAL_CACHE_BLOCK = 0;
#else
constexpr static size_t LOCAL_CACHE_BLOCK = bits::one_at_bit(21);
#endif
#ifdef SNMALLOC_META_PROTECTED
// When protecting the meta-data, we use a smaller block for the meta-data
// that is randomised inside a larger block. This needs to be at least a
// page so that we can use guard pages.
constexpr static size_t LOCAL_CACHE_META_BLOCK =
bits::max(MIN_CHUNK_SIZE * 2, OS_PAGE_SIZE);
static_assert(
LOCAL_CACHE_META_BLOCK <= LOCAL_CACHE_BLOCK,
"LOCAL_CACHE_META_BLOCK must be smaller than LOCAL_CACHE_BLOCK");
#endif
public:
/**
* Provide a block of meta-data with size and align.
*
* Backend allocator may use guard pages and separate area of
* address space to protect this from corruption.
*/
static capptr::Chunk<void> alloc_meta_data(
AddressSpaceManager<PAL, Pagemap>& global,
LocalState* local_state,
size_t size)
{
return reserve<true>(global, local_state, size);
}
/**
* Returns a chunk of memory with alignment and size of `size`, and a
* metaslab block.
*
* It additionally set the meta-data for this chunk of memory to
* be
* (remote, sizeclass, metaslab)
* where metaslab, is the second element of the pair return.
*/
static std::pair<capptr::Chunk<void>, Metaslab*> alloc_chunk(
AddressSpaceManager<PAL, Pagemap>& global,
LocalState* local_state,
size_t size,
RemoteAllocator* remote,
sizeclass_t sizeclass)
{
SNMALLOC_ASSERT(bits::is_pow2(size));
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
auto meta = reinterpret_cast<Metaslab*>(
reserve<true>(global, local_state, sizeof(Metaslab)).unsafe_ptr());
if (meta == nullptr)
return {nullptr, nullptr};
capptr::Chunk<void> p = reserve<false>(global, local_state, size);
#ifdef SNMALLOC_TRACING
std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")"
<< std::endl;
#endif
if (p == nullptr)
{
// TODO: This is leaking `meta`. Currently there is no facility for
// meta-data reuse, so will leave until we develop more expressive
// meta-data management.
#ifdef SNMALLOC_TRACING
std::cout << "Out of memory" << std::endl;
#endif
return {p, nullptr};
}
meta->meta_common.chunk = p;
MetaEntry t(meta, remote, sizeclass);
Pagemap::set_metaentry(address_cast(p), size, t);
return {p, meta};
}
private:
/**
* Internal method for acquiring state from the local and global address
* space managers.
*/
template<bool is_meta>
static capptr::Chunk<void> reserve(
AddressSpaceManager<PAL, Pagemap>& global,
LocalState* local_state,
size_t size)
{
#ifdef SNMALLOC_META_PROTECTED
constexpr auto MAX_CACHED_SIZE =
is_meta ? LOCAL_CACHE_META_BLOCK : LOCAL_CACHE_BLOCK;
#else
constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK;
#endif
capptr::Chunk<void> p;
if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE))
{
#ifdef SNMALLOC_META_PROTECTED
auto& local = is_meta ? local_state->local_meta_address_space :
local_state->local_address_space;
#else
auto& local = local_state->local_address_space;
#endif
p = local.template reserve_with_left_over<PAL>(size);
if (p != nullptr)
{
return p;
}
auto refill_size = LOCAL_CACHE_BLOCK;
auto refill = global.template reserve<false>(refill_size);
if (refill == nullptr)
return nullptr;
#ifdef SNMALLOC_META_PROTECTED
if (is_meta)
{
refill = sub_range(refill, LOCAL_CACHE_BLOCK, LOCAL_CACHE_META_BLOCK);
refill_size = LOCAL_CACHE_META_BLOCK;
}
#endif
PAL::template notify_using<NoZero>(refill.unsafe_ptr(), refill_size);
local.template add_range<PAL>(refill, refill_size);
// This should succeed
return local.template reserve_with_left_over<PAL>(size);
}
#ifdef SNMALLOC_META_PROTECTED
// During start up we need meta-data before we have a local allocator
// This code protects that meta-data with randomisation, and guard pages.
if (local_state == nullptr && is_meta)
{
size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size));
size_t size_request = rsize * 64;
p = global.template reserve<false>(size_request);
if (p == nullptr)
return nullptr;
p = sub_range(p, size_request, rsize);
PAL::template notify_using<NoZero>(p.unsafe_ptr(), rsize);
return p;
}
// This path does not apply any guard pages to very large
// meta data requests. There are currently no meta data-requests
// this large. This assert checks for this assumption breaking.
SNMALLOC_ASSERT(!is_meta);
#endif
p = global.template reserve_with_left_over<true>(size);
return p;
}
#ifdef SNMALLOC_META_PROTECTED
/**
* Returns a sub-range of [return, return+sub_size] that is contained in
* the range [base, base+full_size]. The first and last slot are not used
* so that the edges can be used for guard pages.
*/
static capptr::Chunk<void>
sub_range(capptr::Chunk<void> base, size_t full_size, size_t sub_size)
{
SNMALLOC_ASSERT(bits::is_pow2(full_size));
SNMALLOC_ASSERT(bits::is_pow2(sub_size));
SNMALLOC_ASSERT(full_size % sub_size == 0);
SNMALLOC_ASSERT(full_size / sub_size >= 4);
size_t offset_mask = full_size - sub_size;
// Don't use first or last block in the larger reservation
// Loop required to get uniform distribution.
size_t offset;
do
{
offset = get_entropy64<PAL>() & offset_mask;
} while ((offset == 0) || (offset == offset_mask));
return pointer_offset(base, offset);
}
#endif
};
/**
* This class implements the standard backend for handling allocations.
* It abstracts page table management and address space management.
@@ -278,7 +81,7 @@ namespace snmalloc
* Set the metadata associated with a chunk.
*/
SNMALLOC_FAST_PATH
static void set_metaentry(address_t p, size_t size, MetaEntry t)
static void set_metaentry(address_t p, size_t size, const MetaEntry& t)
{
for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE)
{
@@ -313,38 +116,70 @@ namespace snmalloc
}
};
/**
* Local state for the backend allocator.
*
* This class contains thread local structures to make the implementation
* of the backend allocator more efficient.
*/
class LocalState
{
template<
SNMALLOC_CONCEPT(ConceptPAL) PAL2,
typename LocalState,
SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap2>
friend class AddressSpaceAllocatorCommon;
#if defined(_WIN32) || defined(__CHERI_PURE_CAPABILITY__)
static constexpr bool CONSOLIDATE_PAL_ALLOCS = false;
#else
static constexpr bool CONSOLIDATE_PAL_ALLOCS = true;
#endif
AddressSpaceManagerCore<Pagemap> local_address_space;
#if defined(OPEN_ENCLAVE)
// Single global buddy allocator is used on open enclave due to
// the limited address space.
using GlobalR = GlobalRange<StatsR<SmallBuddyRange<
LargeBuddyRange<EmptyRange, bits::BITS - 1, bits::BITS - 1, Pagemap>>>>;
using ObjectRange = GlobalR;
using GlobalMetaRange = ObjectRange;
#else
// Set up source of memory
using P = PalRange<DefaultPal>;
using Base = std::
conditional_t<fixed_range, EmptyRange, PagemapRegisterRange<Pagemap, P>>;
// Global range of memory
using StatsR = StatsRange<LargeBuddyRange<
Base,
24,
bits::BITS - 1,
Pagemap,
CONSOLIDATE_PAL_ALLOCS>>;
using GlobalR = GlobalRange<StatsR>;
# ifdef SNMALLOC_META_PROTECTED
// Source for object allocations
using ObjectRange =
LargeBuddyRange<CommitRange<GlobalR, DefaultPal>, 21, 21, Pagemap>;
// Set up protected range for metadata
using SubR = CommitRange<SubRange<GlobalR, DefaultPal, 6>, DefaultPal>;
using MetaRange =
SmallBuddyRange<LargeBuddyRange<SubR, 21 - 6, bits::BITS - 1, Pagemap>>;
using GlobalMetaRange = GlobalRange<MetaRange>;
# else
// Source for object allocations and metadata
// No separation between the two
using ObjectRange = SmallBuddyRange<
LargeBuddyRange<CommitRange<GlobalR, DefaultPal>, 21, 21, Pagemap>>;
using GlobalMetaRange = GlobalRange<ObjectRange>;
# endif
#endif
struct LocalState
{
typename ObjectRange::State object_range;
#ifdef SNMALLOC_META_PROTECTED
/**
* Secondary local address space, so we can apply some randomisation
* and guard pages to protect the meta-data.
*/
AddressSpaceManagerCore<Pagemap> local_meta_address_space;
typename MetaRange::State meta_range;
typename MetaRange::State& get_meta_range()
{
return meta_range;
}
#else
typename ObjectRange::State& get_meta_range()
{
return object_range;
}
#endif
};
SNMALLOC_REQUIRE_CONSTINIT
static inline AddressSpaceManager<PAL, Pagemap> address_space;
private:
using AddressSpaceAllocator =
AddressSpaceAllocatorCommon<Pal, LocalState, Pagemap>;
public:
template<bool fixed_range_ = fixed_range>
static std::enable_if_t<!fixed_range_> init()
@@ -361,7 +196,17 @@ namespace snmalloc
auto [heap_base, heap_length] =
Pagemap::concretePagemap.init(base, length);
address_space.add_range(capptr::Chunk<void>(heap_base), heap_length);
Pagemap::register_range(address_cast(heap_base), heap_length);
// Push memory into the global range.
range_to_pow_2_blocks<MIN_CHUNK_BITS>(
capptr::Chunk<void>(heap_base),
heap_length,
[&](capptr::Chunk<void> p, size_t sz, bool) {
typename GlobalR::State g;
g->dealloc_range(p, sz);
});
}
/**
@@ -378,8 +223,21 @@ namespace snmalloc
static capptr::Chunk<void>
alloc_meta_data(LocalState* local_state, size_t size)
{
return AddressSpaceAllocator::alloc_meta_data(
address_space, local_state, size);
capptr::Chunk<void> p;
if (local_state != nullptr)
{
p = local_state->get_meta_range()->alloc_range_with_leftover(size);
}
else
{
typename GlobalMetaRange::State global_state;
p = global_state->alloc_range(bits::next_pow2(size));
}
if (p == nullptr)
errno = ENOMEM;
return p;
}
/**
@@ -392,13 +250,73 @@ namespace snmalloc
* where metaslab, is the second element of the pair return.
*/
static std::pair<capptr::Chunk<void>, Metaslab*> alloc_chunk(
LocalState* local_state,
LocalState& local_state,
size_t size,
RemoteAllocator* remote,
sizeclass_t sizeclass)
{
return AddressSpaceAllocator::alloc_chunk(
address_space, local_state, size, remote, sizeclass);
SNMALLOC_ASSERT(bits::is_pow2(size));
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
auto meta_cap =
local_state.get_meta_range()->alloc_range(sizeof(Metaslab));
auto meta = meta_cap.template as_reinterpret<Metaslab>().unsafe_ptr();
if (meta == nullptr)
{
errno = ENOMEM;
return {nullptr, nullptr};
}
auto p = local_state.object_range->alloc_range(size);
#ifdef SNMALLOC_TRACING
std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")"
<< std::endl;
#endif
if (p == nullptr)
{
local_state.get_meta_range()->dealloc_range(meta_cap, sizeof(Metaslab));
errno = ENOMEM;
#ifdef SNMALLOC_TRACING
std::cout << "Out of memory" << std::endl;
#endif
return {p, nullptr};
}
meta->meta_common.chunk = p;
MetaEntry t(meta, remote, sizeclass);
Pagemap::set_metaentry(address_cast(p), size, t);
p = Aal::capptr_bound<void, capptr::bounds::Chunk>(p, size);
return {p, meta};
}
static void dealloc_chunk(
LocalState& local_state, ChunkRecord* chunk_record, size_t size)
{
auto chunk = chunk_record->meta_common.chunk;
local_state.get_meta_range()->dealloc_range(
capptr::Chunk<void>(chunk_record), sizeof(Metaslab));
// TODO, should we set the sizeclass to something specific here?
local_state.object_range->dealloc_range(chunk, size);
}
static size_t get_current_usage()
{
typename StatsR::State stats_state;
return stats_state->get_current_usage();
}
static size_t get_peak_usage()
{
typename StatsR::State stats_state;
return stats_state->get_peak_usage();
}
};
} // namespace snmalloc

4
src/backend/backend_concept.h
View File

@@ -4,7 +4,7 @@
# include <cstddef>
# include "../ds/concept.h"
# include "../pal/pal_concept.h"
# include "../ds/address.h"
namespace snmalloc
{
class MetaEntry;
@@ -20,7 +20,7 @@ namespace snmalloc
requires(
address_t addr,
size_t sz,
MetaEntry t)
const MetaEntry& t)
{
{ Meta::set_metaentry(addr, sz, t) } -> ConceptSame<void>;

122
src/backend/buddy.h Normal file
View File

@@ -0,0 +1,122 @@
#pragma once
#include "../ds/address.h"
#include "../ds/bits.h"
#include "../ds/redblacktree.h"
namespace snmalloc
{
/**
* Class representing a buddy allocator
*
* Underlying node `Rep` representation is passed in.
*
* The allocator can handle blocks between inclusive MIN_SIZE_BITS and
* exclusive MAX_SIZE_BITS.
*/
template<typename Rep, size_t MIN_SIZE_BITS, size_t MAX_SIZE_BITS>
class Buddy
{
std::array<RBTree<Rep>, MAX_SIZE_BITS - MIN_SIZE_BITS> trees;
size_t to_index(size_t size)
{
auto log = snmalloc::bits::next_pow2_bits(size);
SNMALLOC_ASSERT(log >= MIN_SIZE_BITS);
SNMALLOC_ASSERT(log < MAX_SIZE_BITS);
return log - MIN_SIZE_BITS;
}
void validate_block(typename Rep::Contents addr, size_t size)
{
SNMALLOC_ASSERT(bits::is_pow2(size));
SNMALLOC_ASSERT(addr == Rep::align_down(addr, size));
UNUSED(addr, size);
}
public:
constexpr Buddy() = default;
/**
* Add a block to the buddy allocator.
*
* Blocks needs to be power of two size and aligned to the same power of
* two.
*
* Returns null, if the block is successfully added. Otherwise, returns the
* consolidated block that is MAX_SIZE_BITS big, and hence too large for
* this allocator.
*/
typename Rep::Contents add_block(typename Rep::Contents addr, size_t size)
{
auto idx = to_index(size);
validate_block(addr, size);
auto buddy = Rep::buddy(addr, size);
auto path = trees[idx].get_root_path();
bool contains_buddy = trees[idx].find(path, buddy);
if (contains_buddy)
{
// Only check if we can consolidate after we know the buddy is in
// the buddy allocator. This is required to prevent possible segfaults
// from looking at the buddies meta-data, which we only know exists
// once we have found it in the red-black tree.
if (Rep::can_consolidate(addr, size))
{
trees[idx].remove_path(path);
// Add to next level cache
size *= 2;
addr = Rep::align_down(addr, size);
if (size == bits::one_at_bit(MAX_SIZE_BITS))
// Too big for this buddy allocator.
return addr;
return add_block(addr, size);
}
// Re-traverse as the path was to the buddy,
// but the representation says we cannot combine.
// We must find the correct place for this element.
// Something clever could be done here, but it's not worth it.
// path = trees[idx].get_root_path();
trees[idx].find(path, addr);
}
trees[idx].insert_path(path, addr);
return Rep::null;
}
/**
* Removes a block of size from the buddy allocator.
*
* Return Rep::null if this cannot be satisfied.
*/
typename Rep::Contents remove_block(size_t size)
{
auto idx = to_index(size);
auto addr = trees[idx].remove_min();
if (addr != Rep::null)
{
validate_block(addr, size);
return addr;
}
if (size * 2 == bits::one_at_bit(MAX_SIZE_BITS))
// Too big for this buddy allocator
return Rep::null;
auto bigger = remove_block(size * 2);
if (bigger == Rep::null)
return Rep::null;
auto second = Rep::offset(bigger, size);
// Split large block
add_block(second, size);
return bigger;
}
};
} // namespace snmalloc

9
src/mem/chunkallocator.h → src/backend/chunkallocator.h
View File

@@ -1,5 +1,11 @@
#pragma once
/***
* WARNING: This file is not currently in use. The functionality has not
* be transistioned to the new backend. It does not seem to be required
* but further testing is required before we delete it.
*/
#include "../backend/backend_concept.h"
#include "../ds/mpmcstack.h"
#include "../ds/spmcstack.h"
@@ -316,7 +322,10 @@ namespace snmalloc
SharedStateHandle::template alloc_meta_data<U>(local_state, size);
if (p == nullptr)
{
errno = ENOMEM;
return nullptr;
}
return new (p.unsafe_ptr()) U(std::forward<Args>(args)...);
}

44
src/backend/commitrange.h Normal file
View File

@@ -0,0 +1,44 @@
#pragma once
#include "../ds/ptrwrap.h"
namespace snmalloc
{
template<typename ParentRange, typename PAL>
class CommitRange
{
typename ParentRange::State parent{};
public:
class State
{
CommitRange commit_range{};
public:
constexpr State() = default;
CommitRange* operator->()
{
return &commit_range;
}
};
static constexpr bool Aligned = ParentRange::Aligned;
constexpr CommitRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
auto range = parent->alloc_range(size);
if (range != nullptr)
PAL::template notify_using<NoZero>(range.unsafe_ptr(), size);
return range;
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
PAL::notify_not_using(base.unsafe_ptr(), size);
parent->dealloc_range(base, size);
}
};
} // namespace snmalloc

2
src/backend/commonconfig.h
View File

@@ -1,6 +1,8 @@
#pragma once
#include "../backend/backend_concept.h"
#include "../ds/defines.h"
#include "../mem/remoteallocator.h"
namespace snmalloc
{

29
src/backend/empty_range.h Normal file
View File

@@ -0,0 +1,29 @@
#include "../ds/ptrwrap.h"
namespace snmalloc
{
class EmptyRange
{
public:
class State
{
public:
EmptyRange* operator->()
{
static EmptyRange range{};
return &range;
}
constexpr State() = default;
};
static constexpr bool Aligned = true;
constexpr EmptyRange() = default;
capptr::Chunk<void> alloc_range(size_t)
{
return nullptr;
}
};
} // namespace snmalloc

9
src/backend/fixedglobalconfig.h
View File

@@ -1,7 +1,7 @@
#pragma once
#include "../backend/backend.h"
#include "../mem/chunkallocator.h"
#include "../backend/chunkallocator.h"
#include "../mem/corealloc.h"
#include "../mem/pool.h"
#include "commonconfig.h"
@@ -19,17 +19,10 @@ namespace snmalloc
private:
using Backend = BackendAllocator<PAL, true>;
inline static ChunkAllocatorState chunk_allocator_state;
inline static GlobalPoolState alloc_pool;
public:
static ChunkAllocatorState&
get_chunk_allocator_state(typename Backend::LocalState* = nullptr)
{
return chunk_allocator_state;
}
static GlobalPoolState& pool()
{
return alloc_pool;

10
src/backend/globalconfig.h
View File

@@ -1,7 +1,7 @@
#pragma once
#include "../backend/backend.h"
#include "../mem/chunkallocator.h"
#include "../backend/chunkallocator.h"
#include "../mem/corealloc.h"
#include "../mem/pool.h"
#include "commonconfig.h"
@@ -36,8 +36,6 @@ namespace snmalloc
private:
using Backend = BackendAllocator<Pal, false>;
SNMALLOC_REQUIRE_CONSTINIT
inline static ChunkAllocatorState chunk_allocator_state;
SNMALLOC_REQUIRE_CONSTINIT
inline static GlobalPoolState alloc_pool;
@@ -49,12 +47,6 @@ namespace snmalloc
inline static FlagWord initialisation_lock{};
public:
static ChunkAllocatorState&
get_chunk_allocator_state(Backend::LocalState* = nullptr)
{
return chunk_allocator_state;
}
static GlobalPoolState& pool()
{
return alloc_pool;

54
src/backend/globalrange.h Normal file
View File

@@ -0,0 +1,54 @@
#pragma once
#include "../ds/defines.h"
#include "../ds/helpers.h"
#include "../ds/ptrwrap.h"
namespace snmalloc
{
/**
* Makes the supplied ParentRange into a global variable,
* and protects access with a lock.
*/
template<typename ParentRange>
class GlobalRange
{
typename ParentRange::State parent{};
/**
* This is infrequently used code, a spin lock simplifies the code
* considerably, and should never be on the fast path.
*/
FlagWord spin_lock{};
public:
class State
{
SNMALLOC_REQUIRE_CONSTINIT static inline GlobalRange global_range{};
public:
constexpr GlobalRange* operator->()
{
return &global_range;
}
constexpr State() = default;
};
static constexpr bool Aligned = ParentRange::Aligned;
constexpr GlobalRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
FlagLock lock(spin_lock);
return parent->alloc_range(size);
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
FlagLock lock(spin_lock);
parent->dealloc_range(base, size);
}
};
} // namespace snmalloc

289
src/backend/largebuddyrange.h Normal file
View File

@@ -0,0 +1,289 @@
#pragma once
#include "../ds/address.h"
#include "../ds/bits.h"
#include "../mem/allocconfig.h"
#include "../mem/metaslab.h"
#include "../pal/pal.h"
#include "buddy.h"
#include "range_helpers.h"
#include <string>
namespace snmalloc
{
/**
* Class for using the pagemap entries for the buddy allocator.
*/
template<typename Pagemap>
class BuddyChunkRep
{
public:
using Holder = uintptr_t;
using Contents = uintptr_t;
static constexpr address_t RED_BIT = 2;
static constexpr Contents null = 0;
static void set(Holder* ptr, Contents r)
{
SNMALLOC_ASSERT((r & (MIN_CHUNK_SIZE - 1)) == 0);
// Preserve lower bits.
*ptr = r | address_cast(*ptr & (MIN_CHUNK_SIZE - 1)) | BACKEND_MARKER;
}
static Contents get(const Holder* ptr)
{
return *ptr & ~(MIN_CHUNK_SIZE - 1);
}
static Holder& ref(bool direction, Contents k)
{
MetaEntry& entry =
Pagemap::template get_metaentry_mut<false>(address_cast(k));
if (direction)
return *reinterpret_cast<Holder*>(&entry.meta);
return *reinterpret_cast<Holder*>(&entry.remote_and_sizeclass);
}
static bool is_red(Contents k)
{
return (ref(true, k) & RED_BIT) == RED_BIT;
}
static void set_red(Contents k, bool new_is_red)
{
if (new_is_red != is_red(k))
ref(true, k) ^= RED_BIT;
}
static Contents offset(Contents k, size_t size)
{
return k + size;
}
static Contents buddy(Contents k, size_t size)
{
return k ^ size;
}
static Contents align_down(Contents k, size_t size)
{
return k & ~(size - 1);
}
static bool compare(Contents k1, Contents k2)
{
return k1 > k2;
}
static bool equal(Contents k1, Contents k2)
{
return k1 == k2;
}
static uintptr_t printable(Contents k)
{
return k;
}
static bool can_consolidate(Contents k, size_t size)
{
// Need to know both entries exist in the pagemap.
// This must only be called if that has already been
// ascertained.
// The buddy could be in a part of the pagemap that has
// not been registered and thus could segfault on access.
auto larger = bits::max(k, buddy(k, size));
MetaEntry& entry =
Pagemap::template get_metaentry_mut<false>(address_cast(larger));
return !entry.is_boundary();
}
};
template<
typename ParentRange,
size_t REFILL_SIZE_BITS,
size_t MAX_SIZE_BITS,
SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap,
bool Consolidate = true>
class LargeBuddyRange
{
typename ParentRange::State parent{};
static constexpr size_t REFILL_SIZE = bits::one_at_bit(REFILL_SIZE_BITS);
/**
*
*/
Buddy<BuddyChunkRep<Pagemap>, MIN_CHUNK_BITS, MAX_SIZE_BITS> buddy_large;
/**
* The parent might not support deallocation if this buddy allocator covers
* the whole range. Uses template insanity to make this work.
*/
template<bool exists = MAX_SIZE_BITS != (bits::BITS - 1)>
std::enable_if_t<exists>
parent_dealloc_range(capptr::Chunk<void> base, size_t size)
{
static_assert(
MAX_SIZE_BITS != (bits::BITS - 1), "Don't set SFINAE parameter");
parent->dealloc_range(base, size);
}
void dealloc_overflow(capptr::Chunk<void> overflow)
{
if constexpr (MAX_SIZE_BITS != (bits::BITS - 1))
{
if (overflow != nullptr)
{
parent->dealloc_range(overflow, bits::one_at_bit(MAX_SIZE_BITS));
}
}
else
{
if (overflow != nullptr)
abort();
}
}
/**
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(capptr::Chunk<void> base, size_t length)
{
range_to_pow_2_blocks<MIN_CHUNK_BITS>(
base,
length,
[this](capptr::Chunk<void> base, size_t align, bool first) {
if constexpr (!Consolidate)
{
// Tag first entry so we don't consolidate it.
if (first)
{
Pagemap::get_metaentry_mut(address_cast(base)).set_boundary();
}
}
else
{
UNUSED(first);
}
auto overflow = capptr::Chunk<void>(reinterpret_cast<void*>(
buddy_large.add_block(base.unsafe_uintptr(), align)));
dealloc_overflow(overflow);
});
}
capptr::Chunk<void> refill(size_t size)
{
if (ParentRange::Aligned)
{
// TODO have to add consolidation blocker for these cases.
if (size >= REFILL_SIZE)
{
return parent->alloc_range(size);
}
auto refill_range = parent->alloc_range(REFILL_SIZE);
if (refill_range != nullptr)
add_range(pointer_offset(refill_range, size), REFILL_SIZE - size);
return refill_range;
}
// Need to overallocate to get the alignment right.
bool overflow = false;
size_t needed_size = bits::umul(size, 2, overflow);
if (overflow)
{
return nullptr;
}
auto refill_size = bits::max(needed_size, REFILL_SIZE);
while (needed_size <= refill_size)
{
auto refill = parent->alloc_range(refill_size);
if (refill != nullptr)
{
add_range(refill, refill_size);
SNMALLOC_ASSERT(refill_size < bits::one_at_bit(MAX_SIZE_BITS));
static_assert(
(REFILL_SIZE < bits::one_at_bit(MAX_SIZE_BITS)) ||
ParentRange::Aligned,
"Required to prevent overflow.");
return alloc_range(size);
}
refill_size >>= 1;
}
return nullptr;
}
public:
class State
{
LargeBuddyRange buddy_range;
public:
LargeBuddyRange* operator->()
{
return &buddy_range;
}
constexpr State() = default;
};
static constexpr bool Aligned = true;
constexpr LargeBuddyRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
SNMALLOC_ASSERT(bits::is_pow2(size));
if (size >= (bits::one_at_bit(MAX_SIZE_BITS) - 1))
{
if (ParentRange::Aligned)
return parent->alloc_range(size);
return nullptr;
}
auto result = capptr::Chunk<void>(
reinterpret_cast<void*>(buddy_large.remove_block(size)));
if (result != nullptr)
return result;
return refill(size);
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
SNMALLOC_ASSERT(bits::is_pow2(size));
if constexpr (MAX_SIZE_BITS != (bits::BITS - 1))
{
if (size >= (bits::one_at_bit(MAX_SIZE_BITS) - 1))
{
parent_dealloc_range(base, size);
return;
}
}
auto overflow = capptr::Chunk<void>(reinterpret_cast<void*>(
buddy_large.add_block(base.unsafe_uintptr(), size)));
dealloc_overflow(overflow);
}
};
} // namespace snmalloc

6
src/backend/pagemap.h
View File

@@ -115,7 +115,7 @@ namespace snmalloc
static_assert(
has_bounds_ == has_bounds, "Don't set SFINAE template parameter!");
#ifdef SNMALLOC_TRACING
std::cout << "Pagemap.init " << b << " (" << s << ")" << std::endl;
message<1024>("Pagemap.init {} ({})", b, s);
#endif
SNMALLOC_ASSERT(s != 0);
// TODO take account of pagemap size in the calculation of how big it
@@ -311,10 +311,10 @@ namespace snmalloc
return base + (entry_index << GRANULARITY_BITS);
}
void set(address_t p, T t)
void set(address_t p, const T& t)
{
#ifdef SNMALLOC_TRACING
std::cout << "Pagemap.Set " << (void*)(uintptr_t)p << std::endl;
message<1024>("Pagemap.Set {}", p);
#endif
if constexpr (has_bounds)
{

43
src/backend/pagemapregisterrange.h Normal file
View File

@@ -0,0 +1,43 @@
#pragma once
#include "../ds/address.h"
#include "../pal/pal.h"
namespace snmalloc
{
template<
SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap,
typename ParentRange>
class PagemapRegisterRange
{
typename ParentRange::State state{};
public:
class State
{
PagemapRegisterRange range;
public:
PagemapRegisterRange* operator->()
{
return &range;
}
constexpr State() = default;
};
constexpr PagemapRegisterRange() = default;
static constexpr bool Aligned = ParentRange::Aligned;
capptr::Chunk<void> alloc_range(size_t size)
{
auto base = state->alloc_range(size);
if (base != nullptr)
Pagemap::register_range(address_cast(base), size);
return base;
}
};
} // namespace snmalloc

59
src/backend/palrange.h Normal file
View File

@@ -0,0 +1,59 @@
#pragma once
#include "../ds/address.h"
#include "../pal/pal.h"
namespace snmalloc
{
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
class PalRange
{
public:
class State
{
public:
PalRange* operator->()
{
// There is no state required for the PalRange
// using a global just to satisfy the typing.
static PalRange range{};
return &range;
}
constexpr State() = default;
};
static constexpr bool Aligned = pal_supports<AlignedAllocation, PAL>;
constexpr PalRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
if (bits::next_pow2_bits(size) >= bits::BITS - 1)
{
return nullptr;
}
if constexpr (pal_supports<AlignedAllocation, PAL>)
{
SNMALLOC_ASSERT(size >= PAL::minimum_alloc_size);
auto result =
capptr::Chunk<void>(PAL::template reserve_aligned<false>(size));
#ifdef SNMALLOC_TRACING
message<1024>("Pal range alloc: {} ({})", result.unsafe_ptr(), size);
#endif
return result;
}
else
{
auto result = capptr::Chunk<void>(PAL::reserve(size));
#ifdef SNMALLOC_TRACING
message<1024>("Pal range alloc: {} ({})", result.unsafe_ptr(), size);
#endif
return result;
}
}
};
} // namespace snmalloc

37
src/backend/range_helpers.h Normal file
View File

@@ -0,0 +1,37 @@
#pragma once
#include "../ds/ptrwrap.h"
namespace snmalloc
{
template<size_t MIN_BITS, typename F>
void range_to_pow_2_blocks(capptr::Chunk<void> base, size_t length, F f)
{
auto end = pointer_offset(base, length);
base = pointer_align_up(base, bits::one_at_bit(MIN_BITS));
end = pointer_align_down(end, bits::one_at_bit(MIN_BITS));
length = pointer_diff(base, end);
bool first = true;
// Find the minimum set of maximally aligned blocks in this range.
// Each block's alignment and size are equal.
while (length >= sizeof(void*))
{
size_t base_align_bits = bits::ctz(address_cast(base));
size_t length_align_bits = (bits::BITS - 1) - bits::clz(length);
size_t align_bits = bits::min(base_align_bits, length_align_bits);
size_t align = bits::one_at_bit(align_bits);
/*
* Now that we have found a maximally-aligned block, we can set bounds
* and be certain that we won't hit representation imprecision.
*/
f(base, align, first);
first = false;
base = pointer_offset(base, align);
length -= align;
}
}
} // namespace snmalloc

226
src/backend/smallbuddyrange.h Normal file
View File

@@ -0,0 +1,226 @@
#pragma once
#include "../ds/address.h"
#include "../pal/pal.h"
#include "range_helpers.h"
namespace snmalloc
{
/**
* struct for representing the redblack nodes
* directly inside the meta data.
*/
struct FreeChunk
{
capptr::Chunk<FreeChunk> left;
capptr::Chunk<FreeChunk> right;
};
/**
* Class for using the allocations own space to store in the RBTree.
*/
class BuddyInplaceRep
{
public:
using Holder = capptr::Chunk<FreeChunk>;
using Contents = capptr::Chunk<FreeChunk>;
static constexpr Contents null = nullptr;
static constexpr address_t MASK = 1;
static void set(Holder* ptr, Contents r)
{
SNMALLOC_ASSERT((address_cast(r) & MASK) == 0);
if (r == nullptr)
*ptr = capptr::Chunk<FreeChunk>(
reinterpret_cast<FreeChunk*>((*ptr).unsafe_uintptr() & MASK));
else
// Preserve lower bit.
*ptr = pointer_offset(r, (address_cast(*ptr) & MASK))
.template as_static<FreeChunk>();
}
static Contents get(Holder* ptr)
{
return pointer_align_down<2, FreeChunk>((*ptr).as_void());
}
static Holder& ref(bool direction, Contents r)
{
if (direction)
return r->left;
return r->right;
}
static bool is_red(Contents k)
{
if (k == nullptr)
return false;
return (address_cast(ref(false, k)) & MASK) == MASK;
}
static void set_red(Contents k, bool new_is_red)
{
if (new_is_red != is_red(k))
{
auto& r = ref(false, k);
auto old_addr = pointer_align_down<2, FreeChunk>(r.as_void());
if (new_is_red)
{
if (old_addr == nullptr)
r = capptr::Chunk<FreeChunk>(reinterpret_cast<FreeChunk*>(MASK));
else
r = pointer_offset(old_addr, MASK).template as_static<FreeChunk>();
}
else
{
r = old_addr;
}
}
}
static Contents offset(Contents k, size_t size)
{
return pointer_offset(k, size).template as_static<FreeChunk>();
}
static Contents buddy(Contents k, size_t size)
{
// This is just doing xor size, but with what API
// exists on capptr.
auto base = pointer_align_down<FreeChunk>(k.as_void(), size * 2);
auto offset = (address_cast(k) & size) ^ size;
return pointer_offset(base, offset).template as_static<FreeChunk>();
}
static Contents align_down(Contents k, size_t size)
{
return pointer_align_down<FreeChunk>(k.as_void(), size);
}
static bool compare(Contents k1, Contents k2)
{
return address_cast(k1) > address_cast(k2);
}
static bool equal(Contents k1, Contents k2)
{
return address_cast(k1) == address_cast(k2);
}
static address_t printable(Contents k)
{
return address_cast(k);
}
static bool can_consolidate(Contents k, size_t size)
{
UNUSED(k, size);
return true;
}
};
template<typename ParentRange>
class SmallBuddyRange
{
typename ParentRange::State parent{};
static constexpr size_t MIN_BITS =
bits::next_pow2_bits_const(sizeof(FreeChunk));
Buddy<BuddyInplaceRep, MIN_BITS, MIN_CHUNK_BITS> buddy_small;
/**
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(capptr::Chunk<void> base, size_t length)
{
range_to_pow_2_blocks<MIN_BITS>(
base, length, [this](capptr::Chunk<void> base, size_t align, bool) {
capptr::Chunk<void> overflow =
buddy_small.add_block(base.as_reinterpret<FreeChunk>(), align)
.template as_reinterpret<void>();
if (overflow != nullptr)
parent->dealloc_range(overflow, bits::one_at_bit(MIN_CHUNK_BITS));
});
}
capptr::Chunk<void> refill(size_t size)
{
auto refill = parent->alloc_range(MIN_CHUNK_SIZE);
if (refill != nullptr)
add_range(pointer_offset(refill, size), MIN_CHUNK_SIZE - size);
return refill;
}
public:
class State
{
SmallBuddyRange buddy_range;
public:
SmallBuddyRange* operator->()
{
return &buddy_range;
}
constexpr State() = default;
};
static constexpr bool Aligned = true;
static_assert(ParentRange::Aligned, "ParentRange must be aligned");
constexpr SmallBuddyRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
if (size >= MIN_CHUNK_SIZE)
{
return parent->alloc_range(size);
}
auto result = buddy_small.remove_block(size);
if (result != nullptr)
{
result->left = nullptr;
result->right = nullptr;
return result.template as_reinterpret<void>();
}
return refill(size);
}
capptr::Chunk<void> alloc_range_with_leftover(size_t size)
{
SNMALLOC_ASSERT(size <= MIN_CHUNK_SIZE);
auto rsize = bits::next_pow2(size);
auto result = alloc_range(rsize);
if (result == nullptr)
return nullptr;
auto remnant = pointer_offset(result, size);
add_range(remnant, rsize - size);
return result.template as_reinterpret<void>();
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
if (size >= MIN_CHUNK_SIZE)
{
parent->dealloc_range(base, size);
return;
}
add_range(base, size);
}
};
} // namespace snmalloc

68
src/backend/statsrange.h Normal file
View File

@@ -0,0 +1,68 @@
#pragma once
#include <atomic>
namespace snmalloc
{
/**
* Used to measure memory usage.
*/
template<typename ParentRange>
class StatsRange
{
typename ParentRange::State parent{};
static inline std::atomic<size_t> current_usage{};
static inline std::atomic<size_t> peak_usage{};
public:
class State
{
StatsRange stats_range{};
public:
constexpr StatsRange* operator->()
{
return &stats_range;
}
constexpr State() = default;
};
static constexpr bool Aligned = ParentRange::Aligned;
constexpr StatsRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
auto result = parent->alloc_range(size);
if (result != nullptr)
{
auto prev = current_usage.fetch_add(size);
auto curr = peak_usage.load();
while (curr < prev + size)
{
if (peak_usage.compare_exchange_weak(curr, prev + size))
break;
}
}
return result;
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
current_usage -= size;
parent->dealloc_range(base, size);
}
size_t get_current_usage()
{
return current_usage.load();
}
size_t get_peak_usage()
{
return peak_usage.load();
}
};
} // namespace snmalloc

55
src/backend/subrange.h Normal file
View File

@@ -0,0 +1,55 @@
#pragma once
#include "../mem/entropy.h"
namespace snmalloc
{
/**
* Creates an area inside a large allocation that is larger by
* 2^RATIO_BITS. Will not return a the block at the start or
* the end of the large allocation.
*/
template<typename ParentRange, typename PAL, size_t RATIO_BITS>
class SubRange
{
typename ParentRange::State parent{};
public:
class State
{
SubRange sub_range{};
public:
constexpr State() = default;
SubRange* operator->()
{
return &sub_range;
}
};
constexpr SubRange() = default;
static constexpr bool Aligned = ParentRange::Aligned;
capptr::Chunk<void> alloc_range(size_t sub_size)
{
SNMALLOC_ASSERT(bits::is_pow2(sub_size));
auto full_size = sub_size << RATIO_BITS;
auto overblock = parent->alloc_range(full_size);
if (overblock == nullptr)
return nullptr;
size_t offset_mask = full_size - sub_size;
// Don't use first or last block in the larger reservation
// Loop required to get uniform distribution.
size_t offset;
do
{
offset = get_entropy64<PAL>() & offset_mask;
} while ((offset == 0) || (offset == offset_mask));
return pointer_offset(overblock, offset);
}
};
} // namespace snmalloc

6
src/ds/address.h
View File

@@ -72,13 +72,17 @@ namespace snmalloc
* as per above, and uses the wrapper types in its own definition, e.g., of
* capptr_bound.
*/
template<typename T, SNMALLOC_CONCEPT(capptr::ConceptBound) bounds>
inline SNMALLOC_FAST_PATH address_t address_cast(CapPtr<T, bounds> a)
{
return address_cast(a.unsafe_ptr());
}
inline SNMALLOC_FAST_PATH address_t address_cast(uintptr_t a)
{
return static_cast<address_t>(a);
}
/**
* Test if a pointer is aligned to a given size, which must be a power of
* two.

188
src/ds/redblacktree.h
View File

@@ -1,44 +1,15 @@
#pragma once
#include "../pal/pal.h"
#include "concept.h"
#include "defines.h"
#include <array>
#include <cstddef>
#include <cstdint>
#include <string>
namespace snmalloc
{
template<bool TRACE>
class debug_out
{
public:
template<bool new_line = true, typename A, typename... Args>
static void msg(A a, Args... args)
{
if constexpr (TRACE)
{
#ifdef SNMALLOC_TRACING
std::cout << a;
#else
UNUSED(a);
#endif
msg<new_line>(args...);
}
}
template<bool new_line = true>
static void msg()
{
if constexpr (TRACE && new_line)
{
#ifdef SNMALLOC_TRACING
std::cout << std::endl;
#endif
}
}
};
#ifdef __cpp_concepts
template<typename Rep>
concept RBRepTypes = requires()
@@ -118,20 +89,20 @@ namespace snmalloc
return Rep::get(ptr);
}
K operator=(K t)
ChildRef& operator=(const K& t)
{
// Use representations assigment, so we update the correct bits
// color and other things way also be stored in the Holder.
Rep::set(ptr, t);
return t;
return *this;
}
bool operator==(ChildRef& t)
bool operator==(const ChildRef t) const
{
return ptr == t.ptr;
}
bool operator!=(ChildRef& t)
bool operator!=(const ChildRef t) const
{
return ptr != t.ptr;
}
@@ -140,6 +111,11 @@ namespace snmalloc
{
return ptr;
}
bool is_null()
{
return Rep::get(ptr) == Rep::null;
}
};
// Root field of the tree
@@ -171,60 +147,47 @@ namespace snmalloc
UNUSED(curr, lower, upper);
return 0;
}
if (curr == Rep::null)
return 1;
if (
((lower != Rep::null) && curr < lower) ||
((upper != Rep::null) && curr > upper))
{
if constexpr (TRACE)
{
debug_out<true>::msg(
"Invariant failed: ",
curr,
" is out of bounds ",
lower,
", ",
upper);
print();
}
snmalloc::error("Invariant failed");
}
if (
Rep::is_red(curr) &&
(Rep::is_red(get_dir(true, curr)) || Rep::is_red(get_dir(false, curr))))
{
if constexpr (TRACE)
{
debug_out<true>::msg(
"Red invariant failed: ", curr, " is red and has red children");
print();
}
snmalloc::error("Invariant failed");
}
int left_inv = invariant(get_dir(true, curr), lower, curr);
int right_inv = invariant(get_dir(false, curr), curr, upper);
if (left_inv != right_inv)
{
if constexpr (TRACE)
{
debug_out<true>::msg(
"Balance failed: ",
curr,
" has different black depths on left and right");
print();
}
snmalloc::error("Invariant failed");
}
if (Rep::is_red(curr))
return left_inv;
else
{
if (curr == Rep::null)
return 1;
if (
((lower != Rep::null) && Rep::compare(lower, curr)) ||
((upper != Rep::null) && Rep::compare(curr, upper)))
{
report_fatal_error(
"Invariant failed: {} is out of bounds {}..{}",
Rep::printable(curr),
Rep::printable(lower),
Rep::printable(upper));
}
if (
Rep::is_red(curr) &&
(Rep::is_red(get_dir(true, curr)) ||
Rep::is_red(get_dir(false, curr))))
{
report_fatal_error(
"Invariant failed: {} is red and has red child",
Rep::printable(curr));
}
int left_inv = invariant(get_dir(true, curr), lower, curr);
int right_inv = invariant(get_dir(false, curr), curr, upper);
if (left_inv != right_inv)
{
report_fatal_error(
"Invariant failed: {} has different black depths",
Rep::printable(curr));
}
if (Rep::is_red(curr))
return left_inv;
return left_inv + 1;
}
}
struct RBStep
@@ -293,7 +256,7 @@ namespace snmalloc
bool move(bool direction)
{
auto next = get_dir(direction, curr());
if (next == Rep::null)
if (next.is_null())
return false;
path[length] = {next, direction};
length++;
@@ -308,7 +271,7 @@ namespace snmalloc
auto next = get_dir(direction, curr());
path[length] = {next, direction};
length++;
return next != Rep::null;
return !(next.is_null());
}
// Remove top element from the path.
@@ -354,16 +317,12 @@ namespace snmalloc
{
for (size_t i = 0; i < length; i++)
{
debug_out<true>::msg<false>(
"->",
message<1024>(
" -> {} @ {} ({})",
K(path[i].node),
"@",
path[i].node.addr(),
" (",
path[i].dir,
") ");
path[i].dir);
}
debug_out<true>::msg<true>();
}
}
};
@@ -378,8 +337,8 @@ namespace snmalloc
{
if constexpr (TRACE)
{
debug_out<true>::msg("-------");
debug_out<true>::msg(msg);
message<100>("-------");
message<1024>(msg);
path.print();
print(base);
}
@@ -390,7 +349,7 @@ namespace snmalloc
}
public:
RBTree() {}
constexpr RBTree() = default;
void print()
{
@@ -401,9 +360,9 @@ namespace snmalloc
{
if constexpr (TRACE)
{
if (curr == Rep::null)
if (curr.is_null())
{
debug_out<true>::msg(indent, "\\_", "null");
message<1024>("{}\\_null", indent);
return;
}
@@ -415,17 +374,14 @@ namespace snmalloc
auto reset = "\e[0m";
#endif
debug_out<true>::msg(
message<1024>(
"{}\\_{}{}{}@{} ({})",
indent,
"\\_",
colour,
curr,
reset,
"@",
curr.addr(),
" (",
depth,
")");
depth);
if ((get_dir(true, curr) != 0) || (get_dir(false, curr) != 0))
{
auto s_indent = std::string(indent);
@@ -439,14 +395,14 @@ namespace snmalloc
{
bool dir;
if (path.curr() == Rep::null)
if (path.curr().is_null())
return false;
do
{
if (path.curr() == value)
if (Rep::equal(path.curr(), value))
return true;
dir = path.curr() > value;
dir = Rep::compare(path.curr(), value);
} while (path.move_inc_null(dir));
return false;
@@ -455,7 +411,7 @@ namespace snmalloc
bool remove_path(RBPath& path)
{
ChildRef splice = path.curr();
SNMALLOC_ASSERT(splice != Rep::null);
SNMALLOC_ASSERT(!(splice.is_null()));
debug_log("Removing", path);
@@ -494,6 +450,8 @@ namespace snmalloc
debug_log("Splice done", path);
// TODO: Clear node contents?
// Red leaf removal requires no rebalancing.
if (leaf_red)
return true;
@@ -618,7 +576,7 @@ namespace snmalloc
// Insert an element at the given path.
void insert_path(RBPath path, K value)
{
SNMALLOC_ASSERT(path.curr() == Rep::null);
SNMALLOC_ASSERT(path.curr().is_null());
path.curr() = value;
get_dir(true, path.curr()) = Rep::null;
get_dir(false, path.curr()) = Rep::null;
@@ -706,7 +664,7 @@ namespace snmalloc
K remove_min()
{
if (get_root() == Rep::null)
if (get_root().is_null())
return Rep::null;
auto path = get_root_path();
@@ -722,7 +680,7 @@ namespace snmalloc
bool remove_elem(K value)
{
if (get_root() == Rep::null)
if (get_root().is_null())
return false;
auto path = get_root_path();
@@ -749,4 +707,4 @@ namespace snmalloc
return RBPath(root);
}
};
}
} // namespace snmalloc

44
src/mem/corealloc.h
View File

@@ -1,8 +1,8 @@
#pragma once
#include "../backend/chunkallocator.h"
#include "../ds/defines.h"
#include "allocconfig.h"
#include "chunkallocator.h"
#include "localcache.h"
#include "metaslab.h"
#include "pool.h"
@@ -58,11 +58,6 @@ namespace snmalloc
*/
MetaslabCache alloc_classes[NUM_SMALL_SIZECLASSES];
/**
* Local cache for the Chunk allocator.
*/
ChunkAllocatorLocalState chunk_local_state;
/**
* Local entropy source and current version of keys for
* this thread
@@ -392,11 +387,11 @@ namespace snmalloc
// TODO delay the clear to the next user of the slab, or teardown so
// don't touch the cache lines at this point in snmalloc_check_client.
auto chunk_record = clear_slab(meta, sizeclass);
ChunkAllocator::dealloc<SharedStateHandle>(
SharedStateHandle::dealloc_chunk(
get_backend_local_state(),
chunk_local_state,
chunk_record,
sizeclass_to_slab_sizeclass(sizeclass));
sizeclass_to_slab_size(sizeclass));
return true;
});
@@ -419,23 +414,17 @@ namespace snmalloc
// Handle large deallocation here.
size_t entry_sizeclass = entry.get_sizeclass().as_large();
size_t size = bits::one_at_bit(entry_sizeclass);
size_t slab_sizeclass =
metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass);
#ifdef SNMALLOC_TRACING
std::cout << "Large deallocation: " << size
<< " chunk sizeclass: " << slab_sizeclass << std::endl;
std::cout << "Large deallocation: " << size << std::endl;
#else
UNUSED(size);
#endif
auto slab_record = reinterpret_cast<ChunkRecord*>(meta);
ChunkAllocator::dealloc<SharedStateHandle>(
get_backend_local_state(),
chunk_local_state,
slab_record,
slab_sizeclass);
SharedStateHandle::dealloc_chunk(
get_backend_local_state(), slab_record, size);
return;
}
@@ -594,9 +583,6 @@ namespace snmalloc
message_queue().invariant();
}
ChunkAllocator::register_local_state<SharedStateHandle>(
get_backend_local_state(), chunk_local_state);
if constexpr (DEBUG)
{
for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++)
@@ -686,7 +672,7 @@ namespace snmalloc
SNMALLOC_FAST_PATH void
dealloc_local_object(CapPtr<void, capptr::bounds::Alloc> p)
{
auto entry =
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p));
if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy)))
return;
@@ -791,21 +777,17 @@ namespace snmalloc
// No existing free list get a new slab.
size_t slab_size = sizeclass_to_slab_size(sizeclass);
size_t slab_sizeclass = sizeclass_to_slab_sizeclass(sizeclass);
#ifdef SNMALLOC_TRACING
std::cout << "rsize " << rsize << std::endl;
std::cout << "slab size " << slab_size << std::endl;
#endif
auto [slab, meta] =
snmalloc::ChunkAllocator::alloc_chunk<SharedStateHandle>(
get_backend_local_state(),
chunk_local_state,
sizeclass_t::from_small_class(sizeclass),
slab_sizeclass,
slab_size,
public_state());
auto [slab, meta] = SharedStateHandle::alloc_chunk(
get_backend_local_state(),
slab_size,
public_state(),
sizeclass_t::from_small_class(sizeclass));
if (slab == nullptr)
{

18
src/mem/localalloc.h
View File

@@ -180,13 +180,11 @@ namespace snmalloc
return check_init([&](CoreAlloc* core_alloc) {
// Grab slab of correct size
// Set remote as large allocator remote.
auto [chunk, meta] = ChunkAllocator::alloc_chunk<SharedStateHandle>(
auto [chunk, meta] = SharedStateHandle::alloc_chunk(
core_alloc->get_backend_local_state(),
core_alloc->chunk_local_state,
size_to_sizeclass_full(size),
large_size_to_chunk_sizeclass(size),
large_size_to_chunk_size(size),
core_alloc->public_state());
core_alloc->public_state(),
size_to_sizeclass_full(size));
// set up meta data so sizeclass is correct, and hence alloc size, and
// external pointer.
#ifdef SNMALLOC_TRACING
@@ -201,7 +199,7 @@ namespace snmalloc
if (zero_mem == YesZero && chunk.unsafe_ptr() != nullptr)
{
SharedStateHandle::Pal::template zero<false>(
chunk.unsafe_ptr(), size);
chunk.unsafe_ptr(), bits::next_pow2(size));
}
return capptr_chunk_is_alloc(capptr_to_user_address_control(chunk));
@@ -268,7 +266,7 @@ namespace snmalloc
std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size "
<< alloc_size(p.unsafe_ptr()) << std::endl;
#endif
MetaEntry entry =
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(p));
local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
entry.get_remote()->trunc_id(), p, key_global);
@@ -716,7 +714,7 @@ namespace snmalloc
// To handle this case we require the uninitialised pagemap contain an
// entry for the first chunk of memory, that states it represents a
// large object, so we can pull the check for null off the fast path.
MetaEntry entry =
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw));
return sizeclass_full_to_size(entry.get_sizeclass());
@@ -761,7 +759,7 @@ namespace snmalloc
size_t remaining_bytes(const void* p)
{
#ifndef SNMALLOC_PASS_THROUGH
MetaEntry entry =
const MetaEntry& entry =
SharedStateHandle::Pagemap::template get_metaentry<true>(
address_cast(p));
@@ -790,7 +788,7 @@ namespace snmalloc
size_t index_in_object(const void* p)
{
#ifndef SNMALLOC_PASS_THROUGH
MetaEntry entry =
const MetaEntry& entry =
SharedStateHandle::Pagemap::template get_metaentry<true>(
address_cast(p));

71
src/mem/metaslab.h
View File

@@ -227,7 +227,26 @@ namespace snmalloc
*/
class MetaEntry
{
Metaslab* meta{nullptr}; // may also be ChunkRecord*
template<typename Pagemap>
friend class BuddyChunkRep;
/**
* The pointer to the metaslab, the bottom bit is used to indicate if this
* is the first chunk in a PAL allocation, that cannot be combined with
* the preceeding chunk.
*/
uintptr_t meta{0};
/**
* Bit used to indicate this should not be considered part of the previous
* PAL allocation.
*
* Some platforms cannot treat different PalAllocs as a single allocation.
* This is true on CHERI as the combined permission might not be
* representable. It is also true on Windows as you cannot Commit across
* multiple continuous VirtualAllocs.
*/
static constexpr address_t BOUNDARY_BIT = 1;
/**
* A bit-packed pointer to the owning allocator (if any), and the sizeclass
@@ -253,7 +272,8 @@ namespace snmalloc
*/
SNMALLOC_FAST_PATH
MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass)
: meta(meta), remote_and_sizeclass(remote_and_sizeclass)
: meta(reinterpret_cast<uintptr_t>(meta)),
remote_and_sizeclass(remote_and_sizeclass)
{}
SNMALLOC_FAST_PATH
@@ -261,23 +281,13 @@ namespace snmalloc
Metaslab* meta,
RemoteAllocator* remote,
sizeclass_t sizeclass = sizeclass_t())
: meta(meta)
: meta(reinterpret_cast<uintptr_t>(meta))
{
/* remote might be nullptr; cast to uintptr_t before offsetting */
remote_and_sizeclass =
pointer_offset(reinterpret_cast<uintptr_t>(remote), sizeclass.raw());
}
/**
* Return the Metaslab field as a void*, guarded by an assert that there is
* no remote that owns this chunk.
*/
[[nodiscard]] SNMALLOC_FAST_PATH void* get_metaslab_no_remote() const
{
SNMALLOC_ASSERT(get_remote() == nullptr);
return static_cast<void*>(meta);
}
/**
* Return the Metaslab metadata associated with this chunk, guarded by an
* assert that this chunk is being used as a slab (i.e., has an associated
@@ -286,7 +296,7 @@ namespace snmalloc
[[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const
{
SNMALLOC_ASSERT(get_remote() != nullptr);
return meta;
return reinterpret_cast<Metaslab*>(meta & ~BOUNDARY_BIT);
}
/**
@@ -295,7 +305,7 @@ namespace snmalloc
* only safe use for this is to pass it to the two-argument constructor of
* this class.
*/
[[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass()
[[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const
{
return remote_and_sizeclass;
}
@@ -303,7 +313,8 @@ namespace snmalloc
[[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const
{
return reinterpret_cast<RemoteAllocator*>(
pointer_align_down<alignof(RemoteAllocator)>(remote_and_sizeclass));
pointer_align_down<REMOTE_WITH_BACKEND_MARKER_ALIGN>(
remote_and_sizeclass));
}
[[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const
@@ -312,7 +323,32 @@ namespace snmalloc
// https://github.com/CTSRD-CHERI/llvm-project/issues/588
return sizeclass_t::from_raw(
static_cast<size_t>(remote_and_sizeclass) &
(alignof(RemoteAllocator) - 1));
(REMOTE_WITH_BACKEND_MARKER_ALIGN - 1));
}
MetaEntry(const MetaEntry&) = delete;
MetaEntry& operator=(const MetaEntry& other)
{
// Don't overwrite the boundary bit with the other's
meta = (other.meta & ~BOUNDARY_BIT) | address_cast(meta & BOUNDARY_BIT);
remote_and_sizeclass = other.remote_and_sizeclass;
return *this;
}
void set_boundary()
{
meta |= BOUNDARY_BIT;
}
[[nodiscard]] bool is_boundary() const
{
return meta & BOUNDARY_BIT;
}
bool clear_boundary_bit()
{
return meta &= ~BOUNDARY_BIT;
}
};
@@ -328,5 +364,4 @@ namespace snmalloc
uint16_t unused = 0;
uint16_t length = 0;
};
} // namespace snmalloc

2
src/mem/pool.h
View File

@@ -1,9 +1,9 @@
#pragma once
#include "../backend/chunkallocator.h"
#include "../ds/flaglock.h"
#include "../ds/mpmcstack.h"
#include "../pal/pal_concept.h"
#include "chunkallocator.h"
#include "pooled.h"
namespace snmalloc

15
src/mem/remoteallocator.h
View File

@@ -2,7 +2,6 @@
#include "../mem/allocconfig.h"
#include "../mem/freelist.h"
#include "../mem/metaslab.h"
#include "../mem/sizeclasstable.h"
#include <array>
@@ -11,9 +10,19 @@
namespace snmalloc
{
// Remotes need to be aligned enough that the bottom bits have enough room for
// all the size classes, both large and small.
// all the size classes, both large and small. An additional bit is required
// to separate backend uses.
static constexpr size_t REMOTE_MIN_ALIGN =
bits::max<size_t>(CACHELINE_SIZE, SIZECLASS_REP_SIZE);
bits::max<size_t>(CACHELINE_SIZE, SIZECLASS_REP_SIZE) << 1;
// This bit is set on the RemoteAllocator* to indicate it is
// actually being used by the backend for some other use.
static constexpr size_t BACKEND_MARKER = REMOTE_MIN_ALIGN >> 1;
// The bit above the sizeclass is always zero unless this is used
// by the backend to represent another datastructure such as the buddy
// allocator entries.
constexpr size_t REMOTE_WITH_BACKEND_MARKER_ALIGN = BACKEND_MARKER;
/**
* Global key for all remote lists.

15
src/mem/remotecache.h
View File

@@ -101,18 +101,25 @@ namespace snmalloc
if (!list[i].empty())
{
auto [first, last] = list[i].extract_segment(key);
MetaEntry entry =
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(first));
auto remote = entry.get_remote();
// If the allocator is not correctly aligned, then the bit that is
// set implies this is used by the backend, and we should not be
// deallocating memory here.
snmalloc_check_client(
(address_cast(remote) & BACKEND_MARKER) == 0,
"Delayed detection of attempt to free internal structure.");
if constexpr (SharedStateHandle::Options.QueueHeadsAreTame)
{
auto domesticate_nop = [](freelist::QueuePtr p) {
return freelist::HeadPtr(p.unsafe_ptr());
};
entry.get_remote()->enqueue(first, last, key, domesticate_nop);
remote->enqueue(first, last, key, domesticate_nop);
}
else
{
entry.get_remote()->enqueue(first, last, key, domesticate);
remote->enqueue(first, last, key, domesticate);
}
sent_something = true;
}
@@ -134,7 +141,7 @@ namespace snmalloc
// Use the next N bits to spread out remote deallocs in our own
// slot.
auto r = resend.take(key, domesticate);
MetaEntry entry =
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(r));
auto i = entry.get_remote()->trunc_id();
size_t slot = get_slot<allocator_size>(i, post_round);

6
src/override/malloc-extensions.cc
View File

@@ -6,8 +6,8 @@ using namespace snmalloc;
void get_malloc_info_v1(malloc_info_v1* stats)
{
auto unused_chunks = Globals::get_chunk_allocator_state().unused_memory();
auto peak = Globals::get_chunk_allocator_state().peak_memory_usage();
stats->current_memory_usage = peak - unused_chunks;
auto curr = Globals::get_current_usage();
auto peak = Globals::get_peak_usage();
stats->current_memory_usage = curr;
stats->peak_memory_usage = peak;
}

6
src/override/rust.cc
View File

@@ -48,8 +48,6 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_realloc)(
extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(rust_statistics)(
size_t* current_memory_usage, size_t* peak_memory_usage)
{
auto unused_chunks = Globals::get_chunk_allocator_state().unused_memory();
auto peak = Globals::get_chunk_allocator_state().peak_memory_usage();
*current_memory_usage = peak - unused_chunks;
*peak_memory_usage = peak;
*current_memory_usage = Globals::get_current_usage();
*peak_memory_usage = Globals::get_peak_usage();
}

6
src/pal/pal.h
View File

@@ -168,4 +168,10 @@ namespace snmalloc
Pal::error(msg.get_message());
}
template<size_t BufferSize, typename... Args>
inline void message(Args... args)
{
MessageBuilder<BufferSize> msg{std::forward<Args>(args)...};
Pal::message(msg.get_message());
}
} // namespace snmalloc

3
src/pal/pal_windows.h
View File

@@ -150,7 +150,8 @@ namespace snmalloc
void* r = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE);
if (r == nullptr)
error("out of memory");
report_fatal_error(
"out of memory: {} ({}) could not be committed", p, size);
}
/// OS specific function for zeroing memory

1
src/snmalloc_core.h
View File

@@ -1,6 +1,5 @@
#pragma once
#include "backend/address_space.h"
#include "backend/commonconfig.h"
#include "backend/pagemap.h"
#include "mem/globalalloc.h"

5
src/test/func/malloc/malloc.cc
View File

@@ -135,10 +135,11 @@ void test_realloc(void* p, size_t size, int err, bool null)
START_TEST("realloc({}({}), {})", p, old_size, size);
errno = SUCCESS;
auto new_p = our_realloc(p, size);
// Realloc failure case, deallocate original block
check_result(size, 1, new_p, err, null);
// Realloc failure case, deallocate original block as not
// handled by check_result.
if (new_p == nullptr && size != 0)
our_free(p);
check_result(size, 1, new_p, err, null);
}
void test_posix_memalign(size_t size, size_t align, int err, bool null)

15
src/test/func/redblack/redblack.cc
View File

@@ -73,6 +73,21 @@ public:
if (new_is_red != is_red(k))
array[k].left.value ^= 1;
}
static bool compare(key k1, key k2)
{
return k1 > k2;
}
static bool equal(key k1, key k2)
{
return k1 == k2;
}
static size_t printable(key k)
{
return k;
}
};
template<bool TRACE>