Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make address space manager use pagemap for next pointers (#356)

Browse Source 
* Make address space manager use pagemap for next pointers

The address space manager uses the pagemap entry to form linked lists of
unused address space above MIN_CHUNK_SIZE.  It continues to use
references in the block below that threshold.

In the CHECK_CLIENT mode this makes it hard to corrupt the ASM as only
meta-data uses allocations below MIN_CHUNK_SIZE from the ASM. This
allocations will be protected with guard pages by the backend.

* address_space_core: use FreeChunk struct

Purely stylistic, NFCI.  This hides some somewhat gnarly reinterpret_cast-s in
favor of more, but hopefully less gnarly, casts elsewhere.

* Apply suggestions from code review

Co-authored-by: Nathaniel Wesley Filardo <nfilardo@microsoft.com>
This commit is contained in:
Matthew Parkinson
2021-07-23 15:09:30 +01:00
committed by GitHub
parent 84a5fb9450
commit b69505fc5d
3 changed files with 133 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/backend/address_space.h
View File

@@ -71,7 +71,7 @@ namespace snmalloc
CapPtr<void, CBChunk> res;
{
FlagLock lock(spin_lock);
res = core.template reserve<PAL>(size);
res = core.template reserve<PAL>(size, pagemap);
if (res == nullptr)
{
// Allocation failed ask OS for more memory
@@ -130,10 +130,10 @@ namespace snmalloc
pagemap.register_range(address_cast(block), block_size);
core.template add_range<PAL>(block, block_size);
core.template add_range<PAL>(block, block_size, pagemap);
// still holding lock so guaranteed to succeed.
res = core.template reserve<PAL>(size);
res = core.template reserve<PAL>(size, pagemap);
}
}
@@ -167,7 +167,8 @@ namespace snmalloc
if (rsize > size)
{
FlagLock lock(spin_lock);
core.template add_range<PAL>(pointer_offset(res, size), rsize - size);
core.template add_range<PAL>(
pointer_offset(res, size), rsize - size, pagemap);
}
if constexpr (committed)
@@ -187,10 +188,11 @@ namespace snmalloc
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(CapPtr<void, CBChunk> base, size_t length)
template<typename Pagemap>
void add_range(CapPtr<void, CBChunk> base, size_t length, Pagemap& pagemap)
{
FlagLock lock(spin_lock);
core.add_range<PAL>(base, length);
core.add_range<PAL>(base, length, pagemap);
}
};
} // namespace snmalloc

192
src/backend/address_space_core.h
View File

@@ -11,9 +11,6 @@
namespace snmalloc
{
/**
* TODO all comment in this file need revisiting. Core versus locking global
* version.
*
* Implements a power of two allocator, where all blocks are aligned to the
* same power of two as their size. This is what snmalloc uses to get
* alignment of very large sizeclasses.
@@ -23,33 +20,36 @@ namespace snmalloc
*/
class AddressSpaceManagerCore
{
struct FreeChunk
{
CapPtr<FreeChunk, CBChunk> next;
};
/**
* Stores the blocks of address space
*
* The first level of array indexes based on power of two size.
* The array indexes based on power of two size.
*
* The first entry ranges[n][0] is just a pointer to an address range
* of size 2^n.
*
* The second entry ranges[n][1] is a pointer to a linked list of blocks
* of this size. The final block in the list is not committed, so we commit
* on pop for this corner case.
*
* Invariants
* ranges[n][1] != nullptr => ranges[n][0] != nullptr
* The entries for each size form a linked list. For sizes below
* MIN_CHUNK_SIZE they are linked through the first location in the
* block of memory. For sizes of, and above, MIN_CHUNK_SIZE they are
* linked using the pagemap. We only use the smaller than MIN_CHUNK_SIZE
* allocations for meta-data, so we can be sure that the next pointers
* never occur in a blocks that are ultimately used for object allocations.
*
* bits::BITS is used for simplicity, we do not use below the pointer size,
* and large entries will be unlikely to be supported by the platform.
*/
std::array<std::array<CapPtr<void, CBChunk>, 2>, bits::BITS> ranges = {};
std::array<CapPtr<FreeChunk, CBChunk>, bits::BITS> ranges = {};
/**
* Checks a block satisfies its invariant.
*/
inline void check_block(CapPtr<void, CBChunk> base, size_t align_bits)
inline void check_block(CapPtr<FreeChunk, CBChunk> base, size_t align_bits)
{
SNMALLOC_ASSERT(
base == pointer_align_up(base, bits::one_at_bit(align_bits)));
address_cast(base) ==
bits::align_up(address_cast(base), bits::one_at_bit(align_bits)));
// All blocks need to be bigger than a pointer.
SNMALLOC_ASSERT(bits::one_at_bit(align_bits) >= sizeof(void*));
UNUSED(base);
@@ -57,45 +57,83 @@ namespace snmalloc
}
/**
* Adds a block to `ranges`.
* Set next pointer for a power of two address range.
*
* This abstracts the use of either
* - the pagemap; or
* - the first pointer word of the block
* to store the next pointer for the list of unused address space of a
* particular size.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
void add_block(size_t align_bits, CapPtr<void, CBChunk> base)
template<typename Pagemap>
void set_next(
size_t align_bits,
CapPtr<FreeChunk, CBChunk> base,
CapPtr<FreeChunk, CBChunk> next,
Pagemap& pagemap)
{
check_block(base, align_bits);
SNMALLOC_ASSERT(align_bits < 64);
if (ranges[align_bits][0] == nullptr)
if (align_bits >= MIN_CHUNK_BITS)
{
// Prefer first slot if available.
ranges[align_bits][0] = base;
// The pagemap stores MetaEntrys, abuse the metaslab field to be the
// next block in the stack of blocks.
//
// The pagemap entries here have nullptr (i.e., fake_large_remote) as
// their remote, and so other accesses to the pagemap (by
// external_pointer, for example) will not attempt to follow this
// "Metaslab" pointer.
MetaEntry t(reinterpret_cast<Metaslab*>(next.unsafe_ptr()), nullptr, 0);
pagemap.set(address_cast(base), t);
return;
}
if (ranges[align_bits][1] != nullptr)
base->next = next;
}
/**
* Get next pointer for a power of two address range.
*
* This abstracts the use of either
* - the pagemap; or
* - the first pointer word of the block
* to store the next pointer for the list of unused address space of a
* particular size.
*/
template<typename Pagemap>
CapPtr<FreeChunk, CBChunk> get_next(
size_t align_bits, CapPtr<FreeChunk, CBChunk> base, Pagemap& pagemap)
{
if (align_bits >= MIN_CHUNK_BITS)
{
#ifdef SNMALLOC_TRACING
std::cout << "Add range linking." << std::endl;
#endif
// Add to linked list.
commit_block<PAL>(base, sizeof(void*));
*(base.template as_static<CapPtr<void, CBChunk>>().unsafe_ptr()) =
ranges[align_bits][1];
check_block(ranges[align_bits][1], align_bits);
const MetaEntry& t = pagemap.template get<false>(address_cast(base));
return CapPtr<FreeChunk, CBChunk>(
reinterpret_cast<FreeChunk*>(t.get_metaslab()));
}
// Update head of list
ranges[align_bits][1] = base;
check_block(ranges[align_bits][1], align_bits);
return base->next;
}
/**
* Adds a block to `ranges`.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL, typename Pagemap>
void add_block(
size_t align_bits, CapPtr<FreeChunk, CBChunk> base, Pagemap& pagemap)
{
check_block(base, align_bits);
SNMALLOC_ASSERT(align_bits < 64);
set_next(align_bits, base, ranges[align_bits], pagemap);
ranges[align_bits] = base.as_static<FreeChunk>();
}
/**
* Find a block of the correct size. May split larger blocks
* to satisfy this request.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
CapPtr<void, CBChunk> remove_block(size_t align_bits)
template<SNMALLOC_CONCEPT(ConceptPAL) PAL, typename Pagemap>
CapPtr<void, CBChunk> remove_block(size_t align_bits, Pagemap& pagemap)
{
CapPtr<void, CBChunk> first = ranges[align_bits][0];
CapPtr<FreeChunk, CBChunk> first = ranges[align_bits];
if (first == nullptr)
{
if (align_bits == (bits::BITS - 1))
@@ -105,37 +143,34 @@ namespace snmalloc
}
// Look for larger block and split up recursively
CapPtr<void, CBChunk> bigger = remove_block<PAL>(align_bits + 1);
CapPtr<void, CBChunk> bigger =
remove_block<PAL>(align_bits + 1, pagemap);
if (bigger != nullptr)
{
// This block is going to be broken up into sub CHUNK_SIZE blocks
// so we need to commit it to enable the next pointers to be used
// inside the block.
if ((align_bits + 1) == MIN_CHUNK_BITS)
{
commit_block<PAL>(bigger, MIN_CHUNK_SIZE);
}
size_t left_over_size = bits::one_at_bit(align_bits);
auto left_over = pointer_offset(bigger, left_over_size);
ranges[align_bits][0] =
Aal::capptr_bound<void, CBChunk>(left_over, left_over_size);
check_block(left_over, align_bits);
add_block<PAL>(
align_bits,
Aal::capptr_bound<FreeChunk, CBChunk>(left_over, left_over_size),
pagemap);
check_block(left_over.as_static<FreeChunk>(), align_bits);
}
check_block(bigger, align_bits + 1);
check_block(bigger.as_static<FreeChunk>(), align_bits + 1);
return bigger;
}
CapPtr<void, CBChunk> second = ranges[align_bits][1];
if (second != nullptr)
{
commit_block<PAL>(second, sizeof(void*));
auto psecond =
second.template as_static<CapPtr<void, CBChunk>>().unsafe_ptr();
auto next = *psecond;
ranges[align_bits][1] = next;
// Zero memory. Client assumes memory contains only zeros.
*psecond = nullptr;
check_block(second, align_bits);
check_block(next, align_bits);
return second;
}
check_block(first, align_bits);
ranges[align_bits][0] = nullptr;
return first;
ranges[align_bits] = get_next(align_bits, first, pagemap);
return first.as_void();
}
public:
@@ -143,9 +178,21 @@ namespace snmalloc
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
void add_range(CapPtr<void, CBChunk> base, size_t length)
template<SNMALLOC_CONCEPT(ConceptPAL) PAL, typename Pagemap>
void add_range(CapPtr<void, CBChunk> base, size_t length, Pagemap& pagemap)
{
// For start and end that are not chunk sized, we need to
// commit the pages to track the allocations.
auto base_chunk = pointer_align_up(base, MIN_CHUNK_SIZE);
auto end = pointer_offset(base, length);
auto end_chunk = pointer_align_down(end, MIN_CHUNK_SIZE);
auto start_length = pointer_diff(base, base_chunk);
auto end_length = pointer_diff(end_chunk, end);
if (start_length != 0)
commit_block<PAL>(base, start_length);
if (end_length != 0)
commit_block<PAL>(end_chunk, end_length);
// Find the minimum set of maximally aligned blocks in this range.
// Each block's alignment and size are equal.
while (length >= sizeof(void*))
@@ -154,9 +201,10 @@ namespace snmalloc
size_t length_align_bits = (bits::BITS - 1) - bits::clz(length);
size_t align_bits = bits::min(base_align_bits, length_align_bits);
size_t align = bits::one_at_bit(align_bits);
auto b = base.as_static<FreeChunk>();
check_block(base, align_bits);
add_block<PAL>(align_bits, base);
check_block(b, align_bits);
add_block<PAL>(align_bits, b, pagemap);
base = pointer_offset(base, align);
length -= align;
@@ -188,8 +236,8 @@ namespace snmalloc
* part of satisfying the request will be registered with the provided
* arena_map for use in subsequent amplification.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
CapPtr<void, CBChunk> reserve(size_t size)
template<SNMALLOC_CONCEPT(ConceptPAL) PAL, typename Pagemap>
CapPtr<void, CBChunk> reserve(size_t size, Pagemap& pagemap)
{
#ifdef SNMALLOC_TRACING
std::cout << "ASM Core reserve request:" << size << std::endl;
@@ -198,7 +246,7 @@ namespace snmalloc
SNMALLOC_ASSERT(bits::is_pow2(size));
SNMALLOC_ASSERT(size >= sizeof(void*));
return remove_block<PAL>(bits::next_pow2_bits(size));
return remove_block<PAL>(bits::next_pow2_bits(size), pagemap);
}
/**
@@ -208,8 +256,8 @@ namespace snmalloc
* This is useful for allowing the space required for alignment to be
* used, by smaller objects.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
CapPtr<void, CBChunk> reserve_with_left_over(size_t size)
template<SNMALLOC_CONCEPT(ConceptPAL) PAL, typename Pagemap>
CapPtr<void, CBChunk> reserve_with_left_over(size_t size, Pagemap& pagemap)
{
SNMALLOC_ASSERT(size >= sizeof(void*));
@@ -217,13 +265,13 @@ namespace snmalloc
size_t rsize = bits::next_pow2(size);
auto res = reserve<PAL>(rsize);
auto res = reserve<PAL>(rsize, pagemap);
if (res != nullptr)
{
if (rsize > size)
{
add_range<PAL>(pointer_offset(res, size), rsize - size);
add_range<PAL>(pointer_offset(res, size), rsize - size, pagemap);
}
}
return res;

9
src/backend/backend.h
View File

@@ -90,7 +90,8 @@ namespace snmalloc
fixed_range_ == fixed_range, "Don't set SFINAE parameter!");
auto [heap_base, heap_length] = pagemap.init(base, length);
address_space.add_range(CapPtr<void, CBChunk>(heap_base), heap_length);
address_space.add_range(
CapPtr<void, CBChunk>(heap_base), heap_length, pagemap);
if constexpr (!fixed_range)
{
@@ -155,7 +156,7 @@ namespace snmalloc
auto& local = local_state->local_address_space;
#endif
p = local.template reserve_with_left_over<PAL>(size);
p = local.template reserve_with_left_over<PAL>(size, h.pagemap);
if (p != nullptr)
{
return p;
@@ -174,10 +175,10 @@ namespace snmalloc
}
#endif
PAL::template notify_using<NoZero>(refill.unsafe_ptr(), refill_size);
local.template add_range<PAL>(refill, refill_size);
local.template add_range<PAL>(refill, refill_size, h.pagemap);
// This should succeed
return local.template reserve_with_left_over<PAL>(size);
return local.template reserve_with_left_over<PAL>(size, h.pagemap);
}
#ifdef SNMALLOC_CHECK_CLIENT