Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SP: LargeAlloc return CBChunk & chase consequences

Browse Source 
Even if we opt not to bound these pointers internally (if they aren't headed out
to the user program or we later derive bounded pointers), they should still be
annotated as something other than CBArena, ensuring that we do not attempt to
use them for general amplification.
This commit is contained in:
Nathaniel Filardo
2021-04-01 13:20:17 +01:00
committed by Nathaniel Wesley Filardo
parent 54fec3821f
commit f7821e11eb
10 changed files with 175 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/mem/address_space.h
View File

@@ -35,7 +35,7 @@ namespace snmalloc
* bits::BITS is used for simplicity, we do not use below the pointer size,
* and large entries will be unlikely to be supported by the platform.
*/
std::array<std::array<CapPtr<void, CBArena>, 2>, bits::BITS> ranges = {};
std::array<std::array<CapPtr<void, CBChunk>, 2>, bits::BITS> ranges = {};
/**
* This is infrequently used code, a spin lock simplifies the code
@@ -46,7 +46,7 @@ namespace snmalloc
/**
* Checks a block satisfies its invariant.
*/
inline void check_block(CapPtr<void, CBArena> base, size_t align_bits)
inline void check_block(CapPtr<void, CBChunk> base, size_t align_bits)
{
SNMALLOC_ASSERT(
base == pointer_align_up(base, bits::one_at_bit(align_bits)));
@@ -59,7 +59,7 @@ namespace snmalloc
/**
* Adds a block to `ranges`.
*/
void add_block(size_t align_bits, CapPtr<void, CBArena> base)
void add_block(size_t align_bits, CapPtr<void, CBChunk> base)
{
check_block(base, align_bits);
SNMALLOC_ASSERT(align_bits < 64);
@@ -74,7 +74,7 @@ namespace snmalloc
{
// Add to linked list.
commit_block(base, sizeof(void*));
*(base.template as_static<CapPtr<void, CBArena>>().unsafe_capptr) =
*(base.template as_static<CapPtr<void, CBChunk>>().unsafe_capptr) =
ranges[align_bits][1];
check_block(ranges[align_bits][1], align_bits);
}
@@ -88,9 +88,9 @@ namespace snmalloc
* Find a block of the correct size. May split larger blocks
* to satisfy this request.
*/
CapPtr<void, CBArena> remove_block(size_t align_bits)
CapPtr<void, CBChunk> remove_block(size_t align_bits)
{
CapPtr<void, CBArena> first = ranges[align_bits][0];
CapPtr<void, CBChunk> first = ranges[align_bits][0];
if (first == nullptr)
{
if (align_bits == (bits::BITS - 1))
@@ -100,7 +100,7 @@ namespace snmalloc
}
// Look for larger block and split up recursively
CapPtr<void, CBArena> bigger = remove_block(align_bits + 1);
CapPtr<void, CBChunk> bigger = remove_block(align_bits + 1);
if (bigger != nullptr)
{
auto left_over = pointer_offset(bigger, bits::one_at_bit(align_bits));
@@ -111,12 +111,12 @@ namespace snmalloc
return bigger;
}
CapPtr<void, CBArena> second = ranges[align_bits][1];
CapPtr<void, CBChunk> second = ranges[align_bits][1];
if (second != nullptr)
{
commit_block(second, sizeof(void*));
auto psecond =
second.template as_static<CapPtr<void, CBArena>>().unsafe_capptr;
second.template as_static<CapPtr<void, CBChunk>>().unsafe_capptr;
auto next = *psecond;
ranges[align_bits][1] = next;
// Zero memory. Client assumes memory contains only zeros.
@@ -135,7 +135,7 @@ namespace snmalloc
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(CapPtr<void, CBArena> base, size_t length)
void add_range(CapPtr<void, CBChunk> base, size_t length)
{
// Find the minimum set of maximally aligned blocks in this range.
// Each block's alignment and size are equal.
@@ -157,7 +157,7 @@ namespace snmalloc
/**
* Commit a block of memory
*/
void commit_block(CapPtr<void, CBArena> base, size_t size)
void commit_block(CapPtr<void, CBChunk> base, size_t size)
{
// Rounding required for sub-page allocations.
auto page_start = pointer_align_down<OS_PAGE_SIZE, char>(base);
@@ -180,7 +180,7 @@ namespace snmalloc
* arena_map for use in subsequent amplification.
*/
template<bool committed>
CapPtr<void, CBArena> reserve(size_t size, ArenaMap& arena_map)
CapPtr<void, CBChunk> reserve(size_t size, ArenaMap& arena_map)
{
SNMALLOC_ASSERT(bits::is_pow2(size));
SNMALLOC_ASSERT(size >= sizeof(void*));
@@ -194,18 +194,18 @@ namespace snmalloc
pal_supports<AlignedAllocation, PAL> && !aal_supports<StrictProvenance>)
{
if (size >= PAL::minimum_alloc_size)
return CapPtr<void, CBArena>(
return CapPtr<void, CBChunk>(
PAL::template reserve_aligned<committed>(size));
}
CapPtr<void, CBArena> res;
CapPtr<void, CBChunk> res;
{
FlagLock lock(spin_lock);
res = remove_block(bits::next_pow2_bits(size));
if (res == nullptr)
{
// Allocation failed ask OS for more memory
CapPtr<void, CBArena> block = nullptr;
CapPtr<void, CBChunk> block = nullptr;
size_t block_size = 0;
if constexpr (pal_supports<AlignedAllocation, PAL>)
{
@@ -233,7 +233,10 @@ namespace snmalloc
void* block_raw = PAL::template reserve_aligned<false>(block_size);
block = CapPtr<void, CBArena>(block_raw);
// It's a bit of a lie to convert without applying bounds, but the
// platform will have bounded block for us and it's better that the
// rest of our internals expect CBChunk bounds.
block = CapPtr<void, CBChunk>(block_raw);
if constexpr (aal_supports<StrictProvenance>)
{
@@ -254,7 +257,7 @@ namespace snmalloc
// the PAL, and this could lead to suprious OOM. This is
// particularly bad if the PAL gives all the memory on first call.
auto block_and_size = PAL::reserve_at_least(size * 2);
block = CapPtr<void, CBArena>(block_and_size.first);
block = CapPtr<void, CBChunk>(block_and_size.first);
block_size = block_and_size.second;
// Ensure block is pointer aligned.
@@ -294,7 +297,7 @@ namespace snmalloc
* used, by smaller objects.
*/
template<bool committed>
CapPtr<void, CBArena>
CapPtr<void, CBChunk>
reserve_with_left_over(size_t size, ArenaMap& arena_map)
{
SNMALLOC_ASSERT(size >= sizeof(void*));
@@ -330,7 +333,7 @@ namespace snmalloc
* Constructor that pre-initialises the address-space manager with a region
* of memory.
*/
AddressSpaceManager(CapPtr<void, CBArena> base, size_t length)
AddressSpaceManager(CapPtr<void, CBChunk> base, size_t length)
{
add_range(base, length);
}

99
src/mem/alloc.h
View File

@@ -638,10 +638,10 @@ namespace snmalloc
};
SlabList small_classes[NUM_SMALL_CLASSES];
DLList<Mediumslab, CapPtrCBArena> medium_classes[NUM_MEDIUM_CLASSES];
DLList<Mediumslab, CapPtrCBChunk> medium_classes[NUM_MEDIUM_CLASSES];
DLList<Superslab> super_available;
DLList<Superslab> super_only_short_available;
DLList<Superslab, CapPtrCBChunk> super_available;
DLList<Superslab, CapPtrCBChunk> super_only_short_available;
RemoteCache remote;
@@ -901,7 +901,7 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH void dealloc_not_large_local(
CapPtr<Superslab, CBArena> super,
CapPtr<Superslab, CBChunkD> super,
CapPtr<Remote, CBArena> p_auth_offseted,
sizeclass_t sizeclass)
{
@@ -963,28 +963,27 @@ namespace snmalloc
handle_message_queue_inner();
}
Superslab* get_superslab()
CapPtr<Superslab, CBChunk> get_superslab()
{
Superslab* super = super_available.get_head();
auto super = super_available.get_head();
if (super != nullptr)
return super;
super = large_allocator
.template alloc<NoZero>(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE)
.template as_reinterpret<Superslab>()
.unsafe_capptr;
.template as_reinterpret<Superslab>();
if (super == nullptr)
return super;
super->init(public_state());
chunkmap().set_slab(CapPtr<Superslab, CBArena>(super));
chunkmap().set_slab(super);
super_available.insert(super);
return super;
}
void reposition_superslab(Superslab* super)
void reposition_superslab(CapPtr<Superslab, CBChunk> super)
{
switch (super->get_status())
{
@@ -1025,11 +1024,12 @@ namespace snmalloc
{
// Pull a short slab from the list of superslabs that have only the
// short slab available.
Superslab* super = super_only_short_available.pop();
CapPtr<Superslab, CBChunk> super = super_only_short_available.pop();
if (super != nullptr)
{
auto slab = Superslab::alloc_short_slab(super, sizeclass);
auto slab =
Superslab::alloc_short_slab(super.unsafe_capptr, sizeclass);
SNMALLOC_ASSERT(super->is_full());
return CapPtr<Slab, CBArena>(slab);
}
@@ -1039,17 +1039,17 @@ namespace snmalloc
if (super == nullptr)
return nullptr;
auto slab = Superslab::alloc_short_slab(super, sizeclass);
auto slab = Superslab::alloc_short_slab(super.unsafe_capptr, sizeclass);
reposition_superslab(super);
return CapPtr<Slab, CBArena>(slab);
}
Superslab* super = get_superslab();
auto super = get_superslab();
if (super == nullptr)
return nullptr;
auto slab = Superslab::alloc_slab(super, sizeclass);
auto slab = Superslab::alloc_slab(super.unsafe_capptr, sizeclass);
reposition_superslab(super);
return CapPtr<Slab, CBArena>(slab);
}
@@ -1233,7 +1233,7 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH void small_dealloc_unchecked(
CapPtr<Superslab, CBArena> super,
CapPtr<Superslab, CBChunkD> super,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
@@ -1246,7 +1246,7 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH void small_dealloc_checked_chunkmap(
CapPtr<Superslab, CBArena> super,
CapPtr<Superslab, CBChunkD> super,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
@@ -1260,8 +1260,8 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH void small_dealloc_checked_sizeclass(
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBArena> slab,
CapPtr<Superslab, CBChunkD> super,
CapPtr<Slab, CBChunkD> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
@@ -1274,8 +1274,8 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH void small_dealloc_start(
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBArena> slab,
CapPtr<Superslab, CBChunkD> super,
CapPtr<Slab, CBChunkD> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
@@ -1295,8 +1295,8 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH void small_dealloc_offseted(
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBArena> slab,
CapPtr<Superslab, CBChunkD> super,
CapPtr<Slab, CBChunkD> slab,
CapPtr<FreeObject, CBArena> p,
sizeclass_t sizeclass)
{
@@ -1306,8 +1306,8 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH void small_dealloc_offseted_inner(
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBArena> slab,
CapPtr<Superslab, CBChunkD> super,
CapPtr<Slab, CBChunkD> slab,
CapPtr<FreeObject, CBArena> p,
sizeclass_t sizeclass)
{
@@ -1318,8 +1318,8 @@ namespace snmalloc
}
SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow(
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBArena> slab,
CapPtr<Superslab, CBChunkD> super,
CapPtr<Slab, CBChunkD> slab,
CapPtr<FreeObject, CBArena> p,
sizeclass_t sizeclass)
{
@@ -1333,6 +1333,8 @@ namespace snmalloc
if (a == Superslab::NoStatusChange)
return;
auto super_slab = capptr_chunk_from_chunkd(super, SUPERSLAB_SIZE);
switch (super->get_status())
{
case Superslab::Full:
@@ -1345,29 +1347,29 @@ namespace snmalloc
{
if (was_full)
{
super_available.insert(super.unsafe_capptr);
super_available.insert(super_slab);
}
else
{
super_only_short_available.remove(super.unsafe_capptr);
super_available.insert(super.unsafe_capptr);
super_only_short_available.remove(super_slab);
super_available.insert(super_slab);
}
break;
}
case Superslab::OnlyShortSlabAvailable:
{
super_only_short_available.insert(super.unsafe_capptr);
super_only_short_available.insert(super_slab);
break;
}
case Superslab::Empty:
{
super_available.remove(super.unsafe_capptr);
super_available.remove(super_slab);
chunkmap().clear_slab(super);
chunkmap().clear_slab(super_slab);
large_allocator.dealloc(
super.template as_reinterpret<Largeslab>(), 0);
super_slab.template as_reinterpret<Largeslab>(), 0);
stats().superslab_push();
break;
}
@@ -1381,7 +1383,7 @@ namespace snmalloc
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
auto sc = &medium_classes[medium_class];
auto slab = sc->get_head();
CapPtr<Mediumslab, CBChunk> slab = sc->get_head();
CapPtr<void, CBAllocE> p;
if (slab != nullptr)
@@ -1437,7 +1439,7 @@ namespace snmalloc
SNMALLOC_FAST_PATH
void medium_dealloc_unchecked(
CapPtr<Mediumslab, CBArena> slab,
CapPtr<Mediumslab, CBChunkD> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
@@ -1451,7 +1453,7 @@ namespace snmalloc
SNMALLOC_FAST_PATH
void medium_dealloc_checked_chunkmap(
CapPtr<Mediumslab, CBArena> slab,
CapPtr<Mediumslab, CBChunkD> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
@@ -1465,7 +1467,7 @@ namespace snmalloc
SNMALLOC_FAST_PATH
void medium_dealloc_checked_sizeclass(
CapPtr<Mediumslab, CBArena> slab,
CapPtr<Mediumslab, CBChunkD> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
@@ -1480,7 +1482,7 @@ namespace snmalloc
SNMALLOC_FAST_PATH
void medium_dealloc_start(
CapPtr<Mediumslab, CBArena> slab,
CapPtr<Mediumslab, CBChunkD> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
@@ -1498,31 +1500,34 @@ namespace snmalloc
SNMALLOC_FAST_PATH
void medium_dealloc_local(
CapPtr<Mediumslab, CBArena> slab,
CapPtr<Mediumslab, CBChunkD> slab,
CapPtr<void, CBArena> p,
sizeclass_t sizeclass)
{
stats().sizeclass_dealloc(sizeclass);
bool was_full = Mediumslab::dealloc(slab, p);
auto slab_bounded = capptr_chunk_from_chunkd(slab, SUPERSLAB_SIZE);
if (Mediumslab::empty(slab))
{
if (!was_full)
{
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
auto sc = &medium_classes[medium_class];
sc->remove(slab);
sc->remove(slab_bounded);
}
chunkmap().clear_slab(slab);
large_allocator.dealloc(slab.template as_reinterpret<Largeslab>(), 0);
chunkmap().clear_slab(slab_bounded);
large_allocator.dealloc(
slab_bounded.template as_reinterpret<Largeslab>(), 0);
stats().superslab_push();
}
else if (was_full)
{
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
auto sc = &medium_classes[medium_class];
sc->insert(slab);
sc->insert(slab_bounded);
}
}
@@ -1549,7 +1554,7 @@ namespace snmalloc
if (large_class == 0)
size = rsize;
auto p =
CapPtr<Largeslab, CBChunk> p =
large_allocator.template alloc<zero_mem>(large_class, rsize, size);
if (likely(p != nullptr))
{
@@ -1615,13 +1620,13 @@ namespace snmalloc
}
size_t large_class = chunkmap_slab_kind - SUPERSLAB_BITS;
auto slab = Aal::capptr_bound<Largeslab, CBChunk>(p_auth, size);
chunkmap().clear_large_size(p_auth, size);
chunkmap().clear_large_size(slab, size);
stats().large_dealloc(large_class);
// Initialise in order to set the correct SlabKind.
auto slab = p_auth.template as_static<Largeslab>();
slab->init();
large_allocator.dealloc(slab, large_class);
}

12
src/mem/chunkmap.h
View File

@@ -118,21 +118,21 @@ namespace snmalloc
* Set a pagemap entry indicating that there is a superslab at the
* specified index.
*/
static void set_slab(CapPtr<Superslab, CBArena> slab)
static void set_slab(CapPtr<Superslab, CBChunk> slab)
{
set(address_cast(slab), static_cast<size_t>(CMSuperslab));
}
/**
* Add a pagemap entry indicating that a medium slab has been allocated.
*/
static void set_slab(CapPtr<Mediumslab, CBArena> slab)
static void set_slab(CapPtr<Mediumslab, CBChunk> slab)
{
set(address_cast(slab), static_cast<size_t>(CMMediumslab));
}
/**
* Remove an entry from the pagemap corresponding to a superslab.
*/
static void clear_slab(CapPtr<Superslab, CBArena> slab)
static void clear_slab(CapPtr<Superslab, CBChunk> slab)
{
SNMALLOC_ASSERT(get(address_cast(slab)) == CMSuperslab);
set(address_cast(slab), static_cast<size_t>(CMNotOurs));
@@ -140,7 +140,7 @@ namespace snmalloc
/**
* Remove an entry corresponding to a medium slab.
*/
static void clear_slab(CapPtr<Mediumslab, CBArena> slab)
static void clear_slab(CapPtr<Mediumslab, CBChunk> slab)
{
SNMALLOC_ASSERT(get(address_cast(slab)) == CMMediumslab);
set(address_cast(slab), static_cast<size_t>(CMNotOurs));
@@ -149,7 +149,7 @@ namespace snmalloc
* Update the pagemap to reflect a large allocation, of `size` bytes from
* address `p`.
*/
static void set_large_size(CapPtr<Largeslab, CBArena> p, size_t size)
static void set_large_size(CapPtr<Largeslab, CBChunk> p, size_t size)
{
size_t size_bits = bits::next_pow2_bits(size);
set(address_cast(p), static_cast<uint8_t>(size_bits));
@@ -167,7 +167,7 @@ namespace snmalloc
* Update the pagemap to remove a large allocation, of `size` bytes from
* address `p`.
*/
static void clear_large_size(CapPtr<void, CBArena> vp, size_t size)
static void clear_large_size(CapPtr<Largeslab, CBChunk> vp, size_t size)
{
auto p = address_cast(vp);
size_t rounded_size = bits::next_pow2(size);

3
src/mem/freelist.h
View File

@@ -325,7 +325,8 @@ namespace snmalloc
* Start building a new free list.
* Provide pointer to the slab to initialise the system.
*/
void open(CapPtr<void, CBArena> p)
template<capptr_bounds B> // TODO: CBChunk-only
void open(CapPtr<void, B> p)
{
SNMALLOC_ASSERT(empty());
for (size_t i = 0; i < LENGTH; i++)

22
src/mem/largealloc.h
View File

@@ -32,7 +32,7 @@ namespace snmalloc
friend class MPMCStack;
template<SNMALLOC_CONCEPT(ConceptPAL) PAL, typename ArenaMap>
friend class MemoryProviderStateMixin;
AtomicCapPtr<Largeslab, CBArena> next = nullptr;
AtomicCapPtr<Largeslab, CBChunk> next = nullptr;
public:
void init()
@@ -104,7 +104,7 @@ namespace snmalloc
*/
ModArray<
NUM_LARGE_CLASSES,
MPMCStack<Largeslab, RequiresInit, CapPtrCBArena, AtomicCapPtrCBArena>>
MPMCStack<Largeslab, RequiresInit, CapPtrCBChunk, AtomicCapPtrCBChunk>>
large_stack;
public:
@@ -115,7 +115,7 @@ namespace snmalloc
* concurrently with other acceses. If there is no large allocation on a
* particular stack then this will return `nullptr`.
*/
SNMALLOC_FAST_PATH CapPtr<Largeslab, CBArena>
SNMALLOC_FAST_PATH CapPtr<Largeslab, CBChunk>
pop_large_stack(size_t large_class)
{
auto p = large_stack[large_class].pop();
@@ -132,7 +132,7 @@ namespace snmalloc
* class specified by `large_class`. Always succeeds.
*/
SNMALLOC_FAST_PATH void
push_large_stack(CapPtr<Largeslab, CBArena> slab, size_t large_class)
push_large_stack(CapPtr<Largeslab, CBChunk> slab, size_t large_class)
{
const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
available_large_chunks_in_bytes += rsize;
@@ -150,7 +150,7 @@ namespace snmalloc
* memory providers constructed in this way does not have to be able to
* allocate memory, if the initial reservation is sufficient.
*/
MemoryProviderStateMixin(CapPtr<void, CBArena> start, size_t len)
MemoryProviderStateMixin(CapPtr<void, CBChunk> start, size_t len)
: address_space(start, len)
{}
/**
@@ -214,7 +214,7 @@ namespace snmalloc
size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
size_t decommit_size = rsize - OS_PAGE_SIZE;
// Grab all of the chunks of this size class.
CapPtr<Largeslab, CBArena> slab = large_stack[large_class].pop_all();
CapPtr<Largeslab, CBChunk> slab = large_stack[large_class].pop_all();
while (slab != nullptr)
{
// Decommit all except for the first page and then put it back on
@@ -229,7 +229,7 @@ namespace snmalloc
// happens-before relationship, so it's safe to use relaxed loads
// here.
auto next = slab->next.load(std::memory_order_relaxed);
large_stack[large_class].push(CapPtr<Largeslab, CBArena>(
large_stack[large_class].push(CapPtr<Largeslab, CBChunk>(
new (slab.unsafe_capptr) Decommittedslab()));
slab = next;
}
@@ -279,7 +279,7 @@ namespace snmalloc
}
template<bool committed>
CapPtr<Largeslab, CBArena> reserve(size_t large_class) noexcept
CapPtr<Largeslab, CBChunk> reserve(size_t large_class) noexcept
{
size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
peak_memory_used_bytes += size;
@@ -324,13 +324,13 @@ namespace snmalloc
LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {}
template<ZeroMem zero_mem = NoZero>
CapPtr<Largeslab, CBArena>
CapPtr<Largeslab, CBChunk>
alloc(size_t large_class, size_t rsize, size_t size)
{
SNMALLOC_ASSERT(
(bits::one_at_bit(SUPERSLAB_BITS) << large_class) == rsize);
CapPtr<Largeslab, CBArena> p =
CapPtr<Largeslab, CBChunk> p =
memory_provider.pop_large_stack(large_class);
if (p == nullptr)
@@ -381,7 +381,7 @@ namespace snmalloc
return p;
}
void dealloc(CapPtr<Largeslab, CBArena> p, size_t large_class)
void dealloc(CapPtr<Largeslab, CBChunk> p, size_t large_class)
{
if constexpr (decommit_strategy == DecommitSuperLazy)
{

32
src/mem/mediumslab.h
View File

@@ -12,12 +12,12 @@ namespace snmalloc
// This is the view of a 16 mb area when it is being used to allocate
// medium sized classes: 64 kb to 16 mb, non-inclusive.
private:
friend DLList<Mediumslab, CapPtrCBArena>;
friend DLList<Mediumslab, CapPtrCBChunk>;
// Keep the allocator pointer on a separate cache line. It is read by
// other threads, and does not change, so we avoid false sharing.
alignas(CACHELINE_SIZE) CapPtr<Mediumslab, CBArena> next;
CapPtr<Mediumslab, CBArena> prev;
alignas(CACHELINE_SIZE) CapPtr<Mediumslab, CBChunk> next;
CapPtr<Mediumslab, CBChunk> prev;
uint16_t free;
uint8_t head;
@@ -44,16 +44,26 @@ namespace snmalloc
return bits::align_up(sizeof(Mediumslab), min(OS_PAGE_SIZE, SLAB_SIZE));
}
template<typename T, capptr_bounds B>
static SNMALLOC_FAST_PATH CapPtr<Mediumslab, B> get(CapPtr<T, B> p)
/**
* Given a highly-privileged pointer pointing to or within an object in
* this slab, return a pointer to the slab headers.
*
* In debug builds on StrictProvenance architectures, we will enforce the
* slab bounds on this returned pointer. In non-debug builds, we will
* return a highly-privileged pointer (i.e., CBArena) instead as these
* pointers are not exposed from the allocator.
*/
template<typename T>
static SNMALLOC_FAST_PATH CapPtr<Mediumslab, CBChunkD>
get(CapPtr<T, CBArena> p)
{
static_assert(B == CBArena || B == CBChunk);
return pointer_align_down<SUPERSLAB_SIZE, Mediumslab>(p);
return capptr_bound_chunkd(
pointer_align_down<SUPERSLAB_SIZE, Mediumslab>(p.as_void()),
SUPERSLAB_SIZE);
}
static void init(
CapPtr<Mediumslab, CBArena> self,
CapPtr<Mediumslab, CBChunk> self,
RemoteAllocator* alloc,
sizeclass_t sc,
size_t rsize)
@@ -89,7 +99,7 @@ namespace snmalloc
template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
static CapPtr<void, CBAllocE>
alloc(CapPtr<Mediumslab, CBArena> self, size_t size)
alloc(CapPtr<Mediumslab, CBChunk> self, size_t size)
{
SNMALLOC_ASSERT(!full(self));
@@ -106,7 +116,7 @@ namespace snmalloc
}
static bool
dealloc(CapPtr<Mediumslab, CBArena> self, CapPtr<void, CBArena> p)
dealloc(CapPtr<Mediumslab, CBChunkD> self, CapPtr<void, CBArena> p)
{
SNMALLOC_ASSERT(self->head > 0);

36
src/mem/metaslab.h
View File

@@ -4,14 +4,15 @@
#include "../ds/dllist.h"
#include "../ds/helpers.h"
#include "freelist.h"
#include "ptrhelpers.h"
#include "sizeclass.h"
namespace snmalloc
{
class Slab;
using SlabList = CDLLNode<CapPtrCBArena>;
using SlabLink = CDLLNode<CapPtrCBArena>;
using SlabList = CDLLNode<CapPtrCBChunk>;
using SlabLink = CDLLNode<CapPtrCBChunk>;
static_assert(
sizeof(SlabLink) <= MIN_ALLOC_SIZE,
@@ -71,7 +72,7 @@ namespace snmalloc
return free_queue.s.next;
}
void initialise(sizeclass_t sizeclass, CapPtr<Slab, CBArena> slab)
void initialise(sizeclass_t sizeclass, CapPtr<Slab, CBChunk> slab)
{
free_queue.s.sizeclass = static_cast<uint8_t>(sizeclass);
free_queue.init();
@@ -120,8 +121,10 @@ namespace snmalloc
return bits::min(threshold, max);
}
SNMALLOC_FAST_PATH void set_full(CapPtr<Slab, CBArena> slab)
template<capptr_bounds B>
SNMALLOC_FAST_PATH void set_full(CapPtr<Slab, B> slab)
{
static_assert(B == CBChunkD || B == CBChunk);
SNMALLOC_ASSERT(free_queue.empty());
// Prepare for the next free queue to be built.
@@ -134,9 +137,13 @@ namespace snmalloc
}
template<typename T, capptr_bounds B>
static CapPtr<Slab, B> get_slab(CapPtr<T, B> p)
static SNMALLOC_FAST_PATH CapPtr<Slab, capptr_bound_chunkd_bounds<B>()>
get_slab(CapPtr<T, B> p)
{
return pointer_align_down<SLAB_SIZE, Slab>(p.as_void());
static_assert(B == CBArena || B == CBChunkD || B == CBChunk);
return capptr_bound_chunkd(
pointer_align_down<SLAB_SIZE, Slab>(p.as_void()), SLAB_SIZE);
}
template<capptr_bounds B>
@@ -145,8 +152,9 @@ namespace snmalloc
return pointer_align_down<SUPERSLAB_SIZE, Slab>(p.as_void()) == p;
}
SNMALLOC_FAST_PATH
static bool is_start_of_object(CapPtr<Metaslab, CBArena> self, address_t p)
template<capptr_bounds B>
SNMALLOC_FAST_PATH static bool
is_start_of_object(CapPtr<Metaslab, B> self, address_t p)
{
return is_multiple_of_sizeclass(
self->sizeclass(), SLAB_SIZE - (p - address_align_down<SLAB_SIZE>(p)));
@@ -159,7 +167,7 @@ namespace snmalloc
*/
template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
static SNMALLOC_FAST_PATH CapPtr<void, CBAllocE> alloc(
CapPtr<Metaslab, CBArena> self,
CapPtr<Metaslab, CBChunk> self,
FreeListIter& fast_free_list,
size_t rsize,
LocalEntropy& entropy)
@@ -169,17 +177,18 @@ namespace snmalloc
self->free_queue.close(fast_free_list, entropy);
auto n = fast_free_list.take(entropy);
auto n_slab = Aal::capptr_rebound(self.as_void(), n);
entropy.refresh_bits();
// Treat stealing the free list as allocating it all.
self->remove();
self->set_full(Metaslab::get_slab(n));
self->set_full(Metaslab::get_slab(n_slab));
auto p = remove_cache_friendly_offset(n, self->sizeclass());
SNMALLOC_ASSERT(is_start_of_object(self, address_cast(p)));
self->debug_slab_invariant(Metaslab::get_slab(n), entropy);
self->debug_slab_invariant(Metaslab::get_slab(n_slab), entropy);
if constexpr (zero_mem == YesZero)
{
@@ -196,8 +205,11 @@ namespace snmalloc
return capptr_export(Aal::capptr_bound<void, CBAlloc>(p, rsize));
}
void debug_slab_invariant(CapPtr<Slab, CBArena> slab, LocalEntropy& entropy)
template<capptr_bounds B>
void debug_slab_invariant(CapPtr<Slab, B> slab, LocalEntropy& entropy)
{
static_assert(B == CBChunkD || B == CBChunk);
#if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS)
bool is_short = Metaslab::is_short(slab);

18
src/mem/slab.h
View File

@@ -1,6 +1,7 @@
#pragma once
#include "freelist.h"
#include "ptrhelpers.h"
#include "superslab.h"
#include <array>
@@ -20,7 +21,7 @@ namespace snmalloc
template<capptr_bounds B>
static CapPtr<Metaslab, B> get_meta(CapPtr<Slab, B> self)
{
static_assert(B == CBArena || B == CBChunk);
static_assert(B == CBArena || B == CBChunkD || B == CBChunk);
auto super = Superslab::get(self);
return super->get_meta(self);
@@ -102,8 +103,8 @@ namespace snmalloc
// bits. Note that this does remove the use from the meta slab, so it
// doesn't need doing on the slow path.
static SNMALLOC_FAST_PATH bool dealloc_fast(
CapPtr<Slab, CBArena> self,
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBChunkD> self,
CapPtr<Superslab, CBChunkD> super,
CapPtr<FreeObject, CBArena> p,
LocalEntropy& entropy)
{
@@ -124,9 +125,9 @@ namespace snmalloc
// Returns a complex return code for managing the superslab meta data.
// i.e. This deallocation could make an entire superslab free.
static SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow(
CapPtr<Slab, CBArena> self,
CapPtr<Slab, CBChunkD> self,
SlabList* sl,
CapPtr<Superslab, CBArena> super,
CapPtr<Superslab, CBChunkD> super,
CapPtr<FreeObject, CBArena> p,
LocalEntropy& entropy)
{
@@ -154,7 +155,12 @@ namespace snmalloc
allocated - meta->threshold_for_waking_slab(Metaslab::is_short(self));
// Push on the list of slabs for this sizeclass.
sl->insert_prev(meta.template as_static<SlabLink>());
// ChunkD-to-Chunk conversion might apply bounds, so we need to do so to
// the aligned object and then shift over to these bounds.
auto super_chunk = capptr_chunk_from_chunkd(super, SUPERSLAB_SIZE);
auto metalink = Aal::capptr_rebound(
super_chunk.as_void(), meta.template as_static<SlabLink>());
sl->insert_prev(metalink);
meta->debug_slab_invariant(self, entropy);
return Superslab::NoSlabReturn;
}

30
src/mem/superslab.h
View File

@@ -30,15 +30,15 @@ namespace snmalloc
class Superslab : public Allocslab
{
private:
friend DLList<Superslab>;
friend DLList<Superslab, CapPtrCBChunk>;
// Keep the allocator pointer on a separate cache line. It is read by
// other threads, and does not change, so we avoid false sharing.
alignas(CACHELINE_SIZE)
// The superslab is kept on a doubly linked list of superslabs which
// have some space.
Superslab* next;
Superslab* prev;
CapPtr<Superslab, CBChunk> next;
CapPtr<Superslab, CBChunk> prev;
// This is a reference to the first unused slab in the free slab list
// It is does not contain the short slab, which is handled using a bit
@@ -83,12 +83,24 @@ namespace snmalloc
StatusChange = 2
};
/**
* Given a highly-privileged pointer pointing to or within an object in
* this slab, return a pointer to the slab headers.
*
* In debug builds on StrictProvenance architectures, we will enforce the
* slab bounds on this returned pointer. In non-debug builds, we will
* return a highly-privileged pointer (i.e., CBArena) instead as these
* pointers are not exposed from the allocator.
*/
template<typename T, capptr_bounds B>
static SNMALLOC_FAST_PATH CapPtr<Superslab, B> get(CapPtr<T, B> p)
static SNMALLOC_FAST_PATH CapPtr<Superslab, capptr_bound_chunkd_bounds<B>()>
get(CapPtr<T, B> p)
{
static_assert(B == CBArena || B == CBChunk);
static_assert(B == CBArena || B == CBChunkD || B == CBChunk);
return pointer_align_down<SUPERSLAB_SIZE, Superslab>(p.as_void());
return capptr_bound_chunkd(
pointer_align_down<SUPERSLAB_SIZE, Superslab>(p.as_void()),
SUPERSLAB_SIZE);
}
static bool is_short_sizeclass(sizeclass_t sizeclass)
@@ -199,7 +211,7 @@ namespace snmalloc
Slab* slab = reinterpret_cast<Slab*>(self);
auto& metaz = self->meta[0];
metaz.initialise(sizeclass, CapPtr<Slab, CBArena>(slab));
metaz.initialise(sizeclass, CapPtr<Slab, CBChunk>(slab));
self->used++;
return slab;
@@ -216,7 +228,7 @@ namespace snmalloc
auto& metah = self->meta[h];
uint8_t n = metah.next();
metah.initialise(sizeclass, CapPtr<Slab, CBArena>(slab));
metah.initialise(sizeclass, CapPtr<Slab, CBChunk>(slab));
self->head = h + n + 1;
self->used += 2;
@@ -228,7 +240,7 @@ namespace snmalloc
template<capptr_bounds B>
Action dealloc_slab(CapPtr<Slab, B> slab)
{
static_assert(B == CBArena || B == CBChunk);
static_assert(B == CBArena || B == CBChunkD || B == CBChunk);
// This is not the short slab.
uint8_t index = static_cast<uint8_t>(slab_to_index(slab));

8
src/test/func/sandbox/sandbox.cc
View File

@@ -113,7 +113,7 @@ namespace
*
* This method must be implemented for `LargeAlloc` to work.
*/
CapPtr<Largeslab, CBArena> pop_large_stack(size_t large_class)
CapPtr<Largeslab, CBChunk> pop_large_stack(size_t large_class)
{
return real_state->pop_large_stack(large_class);
};
@@ -124,7 +124,7 @@ namespace
*
* This method must be implemented for `LargeAlloc` to work.
*/
void push_large_stack(CapPtr<Largeslab, CBArena> slab, size_t large_class)
void push_large_stack(CapPtr<Largeslab, CBChunk> slab, size_t large_class)
{
real_state->push_large_stack(slab, large_class);
}
@@ -136,7 +136,7 @@ namespace
* This method must be implemented for `LargeAlloc` to work.
*/
template<bool committed>
CapPtr<Largeslab, CBArena> reserve(size_t large_class) noexcept
CapPtr<Largeslab, CBChunk> reserve(size_t large_class) noexcept
{
return real_state->template reserve<committed>(large_class);
}
@@ -207,7 +207,7 @@ namespace
top(pointer_offset(start, sb_size)),
shared_state(new (start) SharedState()),
state(
pointer_offset(CapPtr<void, CBArena>(start), sizeof(SharedState)),
pointer_offset(CapPtr<void, CBChunk>(start), sizeof(SharedState)),
sb_size - sizeof(SharedState)),
alloc(state, SNMALLOC_DEFAULT_CHUNKMAP(), &shared_state->queue)
{