Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SP: start plumbing CapPtr<>s

Browse Source
This commit is contained in:
Nathaniel Filardo
2021-03-05 16:46:25 +00:00
committed by Nathaniel Wesley Filardo
parent 313a682faf
commit 005f5787ef
10 changed files with 431 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

412
src/mem/alloc.h
View File

@@ -105,7 +105,7 @@ namespace snmalloc
* If aligned to a SLAB start, then it is empty, and a new
* slab is required.
*/
void* bump_ptrs[NUM_SMALL_CLASSES] = {nullptr};
CapPtr<void, CBArena> bump_ptrs[NUM_SMALL_CLASSES] = {nullptr};
public:
Stats& stats()
@@ -210,29 +210,32 @@ namespace snmalloc
* external pointer.
*/
template<size_t size>
void dealloc(void* p)
void dealloc(void* p_raw)
{
#ifdef SNMALLOC_PASS_THROUGH
UNUSED(size);
return external_alloc::free(p);
return external_alloc::free(p_raw);
#else
constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size);
auto p_ret = CapPtr<void, CBAllocE>(p_raw);
auto p_auth = CapPtr<void, CBArena>(p_raw);
if (sizeclass < NUM_SMALL_CLASSES)
{
Superslab* super = Superslab::get(p);
auto super = Superslab::get(p_auth);
small_dealloc_unchecked(super, p, sizeclass);
small_dealloc_unchecked(super, p_auth, p_ret, sizeclass);
}
else if (sizeclass < NUM_SIZECLASSES)
{
Mediumslab* slab = Mediumslab::get(p);
auto slab = Mediumslab::get(p_auth);
medium_dealloc_unchecked(slab, p, sizeclass);
medium_dealloc_unchecked(slab, p_auth, p_ret, sizeclass);
}
else
{
large_dealloc_unchecked(p, size);
large_dealloc_unchecked(p_auth, p_ret, size);
}
#endif
}
@@ -241,53 +244,59 @@ namespace snmalloc
* Free memory of a dynamically known size. Must be called with an
* external pointer.
*/
SNMALLOC_FAST_PATH void dealloc(void* p, size_t size)
SNMALLOC_FAST_PATH void dealloc(void* p_raw, size_t size)
{
#ifdef SNMALLOC_PASS_THROUGH
UNUSED(size);
return external_alloc::free(p);
return external_alloc::free(p_raw);
#else
SNMALLOC_ASSERT(p != nullptr);
SNMALLOC_ASSERT(p_raw != nullptr);
auto p_ret = CapPtr<void, CBAllocE>(p_raw);
auto p_auth = CapPtr<void, CBArena>(p_raw);
if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1)))
{
Superslab* super = Superslab::get(p);
auto super = Superslab::get(p_auth);
sizeclass_t sizeclass = size_to_sizeclass(size);
small_dealloc_unchecked(super, p, sizeclass);
small_dealloc_unchecked(super, p_auth, p_ret, sizeclass);
return;
}
dealloc_sized_slow(p, size);
dealloc_sized_slow(p_auth, p_ret, size);
#endif
}
SNMALLOC_SLOW_PATH void dealloc_sized_slow(void* p, size_t size)
SNMALLOC_SLOW_PATH void dealloc_sized_slow(
CapPtr<void, CBArena> p_auth, CapPtr<void, CBAllocE> p_ret, size_t size)
{
if (size == 0)
return dealloc(p, 1);
return dealloc(p_ret.unsafe_capptr, 1);
if (likely(size <= sizeclass_to_size(NUM_SIZECLASSES - 1)))
{
Mediumslab* slab = Mediumslab::get(p);
auto slab = Mediumslab::get(p_auth);
sizeclass_t sizeclass = size_to_sizeclass(size);
medium_dealloc_unchecked(slab, p, sizeclass);
medium_dealloc_unchecked(slab, p_auth, p_ret, sizeclass);
return;
}
large_dealloc_unchecked(p, size);
large_dealloc_unchecked(p_auth, p_ret, size);
}
/*
* Free memory of an unknown size. Must be called with an external
* pointer.
*/
SNMALLOC_FAST_PATH void dealloc(void* p)
SNMALLOC_FAST_PATH void dealloc(void* p_raw)
{
#ifdef SNMALLOC_PASS_THROUGH
return external_alloc::free(p);
return external_alloc::free(p_raw);
#else
uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p));
uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p_raw));
Superslab* super = Superslab::get(p);
auto p_ret = CapPtr<void, CBAllocE>(p_raw);
auto p_auth = CapPtr<void, CBArena>(p_raw);
if (likely(chunkmap_slab_kind == CMSuperslab))
{
@@ -305,22 +314,25 @@ namespace snmalloc
* through the guard in small_dealloc_start(), we must treat this as
* possibly stale and suspect.
*/
Slab* slab = Metaslab::get_slab(p);
Metaslab& meta = super->get_meta(slab);
sizeclass_t sizeclass = meta.sizeclass();
auto super = Superslab::get(p_auth);
auto slab = Metaslab::get_slab(p_auth);
auto meta = super->get_meta(slab);
sizeclass_t sizeclass = meta->sizeclass();
small_dealloc_checked_sizeclass(super, slab, p, sizeclass);
small_dealloc_checked_sizeclass(super, slab, p_auth, p_ret, sizeclass);
return;
}
dealloc_not_small(p, chunkmap_slab_kind);
dealloc_not_small(p_auth, p_ret, chunkmap_slab_kind);
}
SNMALLOC_SLOW_PATH void
dealloc_not_small(void* p, uint8_t chunkmap_slab_kind)
SNMALLOC_SLOW_PATH void dealloc_not_small(
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
uint8_t chunkmap_slab_kind)
{
handle_message_queue();
if (p == nullptr)
if (p_ret == nullptr)
return;
if (chunkmap_slab_kind == CMMediumslab)
@@ -330,10 +342,10 @@ namespace snmalloc
* values are suspect until we complete the double-free check in
* medium_dealloc_smart().
*/
Mediumslab* slab = Mediumslab::get(p);
auto slab = Mediumslab::get(p_auth);
sizeclass_t sizeclass = slab->get_sizeclass();
medium_dealloc_checked_sizeclass(slab, p, sizeclass);
medium_dealloc_checked_sizeclass(slab, p_auth, p_ret, sizeclass);
return;
}
@@ -343,41 +355,46 @@ namespace snmalloc
}
large_dealloc_checked_sizeclass(
p, bits::one_at_bit(chunkmap_slab_kind), chunkmap_slab_kind);
p_auth,
p_ret,
bits::one_at_bit(chunkmap_slab_kind),
chunkmap_slab_kind);
#endif
}
template<Boundary location = Start>
void* external_pointer(void* p)
void* external_pointer(void* p_raw)
{
#ifdef SNMALLOC_PASS_THROUGH
error("Unsupported");
UNUSED(p);
UNUSED(p_raw);
#else
uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p));
uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p_raw));
auto p_ret = CapPtr<void, CBAllocE>(p_raw);
auto p_auth = CapPtr<void, CBArena>(p_raw);
Superslab* super = Superslab::get(p);
auto super = Superslab::get(p_auth);
if (chunkmap_slab_kind == CMSuperslab)
{
Slab* slab = Metaslab::get_slab(p);
Metaslab& meta = super->get_meta(slab);
auto slab = Metaslab::get_slab(p_auth);
auto meta = super->get_meta(slab);
sizeclass_t sc = meta.sizeclass();
void* slab_end = pointer_offset(slab, SLAB_SIZE);
sizeclass_t sc = meta->sizeclass();
auto slab_end = pointer_offset(slab, SLAB_SIZE);
return external_pointer<location>(p, sc, slab_end);
return capptr_reveal(external_pointer<location>(p_ret, sc, slab_end));
}
if (chunkmap_slab_kind == CMMediumslab)
{
Mediumslab* slab = Mediumslab::get(p);
auto slab = Mediumslab::get(p_auth);
sizeclass_t sc = slab->get_sizeclass();
void* slab_end = pointer_offset(slab, SUPERSLAB_SIZE);
auto slab_end = pointer_offset(slab, SUPERSLAB_SIZE);
return external_pointer<location>(p, sc, slab_end);
return capptr_reveal(external_pointer<location>(p_ret, sc, slab_end));
}
auto ss = super;
auto ss = super.unsafe_capptr;
while (chunkmap_slab_kind >= CMLargeRangeMin)
{
@@ -386,7 +403,7 @@ namespace snmalloc
ss,
-(static_cast<ptrdiff_t>(1)
<< (chunkmap_slab_kind - CMLargeRangeMin + SUPERSLAB_BITS)));
chunkmap_slab_kind = chunkmap().get(ss);
chunkmap_slab_kind = chunkmap().get(address_cast(ss));
}
if (chunkmap_slab_kind == CMNotOurs)
@@ -420,29 +437,30 @@ namespace snmalloc
}
public:
SNMALLOC_FAST_PATH size_t alloc_size(const void* p)
SNMALLOC_FAST_PATH size_t alloc_size(const void* p_raw)
{
#ifdef SNMALLOC_PASS_THROUGH
return external_alloc::malloc_usable_size(const_cast<void*>(p));
return external_alloc::malloc_usable_size(const_cast<void*>(p_raw));
#else
// This must be called on an external pointer.
size_t chunkmap_slab_kind = chunkmap().get(address_cast(p));
size_t chunkmap_slab_kind = chunkmap().get(address_cast(p_raw));
auto p_auth = CapPtr<void, CBArena>(const_cast<void*>(p_raw));
if (likely(chunkmap_slab_kind == CMSuperslab))
{
Superslab* super = Superslab::get(p);
auto super = Superslab::get(p_auth);
// Reading a remote sizeclass won't fail, since the other allocator
// can't reuse the slab, as we have no yet deallocated this pointer.
Slab* slab = Metaslab::get_slab(p);
Metaslab& meta = super->get_meta(slab);
auto slab = Metaslab::get_slab(p_auth);
auto meta = super->get_meta(slab);
return sizeclass_to_size(meta.sizeclass());
return sizeclass_to_size(meta->sizeclass());
}
if (likely(chunkmap_slab_kind == CMMediumslab))
{
Mediumslab* slab = Mediumslab::get(p);
auto slab = Mediumslab::get(p_auth);
// Reading a remote sizeclass won't fail, since the other allocator
// can't reuse the slab, as we have no yet deallocated this pointer.
return sizeclass_to_size(slab->get_sizeclass());
@@ -576,7 +594,7 @@ namespace snmalloc
if (!l->empty())
{
// Send all slots to the target at the head of the list.
Superslab* super = Superslab::get(first);
auto super = Superslab::get(CapPtr<Remote, CBArena>(first));
super->get_allocator()->message_queue.enqueue(first, l->last);
l->clear();
}
@@ -611,7 +629,7 @@ namespace snmalloc
};
SlabList small_classes[NUM_SMALL_CLASSES];
DLList<Mediumslab> medium_classes[NUM_MEDIUM_CLASSES];
DLList<Mediumslab, CapPtrCBArena> medium_classes[NUM_MEDIUM_CLASSES];
DLList<Superslab> super_available;
DLList<Superslab> super_only_short_available;
@@ -747,8 +765,8 @@ namespace snmalloc
auto rsize = sizeclass_to_size(i);
FreeListIter ffl;
Superslab* super = Superslab::get(bp);
Slab* slab = Metaslab::get_slab(bp);
auto super = Superslab::get(bp);
auto slab = Metaslab::get_slab(bp);
while (pointer_align_up(bp, SLAB_SIZE) != bp)
{
Slab::alloc_new_list(bp, ffl, rsize, entropy);
@@ -789,24 +807,28 @@ namespace snmalloc
}
template<Boundary location>
static void*
external_pointer(void* p, sizeclass_t sizeclass, void* end_point)
static CapPtr<void, CBAllocE> external_pointer(
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass,
CapPtr<void, CBArena> end_point)
{
size_t rsize = sizeclass_to_size(sizeclass);
void* end_point_correction = location == End ?
auto end_point_correction = location == End ?
pointer_offset_signed(end_point, -1) :
(location == OnePastEnd ?
end_point :
pointer_offset_signed(end_point, -static_cast<ptrdiff_t>(rsize)));
size_t offset_from_end =
pointer_diff(p, pointer_offset_signed(end_point, -1));
pointer_diff(p_ret, pointer_offset_signed(end_point, -1));
size_t end_to_end = round_by_sizeclass(sizeclass, offset_from_end);
return pointer_offset_signed(
auto ret_auth = pointer_offset_signed(
end_point_correction, -static_cast<ptrdiff_t>(end_to_end));
return CapPtr<void, CBAllocE>(ret_auth.unsafe_capptr);
}
void init_message_queue()
@@ -827,14 +849,17 @@ namespace snmalloc
if (likely(p->trunc_target_id() == get_trunc_id()))
{
// Destined for my slabs
Superslab* super = Superslab::get(p);
auto p_auth = CapPtr<Remote, CBArena>(p);
auto super = Superslab::get(p_auth);
check_client(
p->trunc_target_id() == super->get_allocator()->trunc_id(),
"Detected memory corruption. Potential use-after-free");
void* start = remove_cache_friendly_offset(p, p->sizeclass());
dealloc_not_large_local(super, start, p, p->sizeclass());
auto start_auth = CapPtr<void, CBArena>(
remove_cache_friendly_offset(p_auth.unsafe_capptr, p->sizeclass()));
dealloc_not_large_local(
super, start_auth, p_auth.as_void(), p->sizeclass());
}
else
{
@@ -843,14 +868,17 @@ namespace snmalloc
}
}
SNMALLOC_SLOW_PATH void
dealloc_not_large(RemoteAllocator* target, void* p, sizeclass_t sizeclass)
SNMALLOC_SLOW_PATH void dealloc_not_large(
RemoteAllocator* target,
CapPtr<void, CBArena> p_auth,
sizeclass_t sizeclass)
{
void* offseted = apply_cache_friendly_offset(p, sizeclass);
auto offseted = CapPtr<void, CBArena>(
apply_cache_friendly_offset(p_auth.unsafe_capptr, sizeclass));
if (likely(target->trunc_id() == get_trunc_id()))
{
Superslab* super = Superslab::get(p);
dealloc_not_large_local(super, p, offseted, sizeclass);
auto super = Superslab::get(p_auth);
dealloc_not_large_local(super, p_auth, offseted, sizeclass);
}
else
{
@@ -859,7 +887,10 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH void dealloc_not_large_local(
Superslab* super, void* p, void* offseted, sizeclass_t sizeclass)
CapPtr<Superslab, CBArena> super,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBArena> p_auth_offseted,
sizeclass_t sizeclass)
{
// Guard against remote queues that have colliding IDs
SNMALLOC_ASSERT(super->get_allocator() == public_state());
@@ -867,13 +898,15 @@ namespace snmalloc
if (likely(sizeclass < NUM_SMALL_CLASSES))
{
SNMALLOC_ASSERT(super->get_kind() == Super);
Slab* slab = Metaslab::get_slab(p);
small_dealloc_offseted(super, slab, offseted, sizeclass);
auto slab = Metaslab::get_slab(p_auth);
small_dealloc_offseted(
super, slab, FreeObject::make(p_auth_offseted), sizeclass);
}
else
{
SNMALLOC_ASSERT(super->get_kind() == Medium);
medium_dealloc_local(Mediumslab::get(p), p, sizeclass);
medium_dealloc_local(
super.template as_reinterpret<Mediumslab>(), p_auth, sizeclass);
}
}
@@ -930,7 +963,7 @@ namespace snmalloc
return super;
super->init(public_state());
chunkmap().set_slab(super);
chunkmap().set_slab(CapPtr<Superslab, CBArena>(super));
super_available.insert(super);
return super;
}
@@ -969,7 +1002,7 @@ namespace snmalloc
}
}
SNMALLOC_SLOW_PATH Slab* alloc_slab(sizeclass_t sizeclass)
SNMALLOC_SLOW_PATH CapPtr<Slab, CBArena> alloc_slab(sizeclass_t sizeclass)
{
stats().sizeclass_alloc_slab(sizeclass);
if (Superslab::is_short_sizeclass(sizeclass))
@@ -982,7 +1015,7 @@ namespace snmalloc
{
auto slab = Superslab::alloc_short_slab(super, sizeclass);
SNMALLOC_ASSERT(super->is_full());
return slab;
return CapPtr<Slab, CBArena>(slab);
}
super = get_superslab();
@@ -992,7 +1025,7 @@ namespace snmalloc
auto slab = Superslab::alloc_short_slab(super, sizeclass);
reposition_superslab(super);
return slab;
return CapPtr<Slab, CBArena>(slab);
}
Superslab* super = get_superslab();
@@ -1002,7 +1035,7 @@ namespace snmalloc
auto slab = Superslab::alloc_slab(super, sizeclass);
reposition_superslab(super);
return slab;
return CapPtr<Slab, CBArena>(slab);
}
template<ZeroMem zero_mem>
@@ -1023,7 +1056,8 @@ namespace snmalloc
{
stats().alloc_request(size);
stats().sizeclass_alloc(sizeclass);
void* p = remove_cache_friendly_offset(fl.take(entropy), sizeclass);
void* p = remove_cache_friendly_offset(
fl.take(entropy).unsafe_capptr, sizeclass);
if constexpr (zero_mem == YesZero)
{
pal_zero<typename MemoryProvider::Pal>(
@@ -1066,7 +1100,7 @@ namespace snmalloc
stats().alloc_request(size);
stats().sizeclass_alloc(sizeclass);
auto meta = reinterpret_cast<Metaslab*>(sl.get_next());
auto meta = sl.get_next().template as_static<Metaslab>();
auto& ffl = small_fast_free_lists[sizeclass];
return Metaslab::alloc<zero_mem, typename MemoryProvider::Pal>(
meta, ffl, rsize, entropy);
@@ -1135,7 +1169,8 @@ namespace snmalloc
SNMALLOC_ASSERT(ffl.empty());
Slab::alloc_new_list(bp, ffl, rsize, entropy);
auto p = remove_cache_friendly_offset(ffl.take(entropy), sizeclass);
auto p = remove_cache_friendly_offset(
ffl.take(entropy).unsafe_capptr, sizeclass);
if constexpr (zero_mem == YesZero)
{
@@ -1154,64 +1189,87 @@ namespace snmalloc
{
auto& bp = bump_ptrs[sizeclass];
// Fetch new slab
Slab* slab = alloc_slab(sizeclass);
auto slab = alloc_slab(sizeclass);
if (slab == nullptr)
return nullptr;
bp = pointer_offset<void>(
bp = pointer_offset(
slab, get_initial_offset(sizeclass, Metaslab::is_short(slab)));
return small_alloc_build_free_list<zero_mem>(sizeclass);
}
SNMALLOC_FAST_PATH void
small_dealloc_unchecked(Superslab* super, void* p, sizeclass_t sizeclass)
SNMALLOC_FAST_PATH void small_dealloc_unchecked(
CapPtr<Superslab, CBArena> super,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
{
check_client(
chunkmap().get(address_cast(p)) == CMSuperslab,
chunkmap().get(address_cast(p_ret)) == CMSuperslab,
"Claimed small deallocation is not in a Superslab");
small_dealloc_checked_chunkmap(super, p, sizeclass);
small_dealloc_checked_chunkmap(super, p_auth, p_ret, sizeclass);
}
SNMALLOC_FAST_PATH void small_dealloc_checked_chunkmap(
Superslab* super, void* p, sizeclass_t sizeclass)
CapPtr<Superslab, CBArena> super,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
{
Slab* slab = Metaslab::get_slab(p);
auto slab = Metaslab::get_slab(p_auth);
check_client(
sizeclass == super->get_meta(slab).sizeclass(),
sizeclass == super->get_meta(slab)->sizeclass(),
"Claimed small deallocation with mismatching size class");
small_dealloc_checked_sizeclass(super, slab, p, sizeclass);
small_dealloc_checked_sizeclass(super, slab, p_auth, p_ret, sizeclass);
}
SNMALLOC_FAST_PATH void small_dealloc_checked_sizeclass(
Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass)
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBArena> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
{
check_client(
Metaslab::is_start_of_object(&Slab::get_meta(slab), p),
Metaslab::is_start_of_object(Slab::get_meta(slab), address_cast(p_ret)),
"Not deallocating start of an object");
small_dealloc_start(super, slab, p, sizeclass);
small_dealloc_start(super, slab, p_auth, p_ret, sizeclass);
}
SNMALLOC_FAST_PATH void small_dealloc_start(
Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass)
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBArena> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
{
// TODO: with SSM/MTE, guard against double-frees
UNUSED(p_ret);
RemoteAllocator* target = super->get_allocator();
if (likely(target == public_state()))
{
void* offseted = apply_cache_friendly_offset(p, sizeclass);
small_dealloc_offseted(super, slab, offseted, sizeclass);
void* offseted =
apply_cache_friendly_offset(p_auth.unsafe_capptr, sizeclass);
small_dealloc_offseted(
super,
slab,
FreeObject::make(CapPtr<void, CBArena>(offseted)),
sizeclass);
}
else
remote_dealloc(target, p, sizeclass);
remote_dealloc(target, p_auth, sizeclass);
}
SNMALLOC_FAST_PATH void small_dealloc_offseted(
Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass)
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBArena> slab,
CapPtr<FreeObject, CBArena> p,
sizeclass_t sizeclass)
{
stats().sizeclass_dealloc(sizeclass);
@@ -1219,7 +1277,10 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH void small_dealloc_offseted_inner(
Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass)
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBArena> slab,
CapPtr<FreeObject, CBArena> p,
sizeclass_t sizeclass)
{
if (likely(Slab::dealloc_fast(slab, super, p, entropy)))
return;
@@ -1228,7 +1289,10 @@ namespace snmalloc
}
SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow(
Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass)
CapPtr<Superslab, CBArena> super,
CapPtr<Slab, CBArena> slab,
CapPtr<FreeObject, CBArena> p,
sizeclass_t sizeclass)
{
bool was_full = super->is_full();
SlabList* sl = &small_classes[sizeclass];
@@ -1252,28 +1316,29 @@ namespace snmalloc
{
if (was_full)
{
super_available.insert(super);
super_available.insert(super.unsafe_capptr);
}
else
{
super_only_short_available.remove(super);
super_available.insert(super);
super_only_short_available.remove(super.unsafe_capptr);
super_available.insert(super.unsafe_capptr);
}
break;
}
case Superslab::OnlyShortSlabAvailable:
{
super_only_short_available.insert(super);
super_only_short_available.insert(super.unsafe_capptr);
break;
}
case Superslab::Empty:
{
super_available.remove(super);
super_available.remove(super.unsafe_capptr);
chunkmap().clear_slab(super);
large_allocator.dealloc(super, 0);
large_allocator.dealloc(
super.template as_reinterpret<Largeslab>(), 0);
stats().superslab_push();
break;
}
@@ -1285,9 +1350,9 @@ namespace snmalloc
{
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
DLList<Mediumslab>* sc = &medium_classes[medium_class];
Mediumslab* slab = sc->get_head();
void* p;
auto sc = &medium_classes[medium_class];
auto slab = sc->get_head();
CapPtr<void, CBArena> p;
if (slab != nullptr)
{
@@ -1306,9 +1371,9 @@ namespace snmalloc
sizeclass, rsize, size);
});
}
slab =
reinterpret_cast<Mediumslab*>(large_allocator.template alloc<NoZero>(
0, SUPERSLAB_SIZE, SUPERSLAB_SIZE));
slab = CapPtr<void, CBArena>(large_allocator.template alloc<NoZero>(
0, SUPERSLAB_SIZE, SUPERSLAB_SIZE))
.template as_static<Mediumslab>();
if (slab == nullptr)
return nullptr;
@@ -1324,58 +1389,75 @@ namespace snmalloc
stats().alloc_request(size);
stats().sizeclass_alloc(sizeclass);
return p;
return p.unsafe_capptr;
}
SNMALLOC_FAST_PATH
void
medium_dealloc_unchecked(Mediumslab* slab, void* p, sizeclass_t sizeclass)
void medium_dealloc_unchecked(
CapPtr<Mediumslab, CBArena> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
{
check_client(
chunkmap().get(address_cast(p)) == CMMediumslab,
chunkmap().get(address_cast(p_ret)) == CMMediumslab,
"Claimed medium deallocation is not in a Mediumslab");
medium_dealloc_checked_chunkmap(slab, p, sizeclass);
medium_dealloc_checked_chunkmap(slab, p_auth, p_ret, sizeclass);
}
SNMALLOC_FAST_PATH
void medium_dealloc_checked_chunkmap(
Mediumslab* slab, void* p, sizeclass_t sizeclass)
CapPtr<Mediumslab, CBArena> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
{
check_client(
slab->get_sizeclass() == sizeclass,
"Claimed medium deallocation of the wrong sizeclass");
medium_dealloc_checked_sizeclass(slab, p, sizeclass);
medium_dealloc_checked_sizeclass(slab, p_auth, p_ret, sizeclass);
}
SNMALLOC_FAST_PATH
void medium_dealloc_checked_sizeclass(
Mediumslab* slab, void* p, sizeclass_t sizeclass)
CapPtr<Mediumslab, CBArena> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
{
check_client(
is_multiple_of_sizeclass(
sizeclass, pointer_diff(p, pointer_offset(slab, SUPERSLAB_SIZE))),
sizeclass, address_cast(slab) + SUPERSLAB_SIZE - address_cast(p_ret)),
"Not deallocating start of an object");
medium_dealloc_start(slab, p, sizeclass);
medium_dealloc_start(slab, p_auth, p_ret, sizeclass);
}
SNMALLOC_FAST_PATH
void medium_dealloc_start(Mediumslab* slab, void* p, sizeclass_t sizeclass)
void medium_dealloc_start(
CapPtr<Mediumslab, CBArena> slab,
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
sizeclass_t sizeclass)
{
// TODO: with SSM/MTE, guard against double-frees
UNUSED(p_ret);
RemoteAllocator* target = slab->get_allocator();
if (likely(target == public_state()))
medium_dealloc_local(slab, p, sizeclass);
medium_dealloc_local(slab, p_auth, sizeclass);
else
remote_dealloc(target, p, sizeclass);
remote_dealloc(target, p_auth, sizeclass);
}
SNMALLOC_FAST_PATH
void medium_dealloc_local(Mediumslab* slab, void* p, sizeclass_t sizeclass)
void medium_dealloc_local(
CapPtr<Mediumslab, CBArena> slab,
CapPtr<void, CBArena> p,
sizeclass_t sizeclass)
{
stats().sizeclass_dealloc(sizeclass);
bool was_full = Mediumslab::dealloc(slab, p);
@@ -1385,18 +1467,18 @@ namespace snmalloc
if (!was_full)
{
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
DLList<Mediumslab>* sc = &medium_classes[medium_class];
auto sc = &medium_classes[medium_class];
sc->remove(slab);
}
chunkmap().clear_slab(slab);
large_allocator.dealloc(slab, 0);
large_allocator.dealloc(slab.template as_reinterpret<Largeslab>(), 0);
stats().superslab_push();
}
else if (was_full)
{
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
DLList<Mediumslab>* sc = &medium_classes[medium_class];
auto sc = &medium_classes[medium_class];
sc->insert(slab);
}
}
@@ -1433,7 +1515,8 @@ namespace snmalloc
return p;
}
void large_dealloc_unchecked(void* p, size_t size)
void large_dealloc_unchecked(
CapPtr<void, CBArena> p_auth, CapPtr<void, CBAllocE> p_ret, size_t size)
{
uint8_t claimed_chunkmap_slab_kind =
static_cast<uint8_t>(bits::next_pow2_bits(size));
@@ -1442,48 +1525,57 @@ namespace snmalloc
// we're so far from the start that our actual chunkmap slab kind is not a
// legitimate large class
check_client(
chunkmap().get(address_cast(p)) == claimed_chunkmap_slab_kind,
chunkmap().get(address_cast(p_ret)) == claimed_chunkmap_slab_kind,
"Claimed large deallocation with wrong size class");
// round up as we would if we had had to look up the chunkmap_slab_kind
size_t rsize = bits::one_at_bit(claimed_chunkmap_slab_kind);
large_dealloc_checked_sizeclass(p, rsize, claimed_chunkmap_slab_kind);
large_dealloc_checked_sizeclass(
p_auth, p_ret, rsize, claimed_chunkmap_slab_kind);
}
void large_dealloc_checked_sizeclass(
void* p, size_t size, uint8_t chunkmap_slab_kind)
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
size_t size,
uint8_t chunkmap_slab_kind)
{
check_client(
address_cast(Superslab::get(p)) == address_cast(p),
address_cast(Superslab::get(p_auth)) == address_cast(p_ret),
"Not deallocating start of an object");
SNMALLOC_ASSERT(bits::one_at_bit(chunkmap_slab_kind) >= SUPERSLAB_SIZE);
large_dealloc_start(p, size, chunkmap_slab_kind);
large_dealloc_start(p_auth, p_ret, size, chunkmap_slab_kind);
}
void large_dealloc_start(void* p, size_t size, uint8_t chunkmap_slab_kind)
void large_dealloc_start(
CapPtr<void, CBArena> p_auth,
CapPtr<void, CBAllocE> p_ret,
size_t size,
uint8_t chunkmap_slab_kind)
{
// TODO: with SSM/MTE, guard against double-frees
if (NeedsInitialisation(this))
{
InitThreadAllocator([p, size, chunkmap_slab_kind](void* alloc) {
reinterpret_cast<Allocator*>(alloc)->large_dealloc_start(
p, size, chunkmap_slab_kind);
return nullptr;
});
InitThreadAllocator(
[p_auth, p_ret, size, chunkmap_slab_kind](void* alloc) {
reinterpret_cast<Allocator*>(alloc)->large_dealloc_start(
p_auth, p_ret, size, chunkmap_slab_kind);
return nullptr;
});
return;
}
size_t large_class = chunkmap_slab_kind - SUPERSLAB_BITS;
chunkmap().clear_large_size(p, size);
chunkmap().clear_large_size(p_auth, size);
stats().large_dealloc(large_class);
// Initialise in order to set the correct SlabKind.
Largeslab* slab = static_cast<Largeslab*>(p);
auto slab = p_auth.template as_static<Largeslab>();
slab->init();
large_allocator.dealloc(slab, large_class);
}
@@ -1492,7 +1584,8 @@ namespace snmalloc
// called in its slow path. This leads to one fewer unconditional jump in
// Clang.
SNMALLOC_FAST_PATH
void remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass)
void remote_dealloc(
RemoteAllocator* target, CapPtr<void, CBArena> p, sizeclass_t sizeclass)
{
SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id());
@@ -1501,7 +1594,8 @@ namespace snmalloc
// this path.
if (remote.capacity > 0)
{
void* offseted = apply_cache_friendly_offset(p, sizeclass);
void* offseted =
apply_cache_friendly_offset(p.unsafe_capptr, sizeclass);
stats().remote_free(sizeclass);
remote.dealloc(target->trunc_id(), offseted, sizeclass);
return;
@@ -1510,8 +1604,10 @@ namespace snmalloc
remote_dealloc_slow(target, p, sizeclass);
}
SNMALLOC_SLOW_PATH void
remote_dealloc_slow(RemoteAllocator* target, void* p, sizeclass_t sizeclass)
SNMALLOC_SLOW_PATH void remote_dealloc_slow(
RemoteAllocator* target,
CapPtr<void, CBArena> p_auth,
sizeclass_t sizeclass)
{
SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id());
@@ -1520,24 +1616,26 @@ namespace snmalloc
// a real allocator and construct one if we aren't.
if (NeedsInitialisation(this))
{
InitThreadAllocator([target, p, sizeclass](void* alloc) {
InitThreadAllocator([target, p_auth, sizeclass](void* alloc) {
reinterpret_cast<Allocator*>(alloc)->dealloc_not_large(
target, p, sizeclass);
target, p_auth, sizeclass);
return nullptr;
});
return;
}
remote_dealloc_and_post(target, p, sizeclass);
remote_dealloc_and_post(target, p_auth, sizeclass);
}
SNMALLOC_SLOW_PATH void remote_dealloc_and_post(
RemoteAllocator* target, void* offseted, sizeclass_t sizeclass)
RemoteAllocator* target,
CapPtr<void, CBArena> offseted,
sizeclass_t sizeclass)
{
handle_message_queue();
stats().remote_free(sizeclass);
remote.dealloc(target->trunc_id(), offseted, sizeclass);
remote.dealloc(target->trunc_id(), offseted.unsafe_capptr, sizeclass);
stats().remote_post();
remote.post(&large_allocator, get_trunc_id());

36
src/mem/chunkmap.h
View File

@@ -114,44 +114,36 @@ namespace snmalloc
return PagemapProvider::pagemap().get(p);
}
/**
* Get the pagemap entry corresponding to a specific address.
*/
static uint8_t get(void* p)
{
return get(address_cast(p));
}
/**
* Set a pagemap entry indicating that there is a superslab at the
* specified index.
*/
static void set_slab(Superslab* slab)
static void set_slab(CapPtr<Superslab, CBArena> slab)
{
set(slab, static_cast<size_t>(CMSuperslab));
set(address_cast(slab), static_cast<size_t>(CMSuperslab));
}
/**
* Add a pagemap entry indicating that a medium slab has been allocated.
*/
static void set_slab(Mediumslab* slab)
static void set_slab(CapPtr<Mediumslab, CBArena> slab)
{
set(slab, static_cast<size_t>(CMMediumslab));
set(address_cast(slab), static_cast<size_t>(CMMediumslab));
}
/**
* Remove an entry from the pagemap corresponding to a superslab.
*/
static void clear_slab(Superslab* slab)
static void clear_slab(CapPtr<Superslab, CBArena> slab)
{
SNMALLOC_ASSERT(get(slab) == CMSuperslab);
set(slab, static_cast<size_t>(CMNotOurs));
SNMALLOC_ASSERT(get(address_cast(slab)) == CMSuperslab);
set(address_cast(slab), static_cast<size_t>(CMNotOurs));
}
/**
* Remove an entry corresponding to a medium slab.
*/
static void clear_slab(Mediumslab* slab)
static void clear_slab(CapPtr<Mediumslab, CBArena> slab)
{
SNMALLOC_ASSERT(get(slab) == CMMediumslab);
set(slab, static_cast<size_t>(CMNotOurs));
SNMALLOC_ASSERT(get(address_cast(slab)) == CMMediumslab);
set(address_cast(slab), static_cast<size_t>(CMNotOurs));
}
/**
* Update the pagemap to reflect a large allocation, of `size` bytes from
@@ -160,7 +152,7 @@ namespace snmalloc
static void set_large_size(void* p, size_t size)
{
size_t size_bits = bits::next_pow2_bits(size);
set(p, static_cast<uint8_t>(size_bits));
set(address_cast(p), static_cast<uint8_t>(size_bits));
// Set redirect slide
auto ss = address_cast(p) + SUPERSLAB_SIZE;
for (size_t i = 0; i < size_bits - SUPERSLAB_BITS; i++)
@@ -175,7 +167,7 @@ namespace snmalloc
* Update the pagemap to remove a large allocation, of `size` bytes from
* address `p`.
*/
static void clear_large_size(void* vp, size_t size)
static void clear_large_size(CapPtr<void, CBArena> vp, size_t size)
{
auto p = address_cast(vp);
size_t rounded_size = bits::next_pow2(size);
@@ -190,9 +182,9 @@ namespace snmalloc
* interface and exists to make it easy to reuse the code in the public
* methods in other pagemap adaptors.
*/
static void set(void* p, uint8_t x)
static void set(address_t p, uint8_t x)
{
PagemapProvider::pagemap().set(address_cast(p), x);
PagemapProvider::pagemap().set(p, x);
}
};

66
src/mem/freelist.h
View File

@@ -23,7 +23,8 @@ namespace snmalloc
* Used to turn a location into a key. This is currently
* just the slab address truncated to 16bits and offset by 1.
*/
inline static address_t initial_key(void* slab)
template<typename T, capptr_bounds B>
inline static address_t initial_key(CapPtr<T, B> slab)
{
#ifdef CHECK_CLIENT
/**
@@ -46,12 +47,15 @@ namespace snmalloc
return ((p1 ^ p2) >= SLAB_SIZE);
}
static inline bool different_slab(address_t p1, void* p2)
template<typename T>
static inline bool different_slab(address_t p1, CapPtr<T, CBArena> p2)
{
return different_slab(p1, address_cast(p2));
}
static inline bool different_slab(void* p1, void* p2)
template<typename T, typename U>
static inline bool
different_slab(CapPtr<T, CBArena> p1, CapPtr<U, CBArena> p2)
{
return different_slab(address_cast(p1), address_cast(p2));
}
@@ -60,7 +64,7 @@ namespace snmalloc
class EncodeFreeObjectReference
{
FreeObject* reference;
CapPtr<FreeObject, CBArena> reference;
/**
* On architectures which use IntegerPointers, we can obfuscate our free
@@ -78,8 +82,8 @@ namespace snmalloc
public:
#ifdef CHECK_CLIENT
template<typename T = FreeObject>
static std::enable_if_t<do_encode, T*>
encode(uint16_t local_key, T* next_object, LocalEntropy& entropy)
static std::enable_if_t<do_encode, CapPtr<T, CBArena>> encode(
uint16_t local_key, CapPtr<T, CBArena> next_object, LocalEntropy& entropy)
{
// Simple involutional encoding. The bottom half of each word is
// multiplied by a function of both global and local keys (the latter,
@@ -92,25 +96,28 @@ namespace snmalloc
address_t key = (local_key + 1) * entropy.get_constant_key();
next ^= (((next & MASK) + 1) * key) &
~(bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1);
return reinterpret_cast<FreeObject*>(next);
return CapPtr<T, CBArena>(reinterpret_cast<T*>(next));
}
#endif
template<typename T = FreeObject>
static std::enable_if_t<!do_encode, T*>
encode(uint16_t local_key, T* next_object, LocalEntropy& entropy)
static std::enable_if_t<!do_encode, CapPtr<T, CBArena>> encode(
uint16_t local_key, CapPtr<T, CBArena> next_object, LocalEntropy& entropy)
{
UNUSED(local_key);
UNUSED(entropy);
return next_object;
}
void store(FreeObject* value, uint16_t local_key, LocalEntropy& entropy)
void store(
CapPtr<FreeObject, CBArena> value,
uint16_t local_key,
LocalEntropy& entropy)
{
reference = encode(local_key, value, entropy);
}
FreeObject* read(uint16_t local_key, LocalEntropy& entropy)
CapPtr<FreeObject, CBArena> read(uint16_t local_key, LocalEntropy& entropy)
{
return encode(local_key, reference, entropy);
}
@@ -129,15 +136,15 @@ namespace snmalloc
public:
EncodeFreeObjectReference next_object;
static FreeObject* make(void* p)
static CapPtr<FreeObject, CBArena> make(CapPtr<void, CBArena> p)
{
return static_cast<FreeObject*>(p);
return p.template as_static<FreeObject>();
}
/**
* Read the next pointer handling any required decoding of the pointer
*/
FreeObject* read_next(uint16_t key, LocalEntropy& entropy)
CapPtr<FreeObject, CBArena> read_next(uint16_t key, LocalEntropy& entropy)
{
return next_object.read(key, entropy);
}
@@ -150,7 +157,7 @@ namespace snmalloc
*/
class FreeListIter
{
FreeObject* curr = nullptr;
CapPtr<FreeObject, CBArena> curr = nullptr;
#ifdef CHECK_CLIENT
address_t prev = 0;
#endif
@@ -170,7 +177,7 @@ namespace snmalloc
* Currently this is just the value of current before this call.
* Other schemes could be used.
*/
void update_cursor(FreeObject* next)
void update_cursor(CapPtr<FreeObject, CBArena> next)
{
#ifdef CHECK_CLIENT
# ifndef NDEBUG
@@ -187,7 +194,7 @@ namespace snmalloc
}
public:
FreeListIter(FreeObject* head)
FreeListIter(CapPtr<FreeObject, CBArena> head)
: curr(head)
#ifdef CHECK_CLIENT
,
@@ -210,7 +217,7 @@ namespace snmalloc
/**
* Returns current head without affecting the iterator.
*/
void* peek()
CapPtr<FreeObject, CBArena> peek()
{
return curr;
}
@@ -218,7 +225,7 @@ namespace snmalloc
/**
* Moves the iterator on, and returns the current value.
*/
void* take(LocalEntropy& entropy)
CapPtr<FreeObject, CBArena> take(LocalEntropy& entropy)
{
#ifdef CHECK_CLIENT
check_client(
@@ -309,7 +316,7 @@ namespace snmalloc
* Start building a new free list.
* Provide pointer to the slab to initialise the system.
*/
void open(void* p)
void open(CapPtr<void, CBArena> p)
{
SNMALLOC_ASSERT(empty());
for (size_t i = 0; i < LENGTH; i++)
@@ -331,17 +338,17 @@ namespace snmalloc
{
for (size_t i = 0; i < LENGTH; i++)
{
if (end[i] != &head[i])
if (address_cast(end[i]) != address_cast(&head[i]))
return false;
}
return true;
}
bool debug_different_slab(void* n)
bool debug_different_slab(CapPtr<FreeObject, CBArena> n)
{
for (size_t i = 0; i < LENGTH; i++)
{
if (!different_slab(end[i], n))
if (!different_slab(address_cast(end[i]), n))
return false;
}
return true;
@@ -350,18 +357,17 @@ namespace snmalloc
/**
* Adds an element to the builder
*/
void add(void* n, LocalEntropy& entropy)
void add(CapPtr<FreeObject, CBArena> n, LocalEntropy& entropy)
{
SNMALLOC_ASSERT(!debug_different_slab(n) || empty());
FreeObject* next = FreeObject::make(n);
auto index = RANDOM ? entropy.next_bit() : 0;
end[index]->store(next, get_prev(index), entropy);
end[index] = &(next->next_object);
end[index]->store(n, get_prev(index), entropy);
end[index] = &(n->next_object);
#ifdef CHECK_CLIENT
prev[index] = curr[index];
curr[index] = address_cast(next) & 0xffff;
curr[index] = address_cast(n) & 0xffff;
#endif
}
@@ -378,11 +384,11 @@ namespace snmalloc
{
uint16_t local_prev = HEAD_KEY;
EncodeFreeObjectReference* iter = &head[i];
FreeObject* prev_obj = iter->read(local_prev, entropy);
CapPtr<FreeObject, CBArena> prev_obj = iter->read(local_prev, entropy);
uint16_t local_curr = initial_key(prev_obj) & 0xffff;
while (end[i] != iter)
{
FreeObject* next = iter->read(local_prev, entropy);
CapPtr<FreeObject, CBArena> next = iter->read(local_prev, entropy);
check_client(!different_slab(next, prev_obj), "Heap corruption");
local_prev = local_curr;
local_curr = address_cast(next) & 0xffff;

10
src/mem/largealloc.h
View File

@@ -115,11 +115,11 @@ namespace snmalloc
* class specified by `large_class`. Always succeeds.
*/
SNMALLOC_FAST_PATH void
push_large_stack(Largeslab* slab, size_t large_class)
push_large_stack(CapPtr<Largeslab, CBArena> slab, size_t large_class)
{
const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
available_large_chunks_in_bytes += rsize;
large_stack[large_class].push(slab);
large_stack[large_class].push(slab.unsafe_capptr);
}
/**
@@ -342,7 +342,7 @@ namespace snmalloc
return p;
}
void dealloc(void* p, size_t large_class)
void dealloc(CapPtr<Largeslab, CBArena> p, size_t large_class)
{
if constexpr (decommit_strategy == DecommitSuperLazy)
{
@@ -360,11 +360,11 @@ namespace snmalloc
(large_class != 0 || decommit_strategy == DecommitSuper))
{
MemoryProvider::Pal::notify_not_using(
pointer_offset(p, OS_PAGE_SIZE), rsize - OS_PAGE_SIZE);
pointer_offset(p, OS_PAGE_SIZE).unsafe_capptr, rsize - OS_PAGE_SIZE);
}
stats.superslab_push();
memory_provider.push_large_stack(static_cast<Largeslab*>(p), large_class);
memory_provider.push_large_stack(p, large_class);
}
};

37
src/mem/mediumslab.h
View File

@@ -12,12 +12,12 @@ namespace snmalloc
// This is the view of a 16 mb area when it is being used to allocate
// medium sized classes: 64 kb to 16 mb, non-inclusive.
private:
friend DLList<Mediumslab>;
friend DLList<Mediumslab, CapPtrCBArena>;
// Keep the allocator pointer on a separate cache line. It is read by
// other threads, and does not change, so we avoid false sharing.
alignas(CACHELINE_SIZE) Mediumslab* next;
Mediumslab* prev;
alignas(CACHELINE_SIZE) CapPtr<Mediumslab, CBArena> next;
CapPtr<Mediumslab, CBArena> prev;
uint16_t free;
uint8_t head;
@@ -44,16 +44,19 @@ namespace snmalloc
return bits::align_up(sizeof(Mediumslab), min(OS_PAGE_SIZE, SLAB_SIZE));
}
static Mediumslab* get(const void* p)
template<typename T, capptr_bounds B>
static SNMALLOC_FAST_PATH CapPtr<Mediumslab, B> get(CapPtr<T, B> p)
{
return pointer_align_down<SUPERSLAB_SIZE, Mediumslab>(
const_cast<void*>(p));
static_assert(B == CBArena || B == CBChunk);
return pointer_align_down<SUPERSLAB_SIZE, Mediumslab>(p);
}
// This is pre-factored to take an explicit self parameter so that we can
// eventually annotate that pointer with additional information.
static void
init(Mediumslab* self, RemoteAllocator* alloc, sizeclass_t sc, size_t rsize)
static void init(
CapPtr<Mediumslab, CBArena> self,
RemoteAllocator* alloc,
sizeclass_t sc,
size_t rsize)
{
SNMALLOC_ASSERT(sc >= NUM_SMALL_CLASSES);
SNMALLOC_ASSERT((sc - NUM_SMALL_CLASSES) < NUM_MEDIUM_CLASSES);
@@ -85,12 +88,13 @@ namespace snmalloc
}
template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
static void* alloc(Mediumslab* self, size_t size)
static CapPtr<void, CBArena>
alloc(CapPtr<Mediumslab, CBArena> self, size_t size)
{
SNMALLOC_ASSERT(!full(self));
uint16_t index = self->stack[self->head++];
void* p = pointer_offset(self, (static_cast<size_t>(index) << 8));
auto p = pointer_offset(self, (static_cast<size_t>(index) << 8));
self->free--;
if constexpr (zero_mem == YesZero)
@@ -101,7 +105,8 @@ namespace snmalloc
return p;
}
static bool dealloc(Mediumslab* self, void* p)
static bool
dealloc(CapPtr<Mediumslab, CBArena> self, CapPtr<void, CBArena> p)
{
SNMALLOC_ASSERT(self->head > 0);
@@ -113,12 +118,14 @@ namespace snmalloc
return was_full;
}
static bool full(Mediumslab* self)
template<capptr_bounds B>
static bool full(CapPtr<Mediumslab, B> self)
{
return self->free == 0;
}
static bool empty(Mediumslab* self)
template<capptr_bounds B>
static bool empty(CapPtr<Mediumslab, B> self)
{
return self->head == 0;
}

42
src/mem/metaslab.h
View File

@@ -10,8 +10,8 @@ namespace snmalloc
{
class Slab;
using SlabList = CDLLNode<>;
using SlabLink = CDLLNode<>;
using SlabList = CDLLNode<CapPtrCBArena>;
using SlabLink = CDLLNode<CapPtrCBArena>;
static_assert(
sizeof(SlabLink) <= MIN_ALLOC_SIZE,
@@ -71,7 +71,7 @@ namespace snmalloc
return free_queue.s.next;
}
void initialise(sizeclass_t sizeclass, Slab* slab)
void initialise(sizeclass_t sizeclass, CapPtr<Slab, CBArena> slab)
{
free_queue.s.sizeclass = static_cast<uint8_t>(sizeclass);
free_queue.init();
@@ -120,12 +120,12 @@ namespace snmalloc
return bits::min(threshold, max);
}
SNMALLOC_FAST_PATH void set_full(Slab* slab)
SNMALLOC_FAST_PATH void set_full(CapPtr<Slab, CBArena> slab)
{
SNMALLOC_ASSERT(free_queue.empty());
// Prepare for the next free queue to be built.
free_queue.open(slab);
free_queue.open(slab.as_void());
// Set needed to at least one, possibly more so we only use
// a slab when it has a reasonable amount of free elements
@@ -133,34 +133,33 @@ namespace snmalloc
null_prev();
}
static Slab* get_slab(const void* p)
template<typename T, capptr_bounds B>
static CapPtr<Slab, B> get_slab(CapPtr<T, B> p)
{
return pointer_align_down<SLAB_SIZE, Slab>(const_cast<void*>(p));
return pointer_align_down<SLAB_SIZE, Slab>(p.as_void());
}
static bool is_short(Slab* p)
template<capptr_bounds B>
static bool is_short(CapPtr<Slab, B> p)
{
return pointer_align_down<SUPERSLAB_SIZE>(p) == p;
return pointer_align_down<SUPERSLAB_SIZE, Slab>(p.as_void()) == p;
}
SNMALLOC_FAST_PATH static bool is_start_of_object(Metaslab* self, void* p)
SNMALLOC_FAST_PATH
static bool is_start_of_object(CapPtr<Metaslab, CBArena> self, address_t p)
{
return is_multiple_of_sizeclass(
self->sizeclass(),
SLAB_SIZE - pointer_diff(pointer_align_down<SLAB_SIZE>(p), p));
self->sizeclass(), SLAB_SIZE - (p - address_align_down<SLAB_SIZE>(p)));
}
/**
* Takes a free list out of a slabs meta data.
* Returns the link as the allocation, and places the free list into the
* `fast_free_list` for further allocations.
*
* This is pre-factored to take an explicit self parameter so that we can
* eventually annotate that pointer with additional information.
*/
template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
static SNMALLOC_FAST_PATH void* alloc(
Metaslab* self,
CapPtr<Metaslab, CBArena> self,
FreeListIter& fast_free_list,
size_t rsize,
LocalEntropy& entropy)
@@ -169,7 +168,7 @@ namespace snmalloc
SNMALLOC_ASSERT(!self->is_full());
self->free_queue.close(fast_free_list, entropy);
void* n = fast_free_list.take(entropy);
auto n = fast_free_list.take(entropy);
entropy.refresh_bits();
@@ -177,10 +176,11 @@ namespace snmalloc
self->remove();
self->set_full(Metaslab::get_slab(n));
void* p = remove_cache_friendly_offset(n, self->sizeclass());
SNMALLOC_ASSERT(is_start_of_object(self, p));
void* p =
remove_cache_friendly_offset(n.unsafe_capptr, self->sizeclass());
SNMALLOC_ASSERT(is_start_of_object(self, address_cast(p)));
self->debug_slab_invariant(Metaslab::get_slab(p), entropy);
self->debug_slab_invariant(Metaslab::get_slab(n), entropy);
if constexpr (zero_mem == YesZero)
{
@@ -197,7 +197,7 @@ namespace snmalloc
return p;
}
void debug_slab_invariant(Slab* slab, LocalEntropy& entropy)
void debug_slab_invariant(CapPtr<Slab, CBArena> slab, LocalEntropy& entropy)
{
#if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS)
bool is_short = Metaslab::is_short(slab);

86
src/mem/slab.h
View File

@@ -17,9 +17,12 @@ namespace snmalloc
}
public:
static Metaslab& get_meta(Slab* self)
template<capptr_bounds B>
static CapPtr<Metaslab, B> get_meta(CapPtr<Slab, B> self)
{
Superslab* super = Superslab::get(self);
static_assert(B == CBArena || B == CBChunk);
auto super = Superslab::get(self);
return super->get_meta(self);
}
@@ -30,12 +33,12 @@ namespace snmalloc
* page.
*/
static SNMALLOC_FAST_PATH void alloc_new_list(
void*& bumpptr,
CapPtr<void, CBArena>& bumpptr,
FreeListIter& fast_free_list,
size_t rsize,
LocalEntropy& entropy)
{
void* slab_end = pointer_align_up<SLAB_SIZE>(pointer_offset(bumpptr, 1));
auto slab_end = pointer_align_up<SLAB_SIZE>(pointer_offset(bumpptr, 1));
FreeListBuilder<false> b;
SNMALLOC_ASSERT(b.empty());
@@ -46,23 +49,27 @@ namespace snmalloc
// Structure to represent the temporary list elements
struct PreAllocObject
{
PreAllocObject* next;
CapPtr<PreAllocObject, CBArena> next;
};
// The following code implements Sattolo's algorithm for generating
// random cyclic permutations. This implementation is in the opposite
// direction, so that the original space does not need initialising. This
// is described as outside-in without citation on Wikipedia, appears to be
// Folklore algorithm.
PreAllocObject* curr = pointer_offset<PreAllocObject>(bumpptr, 0);
auto curr =
pointer_offset(bumpptr, 0).template as_static<PreAllocObject>();
curr->next = curr;
uint16_t count = 1;
for (PreAllocObject* p = pointer_offset<PreAllocObject>(bumpptr, rsize);
p < slab_end;
p = pointer_offset<PreAllocObject>(p, rsize))
for (auto p = pointer_offset(bumpptr, rsize)
.template as_static<PreAllocObject>();
p.as_void() < slab_end;
p = pointer_offset(p, rsize).template as_static<PreAllocObject>())
{
size_t insert_index = entropy.sample(count);
p->next = std::exchange(
pointer_offset<PreAllocObject>(bumpptr, insert_index * rsize)->next,
pointer_offset(bumpptr, insert_index * rsize)
.template as_static<PreAllocObject>()
->next,
p);
count++;
}
@@ -70,18 +77,18 @@ namespace snmalloc
// Pick entry into space, and then build linked list by traversing cycle
// to the start.
auto start_index = entropy.sample(count);
auto start_ptr =
pointer_offset<PreAllocObject>(bumpptr, start_index * rsize);
auto start_ptr = pointer_offset(bumpptr, start_index * rsize)
.template as_static<PreAllocObject>();
auto curr_ptr = start_ptr;
do
{
b.add(curr_ptr, entropy);
b.add(FreeObject::make(curr_ptr.as_void()), entropy);
curr_ptr = curr_ptr->next;
} while (curr_ptr != start_ptr);
#else
for (void* p = bumpptr; p < slab_end; p = pointer_offset<void>(p, rsize))
for (auto p = bumpptr; p < slab_end; p = pointer_offset(p, rsize))
{
b.add(p, entropy);
b.add(FreeObject::make(p.as_void()), entropy);
}
#endif
// This code consumes everything up to slab_end.
@@ -94,20 +101,20 @@ namespace snmalloc
// Returns true, if it deallocation can proceed without changing any status
// bits. Note that this does remove the use from the meta slab, so it
// doesn't need doing on the slow path.
//
// This is pre-factored to take an explicit self parameter so that we can
// eventually annotate that pointer with additional information.
static SNMALLOC_FAST_PATH bool
dealloc_fast(Slab* self, Superslab* super, void* p, LocalEntropy& entropy)
static SNMALLOC_FAST_PATH bool dealloc_fast(
CapPtr<Slab, CBArena> self,
CapPtr<Superslab, CBArena> super,
CapPtr<FreeObject, CBArena> p,
LocalEntropy& entropy)
{
Metaslab& meta = super->get_meta(self);
SNMALLOC_ASSERT(!meta.is_unused());
auto meta = super->get_meta(self);
SNMALLOC_ASSERT(!meta->is_unused());
if (unlikely(meta.return_object()))
if (unlikely(meta->return_object()))
return false;
// Update the head and the next pointer in the free list.
meta.free_queue.add(p, entropy);
meta->free_queue.add(p, entropy);
return true;
}
@@ -116,23 +123,20 @@ namespace snmalloc
// This does not need to remove the "use" as done by the fast path.
// Returns a complex return code for managing the superslab meta data.
// i.e. This deallocation could make an entire superslab free.
//
// This is pre-factored to take an explicit self parameter so that we can
// eventually annotate that pointer with additional information.
static SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow(
Slab* self,
CapPtr<Slab, CBArena> self,
SlabList* sl,
Superslab* super,
void* p,
CapPtr<Superslab, CBArena> super,
CapPtr<FreeObject, CBArena> p,
LocalEntropy& entropy)
{
Metaslab& meta = super->get_meta(self);
meta.debug_slab_invariant(self, entropy);
auto meta = super->get_meta(self);
meta->debug_slab_invariant(self, entropy);
if (meta.is_full())
if (meta->is_full())
{
auto allocated = get_slab_capacity(
meta.sizeclass(), Metaslab::is_short(Metaslab::get_slab(p)));
meta->sizeclass(), Metaslab::is_short(Metaslab::get_slab(p)));
// We are not on the sizeclass list.
if (allocated == 1)
{
@@ -143,15 +147,15 @@ namespace snmalloc
return super->dealloc_slab(self);
}
meta.free_queue.add(p, entropy);
meta->free_queue.add(p, entropy);
// Remove trigger threshold from how many we need before we have fully
// freed the slab.
meta.needed() =
allocated - meta.threshold_for_waking_slab(Metaslab::is_short(self));
meta->needed() =
allocated - meta->threshold_for_waking_slab(Metaslab::is_short(self));
// Push on the list of slabs for this sizeclass.
sl->insert_prev(&meta);
meta.debug_slab_invariant(self, entropy);
sl->insert_prev(meta.template as_static<SlabLink>());
meta->debug_slab_invariant(self, entropy);
return Superslab::NoSlabReturn;
}
@@ -160,7 +164,7 @@ namespace snmalloc
// Check free list is well-formed on platforms with
// integers as pointers.
FreeListIter fl;
meta.free_queue.close(fl, entropy);
meta->free_queue.close(fl, entropy);
while (!fl.empty())
{
@@ -169,7 +173,7 @@ namespace snmalloc
}
#endif
meta.remove();
meta->remove();
if (Metaslab::is_short(self))
return super->dealloc_short_slab();

27
src/mem/superslab.h
View File

@@ -59,9 +59,10 @@ namespace snmalloc
ModArray<SLAB_COUNT, Metaslab> meta;
// Used size_t as results in better code in MSVC
size_t slab_to_index(Slab* slab)
template<capptr_bounds B>
size_t slab_to_index(CapPtr<Slab, B> slab)
{
auto res = (pointer_diff(this, slab) >> SLAB_BITS);
auto res = (pointer_diff(this, slab.unsafe_capptr) >> SLAB_BITS);
SNMALLOC_ASSERT(res == static_cast<uint8_t>(res));
return static_cast<uint8_t>(res);
}
@@ -82,10 +83,12 @@ namespace snmalloc
StatusChange = 2
};
static Superslab* get(const void* p)
template<typename T, capptr_bounds B>
static SNMALLOC_FAST_PATH CapPtr<Superslab, B> get(CapPtr<T, B> p)
{
return pointer_align_down<SUPERSLAB_SIZE, Superslab>(
const_cast<void*>(p));
static_assert(B == CBArena || B == CBChunk);
return pointer_align_down<SUPERSLAB_SIZE, Superslab>(p.as_void());
}
static bool is_short_sizeclass(sizeclass_t sizeclass)
@@ -180,9 +183,10 @@ namespace snmalloc
return Full;
}
Metaslab& get_meta(Slab* slab)
template<capptr_bounds B>
CapPtr<Metaslab, B> get_meta(CapPtr<Slab, B> slab)
{
return meta[slab_to_index(slab)];
return CapPtr<Metaslab, B>(&meta[slab_to_index(slab)]);
}
// This is pre-factored to take an explicit self parameter so that we can
@@ -195,7 +199,7 @@ namespace snmalloc
Slab* slab = reinterpret_cast<Slab*>(self);
auto& metaz = self->meta[0];
metaz.initialise(sizeclass, slab);
metaz.initialise(sizeclass, CapPtr<Slab, CBArena>(slab));
self->used++;
return slab;
@@ -212,7 +216,7 @@ namespace snmalloc
auto& metah = self->meta[h];
uint8_t n = metah.next();
metah.initialise(sizeclass, slab);
metah.initialise(sizeclass, CapPtr<Slab, CBArena>(slab));
self->head = h + n + 1;
self->used += 2;
@@ -221,8 +225,11 @@ namespace snmalloc
}
// Returns true, if this alters the value of get_status
Action dealloc_slab(Slab* slab)
template<capptr_bounds B>
Action dealloc_slab(CapPtr<Slab, B> slab)
{
static_assert(B == CBArena || B == CBChunk);
// This is not the short slab.
uint8_t index = static_cast<uint8_t>(slab_to_index(slab));
uint8_t n = head - index - 1;

17
src/pal/pal.h
View File

@@ -70,9 +70,22 @@ namespace snmalloc
static constexpr size_t OS_PAGE_SIZE = Pal::page_size;
/**
* A centralized, inlinable wrapper around PAL::zero. This will matter more
* when we introduce AuthPtr-s.
* A convenience wrapper that avoids the need to litter unsafe accesses with
* every call to PAL::zero.
*
* We do this here rather than plumb CapPtr further just to minimize
* disruption and avoid code bloat. This wrapper ought to compile down to
* nothing if SROA is doing its job.
*/
template<typename PAL, bool page_aligned = false, typename T, capptr_bounds B>
static SNMALLOC_FAST_PATH void pal_zero(CapPtr<T, B> p, size_t sz)
{
static_assert(
!page_aligned || B == CBArena || B == CBChunkD || B == CBChunk);
PAL::template zero<page_aligned>(p.unsafe_capptr, sz);
}
// TODO: Remove once CapPtr<>s plumbed everywhere they need to be
template<typename PAL, bool page_aligned = false>
static SNMALLOC_FAST_PATH void pal_zero(void* p, size_t sz)
{

2
src/test/func/sandbox/sandbox.cc
View File

@@ -86,7 +86,7 @@ namespace
*
* This method must be implemented for `LargeAlloc` to work.
*/
void push_large_stack(Largeslab* slab, size_t large_class)
void push_large_stack(CapPtr<Largeslab, CBArena> slab, size_t large_class)
{
real_state->push_large_stack(slab, large_class);
}