Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Free queues hold Wild pointers

Browse Source
This commit is contained in:
Nathaniel Wesley Filardo
2021-09-15 00:19:39 +01:00
committed by Nathaniel Wesley Filardo
parent 7750676598
commit d4c120dfe5
4 changed files with 109 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/mem/corealloc.h
View File

@@ -180,15 +180,19 @@ namespace snmalloc
void* small_alloc_one(size_t size)
{
SNMALLOC_ASSERT(attached_cache != nullptr);
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
auto domesticate =
[this](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(
backend_state_ptr(), p);
};
// Use attached cache, and fill it if it is empty.
return attached_cache->template alloc<NoZero, SharedStateHandle>(
domesticate,
size,
[&](
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>* fl) {
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>* fl) {
return small_alloc<NoZero>(sizeclass, *fl);
});
}
@@ -252,7 +256,8 @@ namespace snmalloc
do
{
b.add(
FreeObject::make<capptr::bounds::Alloc>(
// Here begins our treatment of the heap as containing Wild pointers
FreeObject::make<capptr::bounds::AllocWild>(
capptr_to_user_address_control(curr_ptr.as_void())),
key,
entropy);
@@ -263,7 +268,8 @@ namespace snmalloc
do
{
b.add(
FreeObject::make<capptr::bounds::Alloc>(
// Here begins our treatment of the heap as containing Wild pointers
FreeObject::make<capptr::bounds::AllocWild>(
capptr_to_user_address_control(
Aal::capptr_bound<void, capptr::bounds::AllocFull>(
p.as_void(), rsize))),
@@ -278,11 +284,15 @@ namespace snmalloc
ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass)
{
auto& key = entropy.get_free_list_key();
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc> fl;
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild> fl;
auto more = meta->free_queue.close(fl, key);
UNUSED(more);
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
auto local_state = backend_state_ptr();
auto domesticate =
[local_state](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass);
#ifdef SNMALLOC_CHECK_CLIENT
@@ -601,7 +611,7 @@ namespace snmalloc
Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)),
"Not deallocating start of an object");
auto cp = p.as_static<FreeObject::T<capptr::bounds::Alloc>>();
auto cp = p.as_static<FreeObject::T<capptr::bounds::AllocWild>>();
auto& key = entropy.get_free_list_key();
@@ -614,7 +624,7 @@ namespace snmalloc
template<ZeroMem zero_mem>
SNMALLOC_SLOW_PATH void* small_alloc(
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>&
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>&
fast_free_list)
{
size_t rsize = sizeclass_to_size(sizeclass);
@@ -639,8 +649,12 @@ namespace snmalloc
if (meta->needed() == 0)
alloc_classes[sizeclass].unused--;
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
auto domesticate =
[this](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(
backend_state_ptr(), p);
};
auto [p, still_active] = Metaslab::alloc_free_list(
domesticate, meta, fast_free_list, entropy, sizeclass);
@@ -676,7 +690,7 @@ namespace snmalloc
template<ZeroMem zero_mem>
SNMALLOC_SLOW_PATH void* small_alloc_slow(
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>&
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>&
fast_free_list,
size_t rsize)
{
@@ -708,8 +722,12 @@ namespace snmalloc
// Build a free list for the slab
alloc_new_list(slab, meta, rsize, slab_size, entropy);
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
auto domesticate =
[this](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(
backend_state_ptr(), p);
};
auto [p, still_active] = Metaslab::alloc_free_list(
domesticate, meta, fast_free_list, entropy, sizeclass);
@@ -745,7 +763,8 @@ namespace snmalloc
auto& entry = SharedStateHandle::Pagemap::get_metaentry(
backend_state_ptr(), snmalloc::address_cast(p));
handle_dealloc_remote(entry, p.as_void(), need_post);
// XXX n is not known to be domesticated
// TODO p = SharedStateHandle::capptr_domesticate(backend_state_ptr(),
// n);
p = n;
}
}

95
src/mem/localalloc.h
View File

@@ -211,35 +211,38 @@ namespace snmalloc
SNMALLOC_FAST_PATH void* small_alloc(size_t size)
{
// SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES));
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
auto domesticate =
[this](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(
core_alloc->backend_state_ptr(), p);
};
auto slowpath =
[&](
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>* fl)
SNMALLOC_FAST_PATH_LAMBDA {
if (likely(core_alloc != nullptr))
{
return core_alloc->handle_message_queue(
[](
CoreAlloc* core_alloc,
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>*
fl) {
return core_alloc->template small_alloc<zero_mem>(
sizeclass, *fl);
},
core_alloc,
sizeclass,
fl);
}
return lazy_init(
[&](CoreAlloc*, sizeclass_t sizeclass) {
return small_alloc<zero_mem>(sizeclass_to_size(sizeclass));
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>*
fl) SNMALLOC_FAST_PATH_LAMBDA {
if (likely(core_alloc != nullptr))
{
return core_alloc->handle_message_queue(
[](
CoreAlloc* core_alloc,
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>*
fl) {
return core_alloc->template small_alloc<zero_mem>(
sizeclass, *fl);
},
sizeclass);
};
core_alloc,
sizeclass,
fl);
}
return lazy_init(
[&](CoreAlloc*, sizeclass_t sizeclass) {
return small_alloc<zero_mem>(sizeclass_to_size(sizeclass));
},
sizeclass);
};
return local_cache.template alloc<zero_mem, SharedStateHandle>(
domesticate, size, slowpath);
@@ -278,8 +281,8 @@ namespace snmalloc
return;
}
// Recheck what kind of dealloc we should do incase, the allocator we get
// from lazy_init is the originating allocator.
// Recheck what kind of dealloc we should do incase, the allocator we
// get from lazy_init is the originating allocator.
lazy_init(
[&](CoreAlloc*, CapPtr<void, capptr::bounds::Alloc> p) {
dealloc(p.unsafe_ptr()); // TODO don't double count statistics
@@ -327,8 +330,8 @@ namespace snmalloc
{}
/**
* Call `SharedStateHandle::ensure_init()` if it is implemented, do nothing
* otherwise.
* Call `SharedStateHandle::ensure_init()` if it is implemented, do
* nothing otherwise.
*/
SNMALLOC_FAST_PATH
void ensure_init()
@@ -458,17 +461,17 @@ namespace snmalloc
// TODO:
// Care is needed so that dealloc(nullptr) works before init
// The backend allocator must ensure that a minimal page map exists
// before init, that maps null to a remote_deallocator that will never be
// in thread local state.
// before init, that maps null to a remote_deallocator that will never
// be in thread local state.
capptr::AllocWild<void> p_wild = capptr_from_client(p_raw);
/*
* p_tame may be nullptr, even if p_raw/p_wild are not, in the case where
* domestication fails. We exclusively use p_tame below so that such
* failures become no ops; in the nullptr path, which should be well off
* the fast path, we could be slightly more aggressive and test that p_raw
* is also nullptr and Pal::error() if not. (TODO)
* p_tame may be nullptr, even if p_raw/p_wild are not, in the case
* where domestication fails. We exclusively use p_tame below so that
* such failures become no ops; in the nullptr path, which should be
* well off the fast path, we could be slightly more aggressive and test
* that p_raw is also nullptr and Pal::error() if not. (TODO)
*
* We do not rely on the bounds-checking ability of domestication here,
* and just check the address (and, on other architectures, perhaps
@@ -589,17 +592,17 @@ namespace snmalloc
#ifdef SNMALLOC_PASS_THROUGH
return external_alloc::malloc_usable_size(const_cast<void*>(p_raw));
#else
// TODO What's the domestication policy here? At the moment we just probe
// the pagemap with the raw address, without checks. There could be
// implicit domestication through the `SharedStateHandle::Pagemap` or we
// could just leave well enough alone.
// TODO What's the domestication policy here? At the moment we just
// probe the pagemap with the raw address, without checks. There could
// be implicit domestication through the `SharedStateHandle::Pagemap` or
// we could just leave well enough alone.
// Note that this should return 0 for nullptr.
// Other than nullptr, we know the system will be initialised as it must
// be called with something we have already allocated.
// To handle this case we require the uninitialised pagemap contain an
// entry for the first chunk of memory, that states it represents a large
// object, so we can pull the check for null off the fast path.
// entry for the first chunk of memory, that states it represents a
// large object, so we can pull the check for null off the fast path.
MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry(
core_alloc->backend_state_ptr(), address_cast(p_raw));
@@ -625,10 +628,10 @@ namespace snmalloc
void* external_pointer(void* p_raw)
{
#ifndef SNMALLOC_PASS_THROUGH
// TODO What's the domestication policy here? At the moment we just probe
// the pagemap with the raw address, without checks. There could be
// implicit domestication through the `SharedStateHandle::Pagemap` or we
// could just leave well enough alone.
// TODO What's the domestication policy here? At the moment we just
// probe the pagemap with the raw address, without checks. There could
// be implicit domestication through the `SharedStateHandle::Pagemap` or
// we could just leave well enough alone.
// TODO bring back the CHERI bits. Wes to review if required.
MetaEntry entry =

21
src/mem/localcache.h
View File

@@ -12,9 +12,9 @@ namespace snmalloc
{
using Stats = AllocStats<NUM_SIZECLASSES, NUM_LARGE_CLASSES>;
template<SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero(
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc> p,
sizeclass_t sizeclass)
FreeObject::HeadPtr<capptr::bounds::Alloc, BQueue> p, sizeclass_t sizeclass)
{
SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p)));
UNUSED(sizeclass);
@@ -24,10 +24,12 @@ namespace snmalloc
return r;
}
template<ZeroMem zero_mem, typename SharedStateHandle>
template<
ZeroMem zero_mem,
typename SharedStateHandle,
SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue>
inline static SNMALLOC_FAST_PATH void* finish_alloc(
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc> p,
sizeclass_t sizeclass)
FreeObject::HeadPtr<capptr::bounds::Alloc, BQueue> p, sizeclass_t sizeclass)
{
auto r = finish_alloc_no_zero(p, sizeclass);
@@ -48,7 +50,7 @@ namespace snmalloc
// Free list per small size class. These are used for
// allocation on the fast path. This part of the code is inspired by
// mimalloc.
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>
small_fast_free_lists[NUM_SIZECLASSES];
// This is the entropy for a particular thread.
@@ -82,8 +84,11 @@ namespace snmalloc
typename SharedStateHandle::LocalState* local_state, DeallocFun dealloc)
{
auto& key = entropy.get_free_list_key();
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
auto domesticate =
[local_state](FreeObject::QueuePtr<capptr::bounds::AllocWild> p)
SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
for (size_t i = 0; i < NUM_SIZECLASSES; i++)
{

23
src/mem/metaslab.h
View File

@@ -25,10 +25,10 @@ namespace snmalloc
* Data-structure for building the free list for this slab.
*/
#ifdef SNMALLOC_CHECK_CLIENT
FreeListBuilder<true, capptr::bounds::Alloc, capptr::bounds::Alloc>
FreeListBuilder<true, capptr::bounds::Alloc, capptr::bounds::AllocWild>
free_queue;
#else
FreeListBuilder<false, capptr::bounds::Alloc, capptr::bounds::Alloc>
FreeListBuilder<false, capptr::bounds::Alloc, capptr::bounds::AllocWild>
free_queue;
#endif
@@ -155,15 +155,16 @@ namespace snmalloc
* available objects for this metaslab.
*/
template<typename Domesticator>
static SNMALLOC_FAST_PATH
std::pair<FreeObject::QueuePtr<capptr::bounds::Alloc>, bool>
alloc_free_list(
Domesticator domesticate,
Metaslab* meta,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>&
fast_free_list,
LocalEntropy& entropy,
sizeclass_t sizeclass)
static SNMALLOC_FAST_PATH std::pair<
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::AllocWild>,
bool>
alloc_free_list(
Domesticator domesticate,
Metaslab* meta,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::AllocWild>&
fast_free_list,
LocalEntropy& entropy,
sizeclass_t sizeclass)
{
auto& key = entropy.get_free_list_key();