Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CapPtr: shift free lists to Alloc bounds

Browse Source 
This is incomplete, yet still more reflective of what's going on: we take the
exported pointers back from userspace and thread them directly into the free
lists.

So: move capptr_to_user_address_control to list construction time rather than
list consumption time.
This commit is contained in:
Nathaniel Wesley Filardo
2021-09-21 19:17:51 +01:00
committed by Nathaniel Wesley Filardo
parent 55cd5e87c0
commit 7e53a2e82a
7 changed files with 99 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/mem/corealloc.h
View File

@@ -162,8 +162,8 @@ namespace snmalloc
// Manufacture an allocation to prime the queue
// Using an actual allocation removes a conditional from a critical path.
auto dummy =
capptr::AllocFull<void>(small_alloc_one(MIN_ALLOC_SIZE))
.template as_static<FreeObject::T<capptr::bounds::AllocFull>>();
capptr::Alloc<void>(small_alloc_one(MIN_ALLOC_SIZE))
.template as_static<FreeObject::T<capptr::bounds::Alloc>>();
if (dummy == nullptr)
{
error("Critical error: Out-of-memory during initialisation.");
@@ -185,8 +185,9 @@ namespace snmalloc
size,
[&](
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>*
fl) { return small_alloc<NoZero>(sizeclass, *fl); });
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>* fl) {
return small_alloc<NoZero>(sizeclass, *fl);
});
}
static SNMALLOC_FAST_PATH void alloc_new_list(
@@ -248,7 +249,8 @@ namespace snmalloc
do
{
b.add(
FreeObject::make<capptr::bounds::AllocFull>(curr_ptr.as_void()),
FreeObject::make<capptr::bounds::Alloc>(
capptr_to_user_address_control(curr_ptr.as_void())),
key,
entropy);
curr_ptr = curr_ptr->next;
@@ -258,9 +260,10 @@ namespace snmalloc
do
{
b.add(
Aal::capptr_bound<
FreeObject::T<capptr::bounds::AllocFull>,
capptr::bounds::AllocFull>(p, rsize),
FreeObject::make<capptr::bounds::Alloc>(
capptr_to_user_address_control(
Aal::capptr_bound<void, capptr::bounds::AllocFull>(
p.as_void(), rsize))),
key);
p = pointer_offset(p, rsize);
} while (p < slab_end);
@@ -272,7 +275,7 @@ namespace snmalloc
ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass)
{
auto& key = entropy.get_free_list_key();
FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull> fl;
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc> fl;
auto more = meta->free_queue.close(fl, key);
UNUSED(more);
void* p = finish_alloc_no_zero(fl.take(key), sizeclass);
@@ -435,7 +438,7 @@ namespace snmalloc
*/
void handle_dealloc_remote(
const MetaEntry& entry,
CapPtr<void, capptr::bounds::AllocFull> p,
CapPtr<void, capptr::bounds::Alloc> p,
bool& need_post)
{
// TODO this needs to not double count stats
@@ -444,7 +447,7 @@ namespace snmalloc
if (likely(entry.get_remote() == public_state()))
{
if (likely(dealloc_local_object_fast(entry, p.unsafe_ptr(), entropy)))
if (likely(dealloc_local_object_fast(entry, p.as_void(), entropy)))
return;
dealloc_local_object_slow(entry);
@@ -569,7 +572,8 @@ namespace snmalloc
return handle_message_queue_inner(action, args...);
}
SNMALLOC_FAST_PATH void dealloc_local_object(void* p)
SNMALLOC_FAST_PATH void
dealloc_local_object(CapPtr<void, capptr::bounds::Alloc> p)
{
auto entry = SharedStateHandle::Pagemap::get_metaentry(
backend_state_ptr(), snmalloc::address_cast(p));
@@ -580,7 +584,9 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH static bool dealloc_local_object_fast(
const MetaEntry& entry, void* p, LocalEntropy& entropy)
const MetaEntry& entry,
CapPtr<void, capptr::bounds::Alloc> p,
LocalEntropy& entropy)
{
auto meta = entry.get_metaslab();
@@ -590,8 +596,7 @@ namespace snmalloc
Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)),
"Not deallocating start of an object");
auto cp = FreeObject::QueuePtr<capptr::bounds::AllocFull>(
reinterpret_cast<FreeObject::T<capptr::bounds::AllocFull>*>(p));
auto cp = p.as_static<FreeObject::T<capptr::bounds::Alloc>>();
auto& key = entropy.get_free_list_key();
@@ -604,7 +609,7 @@ namespace snmalloc
template<ZeroMem zero_mem>
SNMALLOC_SLOW_PATH void* small_alloc(
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>&
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>&
fast_free_list)
{
size_t rsize = sizeclass_to_size(sizeclass);
@@ -664,7 +669,7 @@ namespace snmalloc
template<ZeroMem zero_mem>
SNMALLOC_SLOW_PATH void* small_alloc_slow(
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>&
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>&
fast_free_list,
size_t rsize)
{
@@ -717,7 +722,7 @@ namespace snmalloc
{
SNMALLOC_ASSERT(attached_cache != nullptr);
// TODO: Placeholder
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::AllocFull> p)
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
if (destroy_queue)

4
src/mem/freelist.h
View File

@@ -249,8 +249,8 @@ namespace snmalloc
/**
* Decode next. While traversing a queue, BView and BQueue here will often
* be equal (i.e., CBAllocExportWild) rather than dichotomous. However,
* we do occasionally decode an actual head pointer, so be polymorphic here.
* be equal (i.e., AllocUserWild) rather than dichotomous. However, we do
* occasionally decode an actual head pointer, so be polymorphic here.
*
* TODO: We'd like, in some sense, to more tightly couple or integrate this
* into to the domestication process. We could introduce an additional

83
src/mem/localalloc.h
View File

@@ -214,30 +214,29 @@ namespace snmalloc
auto slowpath =
[&](
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>*
fl) SNMALLOC_FAST_PATH_LAMBDA {
if (likely(core_alloc != nullptr))
{
return core_alloc->handle_message_queue(
[](
CoreAlloc* core_alloc,
sizeclass_t sizeclass,
FreeListIter<
capptr::bounds::AllocFull,
capptr::bounds::AllocFull>* fl) {
return core_alloc->template small_alloc<zero_mem>(
sizeclass, *fl);
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>* fl)
SNMALLOC_FAST_PATH_LAMBDA {
if (likely(core_alloc != nullptr))
{
return core_alloc->handle_message_queue(
[](
CoreAlloc* core_alloc,
sizeclass_t sizeclass,
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>*
fl) {
return core_alloc->template small_alloc<zero_mem>(
sizeclass, *fl);
},
core_alloc,
sizeclass,
fl);
}
return lazy_init(
[&](CoreAlloc*, sizeclass_t sizeclass) {
return small_alloc<zero_mem>(sizeclass_to_size(sizeclass));
},
core_alloc,
sizeclass,
fl);
}
return lazy_init(
[&](CoreAlloc*, sizeclass_t sizeclass) {
return small_alloc<zero_mem>(sizeclass_to_size(sizeclass));
},
sizeclass);
};
sizeclass);
};
return local_cache.template alloc<zero_mem, SharedStateHandle>(
size, slowpath);
@@ -260,20 +259,18 @@ namespace snmalloc
* In the second case we need to recheck if this is a remote deallocation,
* as we might acquire the originating allocator.
*/
SNMALLOC_SLOW_PATH void dealloc_remote_slow(void* p)
SNMALLOC_SLOW_PATH void dealloc_remote_slow(capptr::Alloc<void> p)
{
if (core_alloc != nullptr)
{
#ifdef SNMALLOC_TRACING
std::cout << "Remote dealloc post" << p << " size " << alloc_size(p)
<< std::endl;
std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size "
<< alloc_size(p.unsafe_ptr()) << std::endl;
#endif
MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry(
core_alloc->backend_state_ptr(), address_cast(p));
local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
entry.get_remote()->trunc_id(),
capptr::AllocFull<void>(p),
key_global);
entry.get_remote()->trunc_id(), p, key_global);
post_remote_cache();
return;
}
@@ -281,8 +278,8 @@ namespace snmalloc
// Recheck what kind of dealloc we should do incase, the allocator we get
// from lazy_init is the originating allocator.
lazy_init(
[&](CoreAlloc*, void* p) {
dealloc(p); // TODO don't double count statistics
[&](CoreAlloc*, CapPtr<void, capptr::bounds::Alloc> p) {
dealloc(p.unsafe_ptr()); // TODO don't double count statistics
return nullptr;
},
p);
@@ -450,10 +447,10 @@ namespace snmalloc
return alloc<zero_mem>(size);
}
SNMALLOC_FAST_PATH void dealloc(void* p)
SNMALLOC_FAST_PATH void dealloc(void* p_raw)
{
#ifdef SNMALLOC_PASS_THROUGH
external_alloc::free(p);
external_alloc::free(p_raw);
#else
// TODO:
// Care is needed so that dealloc(nullptr) works before init
@@ -461,6 +458,7 @@ namespace snmalloc
// before init, that maps null to a remote_deallocator that will never be
// in thread local state.
auto p = capptr_from_client(p_raw);
const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(
core_alloc->backend_state_ptr(), address_cast(p));
if (likely(local_cache.remote_allocator == entry.get_remote()))
@@ -478,12 +476,10 @@ namespace snmalloc
if (local_cache.remote_dealloc_cache.reserve_space(entry))
{
local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
entry.get_remote()->trunc_id(),
capptr::AllocFull<void>(p),
key_global);
entry.get_remote()->trunc_id(), p, key_global);
# ifdef SNMALLOC_TRACING
std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p)
<< std::endl;
std::cout << "Remote dealloc fast" << p_raw << " size "
<< alloc_size(p_raw) << std::endl;
# endif
return;
}
@@ -493,7 +489,7 @@ namespace snmalloc
}
// Large deallocation or null.
if (likely(p != nullptr))
if (likely(p.unsafe_ptr() != nullptr))
{
// Check this is managed by this pagemap.
check_client(entry.get_sizeclass() != 0, "Not allocated by snmalloc.");
@@ -511,7 +507,14 @@ namespace snmalloc
# endif
ChunkRecord* slab_record =
reinterpret_cast<ChunkRecord*>(entry.get_metaslab());
slab_record->chunk = capptr::Chunk<void>(p);
/*
* StrictProvenance TODO: this is a subversive amplification. p is
* AllocWild-bounded, but we're coercing it to Chunk-bounded. We
* should, instead, not be storing ->chunk here, but should be keeping
* a CapPtr<void, Chunk> to this region internally even while it's
* allocated.
*/
slab_record->chunk = capptr::Chunk<void>(p.unsafe_ptr());
check_init(
[](
CoreAlloc* core_alloc,

11
src/mem/localcache.h
View File

@@ -13,20 +13,20 @@ namespace snmalloc
using Stats = AllocStats<NUM_SIZECLASSES, NUM_LARGE_CLASSES>;
inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero(
FreeObject::HeadPtr<capptr::bounds::AllocFull, capptr::bounds::AllocFull> p,
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc> p,
sizeclass_t sizeclass)
{
SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p)));
UNUSED(sizeclass);
auto r = capptr_reveal(capptr_to_user_address_control(p.as_void()));
auto r = capptr_reveal(p.as_void());
return r;
}
template<ZeroMem zero_mem, typename SharedStateHandle>
inline static SNMALLOC_FAST_PATH void* finish_alloc(
FreeObject::HeadPtr<capptr::bounds::AllocFull, capptr::bounds::AllocFull> p,
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc> p,
sizeclass_t sizeclass)
{
auto r = finish_alloc_no_zero(p, sizeclass);
@@ -48,7 +48,7 @@ namespace snmalloc
// Free list per small size class. These are used for
// allocation on the fast path. This part of the code is inspired by
// mimalloc.
FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>
small_fast_free_lists[NUM_SIZECLASSES];
// This is the entropy for a particular thread.
@@ -90,7 +90,8 @@ namespace snmalloc
while (!small_fast_free_lists[i].empty())
{
auto p = small_fast_free_lists[i].take(key);
dealloc(finish_alloc_no_zero(p, i));
SNMALLOC_ASSERT(Metaslab::is_start_of_object(i, address_cast(p)));
dealloc(p.as_void());
}
}

8
src/mem/metaslab.h
View File

@@ -25,10 +25,10 @@ namespace snmalloc
* Data-structure for building the free list for this slab.
*/
#ifdef SNMALLOC_CHECK_CLIENT
FreeListBuilder<true, capptr::bounds::AllocFull, capptr::bounds::AllocFull>
FreeListBuilder<true, capptr::bounds::Alloc, capptr::bounds::Alloc>
free_queue;
#else
FreeListBuilder<false, capptr::bounds::AllocFull, capptr::bounds::AllocFull>
FreeListBuilder<false, capptr::bounds::Alloc, capptr::bounds::Alloc>
free_queue;
#endif
@@ -155,10 +155,10 @@ namespace snmalloc
* available objects for this metaslab.
*/
static SNMALLOC_FAST_PATH
std::pair<FreeObject::QueuePtr<capptr::bounds::AllocFull>, bool>
std::pair<FreeObject::QueuePtr<capptr::bounds::Alloc>, bool>
alloc_free_list(
Metaslab* meta,
FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>&
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc>&
fast_free_list,
LocalEntropy& entropy,
sizeclass_t sizeclass)

32
src/mem/remoteallocator.h
View File

@@ -36,11 +36,11 @@ namespace snmalloc
// Store the message queue on a separate cacheline. It is mutable data that
// is read by other threads.
alignas(CACHELINE_SIZE)
FreeObject::AtomicQueuePtr<capptr::bounds::AllocFull> back{nullptr};
FreeObject::AtomicQueuePtr<capptr::bounds::Alloc> back{nullptr};
// Store the two ends on different cache lines as access by different
// threads.
alignas(CACHELINE_SIZE)
FreeObject::QueuePtr<capptr::bounds::AllocFull> front{nullptr};
alignas(CACHELINE_SIZE) FreeObject::QueuePtr<capptr::bounds::Alloc> front{
nullptr};
constexpr RemoteAllocator() = default;
@@ -50,7 +50,7 @@ namespace snmalloc
SNMALLOC_ASSERT(front != nullptr);
}
void init(FreeObject::QueuePtr<capptr::bounds::AllocFull> stub)
void init(FreeObject::QueuePtr<capptr::bounds::Alloc> stub)
{
FreeObject::atomic_store_null(stub, key_global);
front = stub;
@@ -58,9 +58,9 @@ namespace snmalloc
invariant();
}
FreeObject::QueuePtr<capptr::bounds::AllocFull> destroy()
FreeObject::QueuePtr<capptr::bounds::Alloc> destroy()
{
FreeObject::QueuePtr<capptr::bounds::AllocFull> fnt = front;
FreeObject::QueuePtr<capptr::bounds::Alloc> fnt = front;
back.store(nullptr, std::memory_order_relaxed);
front = nullptr;
return fnt;
@@ -68,7 +68,7 @@ namespace snmalloc
inline bool is_empty()
{
FreeObject::QueuePtr<capptr::bounds::AllocFull> bk =
FreeObject::QueuePtr<capptr::bounds::Alloc> bk =
back.load(std::memory_order_relaxed);
return bk == front;
@@ -79,22 +79,22 @@ namespace snmalloc
* last should be linked together through their next pointers.
*/
void enqueue(
FreeObject::QueuePtr<capptr::bounds::AllocFull> first,
FreeObject::QueuePtr<capptr::bounds::AllocFull> last,
FreeObject::QueuePtr<capptr::bounds::Alloc> first,
FreeObject::QueuePtr<capptr::bounds::Alloc> last,
const FreeListKey& key)
{
invariant();
FreeObject::atomic_store_null(last, key);
// exchange needs to be a release, so nullptr in next is visible.
FreeObject::QueuePtr<capptr::bounds::AllocFull> prev =
FreeObject::QueuePtr<capptr::bounds::Alloc> prev =
back.exchange(last, std::memory_order_release);
// XXX prev is not known to be domesticated
FreeObject::atomic_store_next(prev, first, key);
}
FreeObject::QueuePtr<capptr::bounds::AllocFull> peek()
FreeObject::QueuePtr<capptr::bounds::Alloc> peek()
{
return front;
}
@@ -103,16 +103,16 @@ namespace snmalloc
* Returns the front message, or null if not possible to return a message.
*/
std::pair<
FreeObject::HeadPtr<capptr::bounds::AllocFull, capptr::bounds::AllocFull>,
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc>,
bool>
dequeue(const FreeListKey& key)
{
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::AllocFull> p)
auto domesticate = [](FreeObject::QueuePtr<capptr::bounds::Alloc> p)
SNMALLOC_FAST_PATH_LAMBDA { return p; };
invariant();
FreeObject::HeadPtr<capptr::bounds::AllocFull, capptr::bounds::AllocFull>
first = domesticate(front);
FreeObject::QueuePtr<capptr::bounds::AllocFull> next =
FreeObject::HeadPtr<capptr::bounds::Alloc, capptr::bounds::Alloc> first =
domesticate(front);
FreeObject::QueuePtr<capptr::bounds::Alloc> next =
first->atomic_read_next(key, domesticate);
if (next != nullptr)

11
src/mem/remotecache.h
View File

@@ -19,8 +19,8 @@ namespace snmalloc
std::array<
FreeListBuilder<
false,
capptr::bounds::AllocFull,
capptr::bounds::AllocFull,
capptr::bounds::Alloc,
capptr::bounds::Alloc,
false>,
REMOTE_SLOTS>
list;
@@ -73,12 +73,12 @@ namespace snmalloc
template<size_t allocator_size>
SNMALLOC_FAST_PATH void dealloc(
RemoteAllocator::alloc_id_t target_id,
capptr::AllocFull<void> p,
capptr::Alloc<void> p,
const FreeListKey& key)
{
SNMALLOC_ASSERT(initialised);
auto r =
p.template as_reinterpret<FreeObject::T<capptr::bounds::AllocFull>>();
p.template as_reinterpret<FreeObject::T<capptr::bounds::Alloc>>();
list[get_slot<allocator_size>(target_id, 0)].add(r, key);
}
@@ -118,8 +118,7 @@ namespace snmalloc
// Entries could map back onto the "resend" list,
// so take copy of the head, mark the last element,
// and clear the original list.
FreeListIter<capptr::bounds::AllocFull, capptr::bounds::AllocFull>
resend;
FreeListIter<capptr::bounds::Alloc, capptr::bounds::Alloc> resend;
list[my_slot].close(resend, key);
post_round++;