Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor MetaEntry remote_and_sizeclass

Browse Source 
Introduce a class that we can use to more completely separate the frontend
encoding details from the backend.
This commit is contained in:
Nathaniel Wesley Filardo
2022-03-16 13:02:26 +00:00
committed by Nathaniel Wesley Filardo
parent 772e46f878
commit 7940fee00c
8 changed files with 124 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/backend/backend.h
View File

@@ -59,10 +59,14 @@ namespace snmalloc
* Set template parameter to true if it not an error
* to access a location that is not backed by a chunk.
*/
template<bool potentially_out_of_range = false>
SNMALLOC_FAST_PATH static const MetaEntry& get_metaentry(address_t p)
template<typename Ret = MetaEntry, bool potentially_out_of_range = false>
SNMALLOC_FAST_PATH static const Ret& get_metaentry(address_t p)
{
return concretePagemap.template get<potentially_out_of_range>(p);
static_assert(
std::is_base_of_v<MetaEntry, Ret> && sizeof(MetaEntry) == sizeof(Ret),
"Backend Pagemap get_metaentry return must look like MetaEntry");
return static_cast<const Ret&>(
concretePagemap.template get<potentially_out_of_range>(p));
}
/**
@@ -250,15 +254,15 @@ namespace snmalloc
* (remote, sizeclass, metaslab)
* where metaslab, is the second element of the pair return.
*/
static std::pair<capptr::Chunk<void>, Metaslab*> alloc_chunk(
LocalState& local_state,
size_t size,
RemoteAllocator* remote,
sizeclass_t sizeclass)
static std::pair<capptr::Chunk<void>, Metaslab*>
alloc_chunk(LocalState& local_state, size_t size, uintptr_t ras)
{
SNMALLOC_ASSERT(bits::is_pow2(size));
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
SNMALLOC_ASSERT((ras & MetaEntry::REMOTE_BACKEND_MARKER) == 0);
ras &= ~MetaEntry::REMOTE_BACKEND_MARKER;
auto meta_cap =
local_state.get_meta_range()->alloc_range(PAGEMAP_METADATA_STRUCT_SIZE);
@@ -289,7 +293,7 @@ namespace snmalloc
meta->meta_common.chunk = p;
MetaEntry t(meta, remote, sizeclass);
MetaEntry t(&meta->meta_common, ras);
Pagemap::set_metaentry(address_cast(p), size, t);
p = Aal::capptr_bound<void, capptr::bounds::Chunk>(p, size);

4
src/backend/backend_concept.h
View File

@@ -24,10 +24,10 @@ namespace snmalloc
{
{ Meta::set_metaentry(addr, sz, t) } -> ConceptSame<void>;
{ Meta::template get_metaentry<true>(addr) }
{ Meta::template get_metaentry<MetaEntry, true>(addr) }
-> ConceptSame<const MetaEntry&>;
{ Meta::template get_metaentry<false>(addr) }
{ Meta::template get_metaentry<MetaEntry, false>(addr) }
-> ConceptSame<const MetaEntry&>;
};

9
src/backend/chunkallocator.h
View File

@@ -183,10 +183,9 @@ namespace snmalloc
static std::pair<capptr::Chunk<void>, Metaslab*> alloc_chunk(
typename SharedStateHandle::LocalState& local_state,
ChunkAllocatorLocalState& chunk_alloc_local_state,
sizeclass_t sizeclass,
chunksizeclass_t slab_sizeclass,
size_t slab_size,
RemoteAllocator* remote)
uintptr_t ras)
{
using PAL = typename SharedStateHandle::Pal;
ChunkAllocatorState& state =
@@ -234,7 +233,7 @@ namespace snmalloc
<< " memory in stacks " << state.memory_in_stacks
<< std::endl;
#endif
MetaEntry entry{meta, remote, sizeclass};
MetaEntry entry{&meta->meta_common, ras};
SharedStateHandle::Pagemap::set_metaentry(
address_cast(slab), slab_size, entry);
return {slab, meta};
@@ -242,8 +241,8 @@ namespace snmalloc
// Allocate a fresh slab as there are no available ones.
// First create meta-data
auto [slab, meta] = SharedStateHandle::alloc_chunk(
&local_state, slab_size, remote, sizeclass);
auto [slab, meta] =
SharedStateHandle::alloc_chunk(&local_state, slab_size, ras);
#ifdef SNMALLOC_TRACING
std::cout << "Create slab:" << slab.unsafe_ptr() << " slab_sizeclass "
<< slab_sizeclass << " size " << slab_size << std::endl;

41
src/backend/metatypes.h
View File

@@ -79,10 +79,6 @@ namespace snmalloc
#endif
// clang-format on
struct RemoteAllocator;
class Metaslab;
class sizeclass_t;
/**
* Entry stored in the pagemap. See docs/AddressSpace.md for the full
* MetaEntry lifecycle.
@@ -140,41 +136,23 @@ namespace snmalloc
* `get_remote_and_sizeclass`.
*/
SNMALLOC_FAST_PATH
MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass)
: meta(unsafe_to_uintptr<Metaslab>(meta)),
MetaEntry(MetaCommon* meta, uintptr_t remote_and_sizeclass)
: meta(unsafe_to_uintptr<MetaCommon>(meta)),
remote_and_sizeclass(remote_and_sizeclass)
{}
/* See mem/metaslab.h */
SNMALLOC_FAST_PATH
MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass);
/**
* Return the Metaslab metadata associated with this chunk, guarded by an
* assert that this chunk is being used as a slab (i.e., has an associated
* owning allocator).
*/
[[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const
{
SNMALLOC_ASSERT(get_remote() != nullptr);
return unsafe_from_uintptr<Metaslab>(meta & ~META_BOUNDARY_BIT);
}
/**
* Return the remote and sizeclass in an implementation-defined encoding.
* This is not guaranteed to be stable across snmalloc releases and so the
* only safe use for this is to pass it to the two-argument constructor of
* this class.
*/
[[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const
[[nodiscard]] SNMALLOC_FAST_PATH const uintptr_t&
get_remote_and_sizeclass() const
{
return remote_and_sizeclass;
}
/* See mem/metaslab.h */
[[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const;
[[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const;
MetaEntry(const MetaEntry&) = delete;
MetaEntry& operator=(const MetaEntry& other)
@@ -186,6 +164,16 @@ namespace snmalloc
return *this;
}
/**
* Return the Metaslab metadata associated with this chunk, guarded by an
* assert that this chunk is being used as a slab (i.e., has an associated
* owning allocator).
*/
[[nodiscard]] SNMALLOC_FAST_PATH MetaCommon* get_meta() const
{
return reinterpret_cast<MetaCommon*>(meta & ~META_BOUNDARY_BIT);
}
void set_boundary()
{
meta |= META_BOUNDARY_BIT;
@@ -201,5 +189,4 @@ namespace snmalloc
return meta &= ~META_BOUNDARY_BIT;
}
};
} // namespace snmalloc

39
src/mem/corealloc.h
View File

@@ -403,7 +403,8 @@ namespace snmalloc
* by this thread, or handling the final deallocation onto a slab,
* so it can be reused by other threads.
*/
SNMALLOC_SLOW_PATH void dealloc_local_object_slow(const MetaEntry& entry)
SNMALLOC_SLOW_PATH void
dealloc_local_object_slow(const MetaslabMetaEntry& entry)
{
// TODO: Handle message queue on this path?
@@ -486,19 +487,20 @@ namespace snmalloc
[local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
auto cb = [this, &need_post](freelist::HeadPtr msg)
SNMALLOC_FAST_PATH_LAMBDA {
auto cb = [this,
&need_post](freelist::HeadPtr msg) SNMALLOC_FAST_PATH_LAMBDA {
#ifdef SNMALLOC_TRACING
std::cout << "Handling remote" << std::endl;
std::cout << "Handling remote" << std::endl;
#endif
auto& entry = SharedStateHandle::Pagemap::get_metaentry(
snmalloc::address_cast(msg));
auto& entry =
SharedStateHandle::Pagemap::template get_metaentry<MetaslabMetaEntry>(
snmalloc::address_cast(msg));
handle_dealloc_remote(entry, msg.as_void(), need_post);
handle_dealloc_remote(entry, msg.as_void(), need_post);
return true;
};
return true;
};
if constexpr (SharedStateHandle::Options.QueueHeadsAreTame)
{
@@ -532,7 +534,7 @@ namespace snmalloc
* need_post will be set to true, if capacity is exceeded.
*/
void handle_dealloc_remote(
const MetaEntry& entry,
const MetaslabMetaEntry& entry,
CapPtr<void, capptr::bounds::Alloc> p,
bool& need_post)
{
@@ -672,8 +674,10 @@ namespace snmalloc
SNMALLOC_FAST_PATH void
dealloc_local_object(CapPtr<void, capptr::bounds::Alloc> p)
{
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p));
// MetaEntry-s seen here are expected to have meaningful Remote pointers
auto& entry =
SharedStateHandle::Pagemap::template get_metaentry<MetaslabMetaEntry>(
snmalloc::address_cast(p));
if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy)))
return;
@@ -681,7 +685,7 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH static bool dealloc_local_object_fast(
const MetaEntry& entry,
const MetaslabMetaEntry& entry,
CapPtr<void, capptr::bounds::Alloc> p,
LocalEntropy& entropy)
{
@@ -786,8 +790,8 @@ namespace snmalloc
auto [slab, meta] = SharedStateHandle::alloc_chunk(
get_backend_local_state(),
slab_size,
public_state(),
sizeclass_t::from_small_class(sizeclass));
MetaslabMetaEntry::encode(
public_state(), sizeclass_t::from_small_class(sizeclass)));
if (slab == nullptr)
{
@@ -840,8 +844,9 @@ namespace snmalloc
{
bool need_post = true; // Always going to post, so ignore.
auto n_tame = p_tame->atomic_read_next(key_global, domesticate);
auto& entry = SharedStateHandle::Pagemap::get_metaentry(
snmalloc::address_cast(p_tame));
const MetaslabMetaEntry& entry =
SharedStateHandle::Pagemap::template get_metaentry<
MetaslabMetaEntry>(snmalloc::address_cast(p_tame));
handle_dealloc_remote(entry, p_tame.as_void(), need_post);
p_tame = n_tame;
}

29
src/mem/localalloc.h
View File

@@ -183,8 +183,8 @@ namespace snmalloc
auto [chunk, meta] = SharedStateHandle::alloc_chunk(
core_alloc->get_backend_local_state(),
large_size_to_chunk_size(size),
core_alloc->public_state(),
size_to_sizeclass_full(size));
MetaslabMetaEntry::encode(
core_alloc->public_state(), size_to_sizeclass_full(size)));
// set up meta data so sizeclass is correct, and hence alloc size, and
// external pointer.
#ifdef SNMALLOC_TRACING
@@ -266,8 +266,9 @@ namespace snmalloc
std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size "
<< alloc_size(p.unsafe_ptr()) << std::endl;
#endif
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(p));
const MetaslabMetaEntry& entry =
SharedStateHandle::Pagemap::template get_metaentry<MetaslabMetaEntry>(
address_cast(p));
local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
entry.get_remote()->trunc_id(), p, key_global);
post_remote_cache();
@@ -624,8 +625,9 @@ namespace snmalloc
capptr::Alloc<void> p_tame = capptr_domesticate<SharedStateHandle>(
core_alloc->backend_state_ptr(), p_wild);
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(p_tame));
const MetaslabMetaEntry& entry =
SharedStateHandle::Pagemap::template get_metaentry<MetaslabMetaEntry>(
address_cast(p_tame));
if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote()))
{
# if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT)
@@ -714,8 +716,9 @@ namespace snmalloc
// To handle this case we require the uninitialised pagemap contain an
// entry for the first chunk of memory, that states it represents a
// large object, so we can pull the check for null off the fast path.
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw));
const MetaslabMetaEntry& entry =
SharedStateHandle::Pagemap::template get_metaentry<MetaslabMetaEntry>(
address_cast(p_raw));
return sizeclass_full_to_size(entry.get_sizeclass());
#endif
@@ -759,9 +762,8 @@ namespace snmalloc
size_t remaining_bytes(const void* p)
{
#ifndef SNMALLOC_PASS_THROUGH
const MetaEntry& entry =
SharedStateHandle::Pagemap::template get_metaentry<true>(
address_cast(p));
const MetaslabMetaEntry& entry = SharedStateHandle::Pagemap::
template get_metaentry<MetaslabMetaEntry, true>(address_cast(p));
auto sizeclass = entry.get_sizeclass();
return snmalloc::remaining_bytes(sizeclass, address_cast(p));
@@ -788,9 +790,8 @@ namespace snmalloc
size_t index_in_object(const void* p)
{
#ifndef SNMALLOC_PASS_THROUGH
const MetaEntry& entry =
SharedStateHandle::Pagemap::template get_metaentry<true>(
address_cast(p));
const MetaslabMetaEntry& entry = SharedStateHandle::Pagemap::
template get_metaentry<MetaslabMetaEntry, true>(address_cast(p));
auto sizeclass = entry.get_sizeclass();
return snmalloc::index_in_object(sizeclass, address_cast(p));

77
src/mem/metaslab.h
View File

@@ -226,39 +226,56 @@ namespace snmalloc
};
/*
* MetaEntry methods that deal with RemoteAllocator* and sizeclass_t are here,
* so that the backend does not need to know the details and can, instead,
* just provide the storage space.
* A convenience wrapper aroun MetaEntry with a meaningful RemoteAllocator
* pointer. This encodes a RemoteAllocator* and a sizeclass_t into a the
* uintptr_t remote_and_sizeclass field.
*
* There's a little bit of an asymmetry here. Since the backend actually sets
* the entry (when associating a metadata structure), MetaslabMetaEntry-s are
* not constructed directly; please use ::encode(). On the other hand, the
* backend's Pagemap::get_metaentry() method is templated on its return type,
* so it is relatively straightforward to view a pagemap entry as a
* MetaslabMetaEntry and then use the accessors here for decoding.
*/
SNMALLOC_FAST_PATH_INLINE
MetaEntry::MetaEntry(
Metaslab* meta,
RemoteAllocator* remote,
sizeclass_t sizeclass = sizeclass_t())
: meta(reinterpret_cast<uintptr_t>(meta))
struct MetaslabMetaEntry : public MetaEntry
{
/* remote might be nullptr; cast to uintptr_t before offsetting */
remote_and_sizeclass =
pointer_offset(reinterpret_cast<uintptr_t>(remote), sizeclass.raw());
}
/// Perform the encoding.
static SNMALLOC_FAST_PATH uintptr_t
encode(RemoteAllocator* remote, sizeclass_t sizeclass)
{
/* remote might be nullptr; cast to uintptr_t before offsetting */
return pointer_offset(
reinterpret_cast<uintptr_t>(remote), sizeclass.raw());
}
[[nodiscard]] SNMALLOC_FAST_PATH_INLINE RemoteAllocator*
MetaEntry::get_remote() const
{
return reinterpret_cast<RemoteAllocator*>(
pointer_align_down<REMOTE_WITH_BACKEND_MARKER_ALIGN>(
remote_and_sizeclass));
}
[[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const
{
return reinterpret_cast<RemoteAllocator*>(
pointer_align_down<REMOTE_WITH_BACKEND_MARKER_ALIGN>(
get_remote_and_sizeclass()));
}
[[nodiscard]] SNMALLOC_FAST_PATH_INLINE sizeclass_t
MetaEntry::get_sizeclass() const
{
// TODO: perhaps remove static_cast with resolution of
// https://github.com/CTSRD-CHERI/llvm-project/issues/588
return sizeclass_t::from_raw(
static_cast<size_t>(remote_and_sizeclass) &
(REMOTE_WITH_BACKEND_MARKER_ALIGN - 1));
}
[[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const
{
// TODO: perhaps remove static_cast with resolution of
// https://github.com/CTSRD-CHERI/llvm-project/issues/588
return sizeclass_t::from_raw(
static_cast<size_t>(get_remote_and_sizeclass()) &
(REMOTE_WITH_BACKEND_MARKER_ALIGN - 1));
}
/**
* Return the Metaslab metadata associated with this chunk, guarded by an
* assert that this chunk is being used as a slab (i.e., has an associated
* owning allocator).
*/
[[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const
{
SNMALLOC_ASSERT(get_remote() != nullptr);
return reinterpret_cast<Metaslab*>(get_meta());
}
};
static_assert(sizeof(MetaslabMetaEntry) == sizeof(MetaEntry));
} // namespace snmalloc

12
src/mem/remotecache.h
View File

@@ -52,7 +52,7 @@ namespace snmalloc
*
* This does not require initialisation to be safely called.
*/
SNMALLOC_FAST_PATH bool reserve_space(const MetaEntry& entry)
SNMALLOC_FAST_PATH bool reserve_space(const MetaslabMetaEntry& entry)
{
auto size =
static_cast<int64_t>(sizeclass_full_to_size(entry.get_sizeclass()));
@@ -101,8 +101,9 @@ namespace snmalloc
if (!list[i].empty())
{
auto [first, last] = list[i].extract_segment(key);
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(first));
const MetaslabMetaEntry& entry =
SharedStateHandle::Pagemap::template get_metaentry<
MetaslabMetaEntry>(address_cast(first));
auto remote = entry.get_remote();
// If the allocator is not correctly aligned, then the bit that is
// set implies this is used by the backend, and we should not be
@@ -141,8 +142,9 @@ namespace snmalloc
// Use the next N bits to spread out remote deallocs in our own
// slot.
auto r = resend.take(key, domesticate);
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(r));
const MetaslabMetaEntry& entry =
SharedStateHandle::Pagemap::template get_metaentry<
MetaslabMetaEntry>(address_cast(r));
auto i = entry.get_remote()->trunc_id();
size_t slot = get_slot<allocator_size>(i, post_round);
list[slot].add(r, key);