Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove local state from Pagemap representation.

Browse Source
This commit is contained in:
Matthew Parkinson
2022-02-10 11:57:13 +00:00
committed by Matthew Parkinson
parent be98a27bf1
commit eb00f3184f
9 changed files with 72 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/backend/address_space.h
View File

@@ -45,8 +45,7 @@ namespace snmalloc
* arena_map for use in subsequent amplification.
*/
template<bool committed>
capptr::Chunk<void>
reserve(typename Pagemap::LocalState* local_state, size_t size)
capptr::Chunk<void> reserve(size_t size)
{
#ifdef SNMALLOC_TRACING
std::cout << "ASM reserve request:" << size << std::endl;
@@ -64,7 +63,7 @@ namespace snmalloc
{
auto base =
capptr::Chunk<void>(PAL::template reserve_aligned<committed>(size));
Pagemap::register_range(local_state, address_cast(base), size);
Pagemap::register_range(address_cast(base), size);
return base;
}
}
@@ -72,7 +71,7 @@ namespace snmalloc
capptr::Chunk<void> res;
{
FlagLock lock(spin_lock);
res = core.template reserve<PAL>(local_state, size);
res = core.template reserve<PAL>(size);
if (res == nullptr)
{
// Allocation failed ask OS for more memory
@@ -134,12 +133,12 @@ namespace snmalloc
return nullptr;
}
Pagemap::register_range(local_state, address_cast(block), block_size);
Pagemap::register_range(address_cast(block), block_size);
core.template add_range<PAL>(local_state, block, block_size);
core.template add_range<PAL>(block, block_size);
// still holding lock so guaranteed to succeed.
res = core.template reserve<PAL>(local_state, size);
res = core.template reserve<PAL>(size);
}
}
@@ -158,8 +157,7 @@ namespace snmalloc
* used, by smaller objects.
*/
template<bool committed>
capptr::Chunk<void> reserve_with_left_over(
typename Pagemap::LocalState* local_state, size_t size)
capptr::Chunk<void> reserve_with_left_over(size_t size)
{
SNMALLOC_ASSERT(size >= sizeof(void*));
@@ -167,15 +165,14 @@ namespace snmalloc
size_t rsize = bits::next_pow2(size);
auto res = reserve<false>(local_state, rsize);
auto res = reserve<false>(rsize);
if (res != nullptr)
{
if (rsize > size)
{
FlagLock lock(spin_lock);
core.template add_range<PAL>(
local_state, pointer_offset(res, size), rsize - size);
core.template add_range<PAL>(pointer_offset(res, size), rsize - size);
}
if constexpr (committed)
@@ -195,13 +192,10 @@ namespace snmalloc
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(
typename Pagemap::LocalState* local_state,
capptr::Chunk<void> base,
size_t length)
void add_range(capptr::Chunk<void> base, size_t length)
{
FlagLock lock(spin_lock);
core.template add_range<PAL>(local_state, base, length);
core.template add_range<PAL>(base, length);
}
};
} // namespace snmalloc

48
src/backend/address_space_core.h
View File

@@ -79,7 +79,6 @@ namespace snmalloc
* particular size.
*/
void set_next(
typename Pagemap::LocalState* local_state,
size_t align_bits,
capptr::Chunk<FreeChunk> base,
capptr::Chunk<FreeChunk> next)
@@ -96,7 +95,7 @@ namespace snmalloc
// dealloc() can reject attempts to free such MetaEntry-s due to the
// zero sizeclass.
MetaEntry t(reinterpret_cast<Metaslab*>(next.unsafe_ptr()), nullptr);
Pagemap::set_metaentry(local_state, address_cast(base), 1, t);
Pagemap::set_metaentry(address_cast(base), 1, t);
return;
}
@@ -112,15 +111,13 @@ namespace snmalloc
* to store the next pointer for the list of unused address space of a
* particular size.
*/
capptr::Chunk<FreeChunk> get_next(
typename Pagemap::LocalState* local_state,
size_t align_bits,
capptr::Chunk<FreeChunk> base)
capptr::Chunk<FreeChunk>
get_next(size_t align_bits, capptr::Chunk<FreeChunk> base)
{
if (align_bits >= MIN_CHUNK_BITS)
{
const MetaEntry& t = Pagemap::template get_metaentry<false>(
local_state, address_cast(base));
const MetaEntry& t =
Pagemap::template get_metaentry<false>(address_cast(base));
return capptr::Chunk<FreeChunk>(
reinterpret_cast<FreeChunk*>(t.get_metaslab_no_remote()));
}
@@ -132,15 +129,12 @@ namespace snmalloc
* Adds a block to `ranges`.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
void add_block(
typename Pagemap::LocalState* local_state,
size_t align_bits,
capptr::Chunk<FreeChunk> base)
void add_block(size_t align_bits, capptr::Chunk<FreeChunk> base)
{
check_block(base, align_bits);
SNMALLOC_ASSERT(align_bits < 64);
set_next(local_state, align_bits, base, ranges[align_bits]);
set_next(align_bits, base, ranges[align_bits]);
ranges[align_bits] = base.template as_static<FreeChunk>();
}
@@ -149,8 +143,7 @@ namespace snmalloc
* to satisfy this request.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
capptr::Chunk<void>
remove_block(typename Pagemap::LocalState* local_state, size_t align_bits)
capptr::Chunk<void> remove_block(size_t align_bits)
{
capptr::Chunk<FreeChunk> first = ranges[align_bits];
if (first == nullptr)
@@ -162,8 +155,7 @@ namespace snmalloc
}
// Look for larger block and split up recursively
capptr::Chunk<void> bigger =
remove_block<PAL>(local_state, align_bits + 1);
capptr::Chunk<void> bigger = remove_block<PAL>(align_bits + 1);
if (SNMALLOC_UNLIKELY(bigger == nullptr))
return nullptr;
@@ -180,7 +172,6 @@ namespace snmalloc
auto left_over = pointer_offset(bigger, half_bigger_size);
add_block<PAL>(
local_state,
align_bits,
Aal::capptr_bound<FreeChunk, capptr::bounds::Chunk>(
left_over, half_bigger_size));
@@ -191,7 +182,7 @@ namespace snmalloc
}
check_block(first, align_bits);
ranges[align_bits] = get_next(local_state, align_bits, first);
ranges[align_bits] = get_next(align_bits, first);
return first.as_void();
}
@@ -201,10 +192,7 @@ namespace snmalloc
* Divides blocks into power of two sizes with natural alignment
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
void add_range(
typename Pagemap::LocalState* local_state,
capptr::Chunk<void> base,
size_t length)
void add_range(capptr::Chunk<void> base, size_t length)
{
// For start and end that are not chunk sized, we need to
// commit the pages to track the allocations.
@@ -235,7 +223,7 @@ namespace snmalloc
Aal::capptr_bound<FreeChunk, capptr::bounds::Chunk>(base, align);
check_block(b, align_bits);
add_block<PAL>(local_state, align_bits, b);
add_block<PAL>(align_bits, b);
base = pointer_offset(base, align);
length -= align;
@@ -268,8 +256,7 @@ namespace snmalloc
* arena_map for use in subsequent amplification.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
capptr::Chunk<void>
reserve(typename Pagemap::LocalState* local_state, size_t size)
capptr::Chunk<void> reserve(size_t size)
{
#ifdef SNMALLOC_TRACING
std::cout << "ASM Core reserve request:" << size << std::endl;
@@ -278,7 +265,7 @@ namespace snmalloc
SNMALLOC_ASSERT(bits::is_pow2(size));
SNMALLOC_ASSERT(size >= sizeof(void*));
return remove_block<PAL>(local_state, bits::next_pow2_bits(size));
return remove_block<PAL>(bits::next_pow2_bits(size));
}
/**
@@ -289,8 +276,7 @@ namespace snmalloc
* used by smaller objects.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
capptr::Chunk<void> reserve_with_left_over(
typename Pagemap::LocalState* local_state, size_t size)
capptr::Chunk<void> reserve_with_left_over(size_t size)
{
SNMALLOC_ASSERT(size >= sizeof(void*));
@@ -298,7 +284,7 @@ namespace snmalloc
size_t rsize = bits::next_pow2(size);
auto res = reserve<PAL>(local_state, rsize);
auto res = reserve<PAL>(rsize);
if (res != nullptr)
{
@@ -311,7 +297,7 @@ namespace snmalloc
size_t residual_size = rsize - size;
auto residual = pointer_offset(res, size);
res = Aal::capptr_bound<void, capptr::bounds::Chunk>(res, size);
add_range<PAL>(local_state, residual, residual_size);
add_range<PAL>(residual, residual_size);
}
}
return res;

49
src/backend/backend.h
View File

@@ -114,7 +114,7 @@ namespace snmalloc
meta->meta_common.chunk = p;
MetaEntry t(meta, remote, sizeclass);
Pagemap::set_metaentry(local_state, address_cast(p), size, t);
Pagemap::set_metaentry(address_cast(p), size, t);
return {p, meta};
}
@@ -146,14 +146,14 @@ namespace snmalloc
auto& local = local_state->local_address_space;
#endif
p = local.template reserve_with_left_over<PAL>(local_state, size);
p = local.template reserve_with_left_over<PAL>(size);
if (p != nullptr)
{
return p;
}
auto refill_size = LOCAL_CACHE_BLOCK;
auto refill = global.template reserve<false>(local_state, refill_size);
auto refill = global.template reserve<false>(refill_size);
if (refill == nullptr)
return nullptr;
@@ -165,10 +165,10 @@ namespace snmalloc
}
#endif
PAL::template notify_using<NoZero>(refill.unsafe_ptr(), refill_size);
local.template add_range<PAL>(local_state, refill, refill_size);
local.template add_range<PAL>(refill, refill_size);
// This should succeed
return local.template reserve_with_left_over<PAL>(local_state, size);
return local.template reserve_with_left_over<PAL>(size);
}
#ifdef SNMALLOC_META_PROTECTED
@@ -179,7 +179,7 @@ namespace snmalloc
size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size));
size_t size_request = rsize * 64;
p = global.template reserve<false>(local_state, size_request);
p = global.template reserve<false>(size_request);
if (p == nullptr)
return nullptr;
@@ -195,7 +195,7 @@ namespace snmalloc
SNMALLOC_ASSERT(!is_meta);
#endif
p = global.template reserve_with_left_over<true>(local_state, size);
p = global.template reserve_with_left_over<true>(size);
return p;
}
@@ -241,8 +241,6 @@ namespace snmalloc
public:
using Pal = PAL;
class LocalState;
class Pagemap
{
friend class BackendAllocator;
@@ -252,12 +250,6 @@ namespace snmalloc
concretePagemap;
public:
/**
* Provide a type alias for LocalState so that we can refer to it without
* needing the whole BackendAllocator type at hand.
*/
using LocalState = BackendAllocator::LocalState;
/**
* Get the metadata associated with a chunk.
*
@@ -265,10 +257,8 @@ namespace snmalloc
* to access a location that is not backed by a chunk.
*/
template<bool potentially_out_of_range = false>
SNMALLOC_FAST_PATH static const MetaEntry&
get_metaentry(LocalState* ls, address_t p)
SNMALLOC_FAST_PATH static const MetaEntry& get_metaentry(address_t p)
{
UNUSED(ls);
return concretePagemap.template get<potentially_out_of_range>(p);
}
@@ -279,10 +269,8 @@ namespace snmalloc
* to access a location that is not backed by a chunk.
*/
template<bool potentially_out_of_range = false>
SNMALLOC_FAST_PATH static MetaEntry&
get_metaentry_mut(LocalState* ls, address_t p)
SNMALLOC_FAST_PATH static MetaEntry& get_metaentry_mut(address_t p)
{
UNUSED(ls);
return concretePagemap.template get_mut<potentially_out_of_range>(p);
}
@@ -290,19 +278,16 @@ namespace snmalloc
* Set the metadata associated with a chunk.
*/
SNMALLOC_FAST_PATH
static void
set_metaentry(LocalState* ls, address_t p, size_t size, MetaEntry t)
static void set_metaentry(address_t p, size_t size, MetaEntry t)
{
UNUSED(ls);
for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE)
{
concretePagemap.set(a, t);
}
}
static void register_range(LocalState* ls, address_t p, size_t sz)
static void register_range(address_t p, size_t sz)
{
UNUSED(ls);
concretePagemap.register_range(p, sz);
}
@@ -314,18 +299,16 @@ namespace snmalloc
template<bool fixed_range_ = fixed_range>
static SNMALLOC_FAST_PATH
std::enable_if_t<fixed_range_, std::pair<address_t, address_t>>
get_bounds(LocalState* local_state)
get_bounds()
{
static_assert(
fixed_range_ == fixed_range, "Don't set SFINAE parameter!");
UNUSED(local_state);
return concretePagemap.get_bounds();
}
static bool is_initialised(LocalState* ls)
static bool is_initialised()
{
UNUSED(ls);
return concretePagemap.is_initialised();
}
};
@@ -372,15 +355,13 @@ namespace snmalloc
}
template<bool fixed_range_ = fixed_range>
static std::enable_if_t<fixed_range_>
init(LocalState* local_state, void* base, size_t length)
static std::enable_if_t<fixed_range_> init(void* base, size_t length)
{
static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!");
auto [heap_base, heap_length] =
Pagemap::concretePagemap.init(base, length);
address_space.add_range(
local_state, capptr::Chunk<void>(heap_base), heap_length);
address_space.add_range(capptr::Chunk<void>(heap_base), heap_length);
}
/**

17
src/backend/backend_concept.h
View File

@@ -18,17 +18,16 @@ namespace snmalloc
template<typename Meta>
concept ConceptBackendMeta =
requires(
typename Meta::LocalState* ls,
address_t addr,
size_t sz,
MetaEntry t)
{
{ Meta::set_metaentry(ls, addr, sz, t) } -> ConceptSame<void>;
{ Meta::set_metaentry(addr, sz, t) } -> ConceptSame<void>;
{ Meta::template get_metaentry<true>(ls, addr) }
{ Meta::template get_metaentry<true>(addr) }
-> ConceptSame<const MetaEntry&>;
{ Meta::template get_metaentry<false>(ls, addr) }
{ Meta::template get_metaentry<false>(addr) }
-> ConceptSame<const MetaEntry&>;
};
@@ -41,9 +40,9 @@ namespace snmalloc
*/
template<typename Meta>
concept ConceptBackendMeta_Range =
requires(typename Meta::LocalState* ls, address_t addr, size_t sz)
requires(address_t addr, size_t sz)
{
{ Meta::register_range(ls, addr, sz) } -> ConceptSame<void>;
{ Meta::register_range(addr, sz) } -> ConceptSame<void>;
};
/**
@@ -66,7 +65,8 @@ namespace snmalloc
*/
template<typename Globals>
concept ConceptBackendDomestication =
requires(typename Globals::LocalState* ls,
requires(
typename Globals::LocalState* ls,
capptr::AllocWild<void> ptr)
{
{ Globals::capptr_domesticate(ls, ptr) }
@@ -96,9 +96,6 @@ namespace snmalloc
std::is_base_of<CommonConfig, Globals>::value &&
ConceptPAL<typename Globals::Pal> &&
ConceptBackendMetaRange<typename Globals::Pagemap> &&
ConceptSame<
typename Globals::LocalState,
typename Globals::Pagemap::LocalState> &&
requires()
{
typename Globals::LocalState;

5
src/backend/fixedglobalconfig.h
View File

@@ -49,7 +49,8 @@ namespace snmalloc
static void
init(typename Backend::LocalState* local_state, void* base, size_t length)
{
Backend::init(local_state, base, length);
UNUSED(local_state);
Backend::init(base, length);
}
/* Verify that a pointer points into the region managed by this config */
@@ -66,7 +67,7 @@ namespace snmalloc
UNUSED(ls);
auto address = address_cast(p);
auto [base, length] = Backend::Pagemap::get_bounds(nullptr);
auto [base, length] = Backend::Pagemap::get_bounds();
if ((address - base > (length - sz)) || (length < sz))
{
return nullptr;

2
src/mem/chunkallocator.h
View File

@@ -234,7 +234,7 @@ namespace snmalloc
#endif
MetaEntry entry{meta, remote, sizeclass};
SharedStateHandle::Pagemap::set_metaentry(
&local_state, address_cast(slab), slab_size, entry);
address_cast(slab), slab_size, entry);
return {slab, meta};
}

10
src/mem/corealloc.h
View File

@@ -486,14 +486,14 @@ namespace snmalloc
[local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA {
return capptr_domesticate<SharedStateHandle>(local_state, p);
};
auto cb = [this, local_state, &need_post](freelist::HeadPtr msg)
auto cb = [this, &need_post](freelist::HeadPtr msg)
SNMALLOC_FAST_PATH_LAMBDA {
#ifdef SNMALLOC_TRACING
std::cout << "Handling remote" << std::endl;
#endif
auto& entry = SharedStateHandle::Pagemap::get_metaentry(
local_state, snmalloc::address_cast(msg));
snmalloc::address_cast(msg));
handle_dealloc_remote(entry, msg.as_void(), need_post);
@@ -675,8 +675,8 @@ namespace snmalloc
SNMALLOC_FAST_PATH void
dealloc_local_object(CapPtr<void, capptr::bounds::Alloc> p)
{
auto entry = SharedStateHandle::Pagemap::get_metaentry(
backend_state_ptr(), snmalloc::address_cast(p));
auto entry =
SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p));
if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy)))
return;
@@ -848,7 +848,7 @@ namespace snmalloc
bool need_post = true; // Always going to post, so ignore.
auto n_tame = p_tame->atomic_read_next(key_global, domesticate);
auto& entry = SharedStateHandle::Pagemap::get_metaentry(
backend_state_ptr(), snmalloc::address_cast(p_tame));
snmalloc::address_cast(p_tame));
handle_dealloc_remote(entry, p_tame.as_void(), need_post);
p_tame = n_tame;
}

19
src/mem/localalloc.h
View File

@@ -268,8 +268,8 @@ namespace snmalloc
std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size "
<< alloc_size(p.unsafe_ptr()) << std::endl;
#endif
MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry(
core_alloc->backend_state_ptr(), address_cast(p));
MetaEntry entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(p));
local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
entry.get_remote()->trunc_id(), p, key_global);
post_remote_cache();
@@ -626,8 +626,8 @@ namespace snmalloc
capptr::Alloc<void> p_tame = capptr_domesticate<SharedStateHandle>(
core_alloc->backend_state_ptr(), p_wild);
const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(
core_alloc->backend_state_ptr(), address_cast(p_tame));
const MetaEntry& entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(p_tame));
if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote()))
{
# if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT)
@@ -716,8 +716,8 @@ namespace snmalloc
// To handle this case we require the uninitialised pagemap contain an
// entry for the first chunk of memory, that states it represents a
// large object, so we can pull the check for null off the fast path.
MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry(
core_alloc->backend_state_ptr(), address_cast(p_raw));
MetaEntry entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw));
return sizeclass_full_to_size(entry.get_sizeclass());
#endif
@@ -763,7 +763,7 @@ namespace snmalloc
#ifndef SNMALLOC_PASS_THROUGH
MetaEntry entry =
SharedStateHandle::Pagemap::template get_metaentry<true>(
core_alloc->backend_state_ptr(), address_cast(p));
address_cast(p));
auto sizeclass = entry.get_sizeclass();
return snmalloc::remaining_bytes(sizeclass, address_cast(p));
@@ -774,8 +774,7 @@ namespace snmalloc
bool check_bounds(const void* p, size_t s)
{
auto ls = core_alloc->backend_state_ptr();
if (SNMALLOC_LIKELY(SharedStateHandle::Pagemap::is_initialised(ls)))
if (SNMALLOC_LIKELY(SharedStateHandle::Pagemap::is_initialised()))
{
return remaining_bytes(p) >= s;
}
@@ -793,7 +792,7 @@ namespace snmalloc
#ifndef SNMALLOC_PASS_THROUGH
MetaEntry entry =
SharedStateHandle::Pagemap::template get_metaentry<true>(
core_alloc->backend_state_ptr(), address_cast(p));
address_cast(p));
auto sizeclass = entry.get_sizeclass();
return snmalloc::index_in_object(sizeclass, address_cast(p));

8
src/mem/remotecache.h
View File

@@ -101,8 +101,8 @@ namespace snmalloc
if (!list[i].empty())
{
auto [first, last] = list[i].extract_segment(key);
MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry(
local_state, address_cast(first));
MetaEntry entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(first));
if constexpr (SharedStateHandle::Options.QueueHeadsAreTame)
{
auto domesticate_nop = [](freelist::QueuePtr p) {
@@ -134,8 +134,8 @@ namespace snmalloc
// Use the next N bits to spread out remote deallocs in our own
// slot.
auto r = resend.take(key, domesticate);
MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry(
local_state, address_cast(r));
MetaEntry entry =
SharedStateHandle::Pagemap::get_metaentry(address_cast(r));
auto i = entry.get_remote()->trunc_id();
size_t slot = get_slot<allocator_size>(i, post_round);
list[slot].add(r, key);