Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Overhaul CapPtr

Browse Source 
* Switch to a multidimensional taxonomy.

  Rather than encoding the abstract bound states in a single enum, move to a
  more algebraic treatment.  The dimensions themselves are within the
  snmalloc::capptr_bounds namespace so that their fairly generic names do not
  conflict with consumer code.  Aliases for many points in the space are
  established outside that namespace for ease of use elsewhere.

* Introduce several new namespaces:

    * snmalloc::capptr::dimension holds each of the dimension enums

    * snmalloc::capptr holds the bound<> type itself and a ConceptBound

    * snmalloc::capptr::bounds gives convenient specializations of bound<>

    * snmalloc::capptr also has aliases for CapPtr<> itself

  All told, rather than `CapPtr<T, CBChunk>`, we now expect client code to read
  `capptr::Chunk<T>` in almost all cases (and this is just an alias for the
  appropriate `CapPtr<T, bounds<...>>` type).  When the bound<>s themselves are
  necessary, as when calling capptr_bound, we expect that they will almost
  always be pronounced using an alias (e.g., `capptr::bounds::Alloc`).

* Chase consequences.

* Prune old taxa and aliases that are no longer in use in snmalloc2.
This commit is contained in:
Nathaniel Wesley Filardo
2021-07-19 09:49:18 +01:00
committed by Nathaniel Wesley Filardo
parent b57390663e
commit 9065893181
17 changed files with 399 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/mem/corealloc.h
View File

@@ -161,7 +161,7 @@ namespace snmalloc
{
// Manufacture an allocation to prime the queue
// Using an actual allocation removes a conditional from a critical path.
auto dummy = CapPtr<void, CBAlloc>(small_alloc_one(MIN_ALLOC_SIZE))
auto dummy = capptr::AllocFull<void>(small_alloc_one(MIN_ALLOC_SIZE))
.template as_static<FreeObject>();
if (dummy == nullptr)
{
@@ -187,7 +187,7 @@ namespace snmalloc
}
static SNMALLOC_FAST_PATH void alloc_new_list(
CapPtr<void, CBChunk>& bumpptr,
capptr::Chunk<void>& bumpptr,
Metaslab* meta,
size_t rsize,
size_t slab_size,
@@ -203,7 +203,7 @@ namespace snmalloc
// Structure to represent the temporary list elements
struct PreAllocObject
{
CapPtr<PreAllocObject, CBAlloc> next;
capptr::AllocFull<PreAllocObject> next;
};
// The following code implements Sattolo's algorithm for generating
// random cyclic permutations. This implementation is in the opposite
@@ -213,9 +213,10 @@ namespace snmalloc
// Note the wide bounds on curr relative to each of the ->next fields;
// curr is not persisted once the list is built.
CapPtr<PreAllocObject, CBChunk> curr =
capptr::Chunk<PreAllocObject> curr =
pointer_offset(bumpptr, 0).template as_static<PreAllocObject>();
curr->next = Aal::capptr_bound<PreAllocObject, CBAlloc>(curr, rsize);
curr->next = Aal::capptr_bound<PreAllocObject, capptr::bounds::AllocFull>(
curr, rsize);
uint16_t count = 1;
for (curr =
@@ -229,12 +230,13 @@ namespace snmalloc
pointer_offset(bumpptr, insert_index * rsize)
.template as_static<PreAllocObject>()
->next,
Aal::capptr_bound<PreAllocObject, CBAlloc>(curr, rsize));
Aal::capptr_bound<PreAllocObject, capptr::bounds::AllocFull>(
curr, rsize));
count++;
}
// Pick entry into space, and then build linked list by traversing cycle
// to the start. Use ->next to jump from CBArena to CBAlloc.
// to the start. Use ->next to jump from Chunk to Alloc.
auto start_index = entropy.sample(count);
auto start_ptr = pointer_offset(bumpptr, start_index * rsize)
.template as_static<PreAllocObject>()
@@ -249,7 +251,9 @@ namespace snmalloc
auto p = bumpptr;
do
{
b.add(Aal::capptr_bound<FreeObject, CBAlloc>(p, rsize), key);
b.add(
Aal::capptr_bound<FreeObject, capptr::bounds::AllocFull>(p, rsize),
key);
p = pointer_offset(p, rsize);
} while (p < slab_end);
#endif
@@ -299,7 +303,7 @@ namespace snmalloc
auto start_of_slab = pointer_align_down<void>(
p, snmalloc::sizeclass_to_slab_size(sizeclass));
// TODO Add bounds correctly here
chunk_record->chunk = CapPtr<void, CBChunk>(start_of_slab);
chunk_record->chunk = capptr::Chunk<void>(start_of_slab);
#ifdef SNMALLOC_TRACING
std::cout << "Slab " << start_of_slab << " is unused, Object sizeclass "
@@ -422,7 +426,7 @@ namespace snmalloc
* need_post will be set to true, if capacity is exceeded.
*/
void handle_dealloc_remote(
const MetaEntry& entry, CapPtr<FreeObject, CBAlloc> p, bool& need_post)
const MetaEntry& entry, capptr::AllocFull<FreeObject> p, bool& need_post)
{
// TODO this needs to not double count stats
// TODO this needs to not double revoke if using MTE
@@ -576,7 +580,7 @@ namespace snmalloc
Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)),
"Not deallocating start of an object");
auto cp = CapPtr<FreeObject, CBAlloc>(reinterpret_cast<FreeObject*>(p));
auto cp = capptr::AllocFull<FreeObject>(reinterpret_cast<FreeObject*>(p));
auto& key = entropy.get_free_list_key();

53
src/mem/freelist.h
View File

@@ -66,9 +66,9 @@ namespace snmalloc
{
union
{
CapPtr<FreeObject, CBAlloc> next_object;
capptr::AllocFull<FreeObject> next_object;
// TODO: Should really use C++20 atomic_ref rather than a union.
AtomicCapPtr<FreeObject, CBAlloc> atomic_next_object;
capptr::AtomicAllocFull<FreeObject> atomic_next_object;
};
#ifdef SNMALLOC_CHECK_CLIENT
// Encoded representation of a back pointer.
@@ -78,7 +78,7 @@ namespace snmalloc
#endif
public:
static CapPtr<FreeObject, CBAlloc> make(CapPtr<void, CBAlloc> p)
static capptr::AllocFull<FreeObject> make(capptr::AllocFull<void> p)
{
return p.template as_static<FreeObject>();
}
@@ -86,8 +86,10 @@ namespace snmalloc
/**
* Encode next
*/
inline static CapPtr<FreeObject, CBAlloc> encode_next(
address_t curr, CapPtr<FreeObject, CBAlloc> next, const FreeListKey& key)
inline static capptr::AllocFull<FreeObject> encode_next(
address_t curr,
capptr::AllocFull<FreeObject> next,
const FreeListKey& key)
{
// Note we can consider other encoding schemes here.
// * XORing curr and next. This doesn't require any key material
@@ -98,7 +100,8 @@ namespace snmalloc
if constexpr (CHECK_CLIENT && !aal_supports<StrictProvenance>)
{
return CapPtr<FreeObject, CBAlloc>(address_cast(next) ^ key.key_next);
return capptr::AllocFull<FreeObject>(reinterpret_cast<FreeObject*>(
reinterpret_cast<uintptr_t>(next.unsafe_ptr()) ^ key.key_next));
}
else
{
@@ -115,9 +118,9 @@ namespace snmalloc
* optimization for repeated snoc operations (in which
* next->next_object is nullptr).
*/
static CapPtr<FreeObject, CBAlloc>* store_next(
CapPtr<FreeObject, CBAlloc>* curr,
CapPtr<FreeObject, CBAlloc> next,
static capptr::AllocFull<FreeObject>* store_next(
capptr::AllocFull<FreeObject>* curr,
capptr::AllocFull<FreeObject> next,
const FreeListKey& key)
{
#ifdef SNMALLOC_CHECK_CLIENT
@@ -131,7 +134,7 @@ namespace snmalloc
}
static void
store_null(CapPtr<FreeObject, CBAlloc>* curr, const FreeListKey& key)
store_null(capptr::AllocFull<FreeObject>* curr, const FreeListKey& key)
{
*curr = encode_next(address_cast(curr), nullptr, key);
}
@@ -141,8 +144,8 @@ namespace snmalloc
*
* Uses the atomic view of next, so can be used in the message queues.
*/
void
atomic_store_next(CapPtr<FreeObject, CBAlloc> next, const FreeListKey& key)
void atomic_store_next(
capptr::AllocFull<FreeObject> next, const FreeListKey& key)
{
#ifdef SNMALLOC_CHECK_CLIENT
next->prev_encoded =
@@ -164,7 +167,7 @@ namespace snmalloc
std::memory_order_relaxed);
}
CapPtr<FreeObject, CBAlloc> atomic_read_next(const FreeListKey& key)
capptr::AllocFull<FreeObject> atomic_read_next(const FreeListKey& key)
{
auto n = encode_next(
address_cast(&next_object),
@@ -194,7 +197,7 @@ namespace snmalloc
/**
* Read the next pointer
*/
CapPtr<FreeObject, CBAlloc> read_next(const FreeListKey& key)
capptr::AllocFull<FreeObject> read_next(const FreeListKey& key)
{
return encode_next(address_cast(&next_object), next_object, key);
}
@@ -211,14 +214,14 @@ namespace snmalloc
*/
class FreeListIter
{
CapPtr<FreeObject, CBAlloc> curr{nullptr};
capptr::AllocFull<FreeObject> curr{nullptr};
#ifdef SNMALLOC_CHECK_CLIENT
address_t prev{0};
#endif
public:
constexpr FreeListIter(
CapPtr<FreeObject, CBAlloc> head, address_t prev_value)
capptr::AllocFull<FreeObject> head, address_t prev_value)
: curr(head)
{
#ifdef SNMALLOC_CHECK_CLIENT
@@ -240,7 +243,7 @@ namespace snmalloc
/**
* Returns current head without affecting the iterator.
*/
CapPtr<FreeObject, CBAlloc> peek()
capptr::AllocFull<FreeObject> peek()
{
return curr;
}
@@ -248,7 +251,7 @@ namespace snmalloc
/**
* Moves the iterator on, and returns the current value.
*/
CapPtr<FreeObject, CBAlloc> take(const FreeListKey& key)
capptr::AllocFull<FreeObject> take(const FreeListKey& key)
{
auto c = curr;
auto next = curr->read_next(key);
@@ -289,11 +292,11 @@ namespace snmalloc
static constexpr size_t LENGTH = RANDOM ? 2 : 1;
// Pointer to the first element.
std::array<CapPtr<FreeObject, CBAlloc>, LENGTH> head;
std::array<capptr::AllocFull<FreeObject>, LENGTH> head;
// Pointer to the reference to the last element.
// In the empty case end[i] == &head[i]
// This enables branch free enqueuing.
std::array<CapPtr<FreeObject, CBAlloc>*, LENGTH> end{nullptr};
std::array<capptr::AllocFull<FreeObject>*, LENGTH> end{nullptr};
std::array<uint16_t, RANDOM ? 2 : 0> length{};
@@ -321,7 +324,7 @@ namespace snmalloc
* Adds an element to the builder
*/
void add(
CapPtr<FreeObject, CBAlloc> n,
capptr::AllocFull<FreeObject> n,
const FreeListKey& key,
LocalEntropy& entropy)
{
@@ -348,7 +351,7 @@ namespace snmalloc
*/
template<bool RANDOM_ = RANDOM>
std::enable_if_t<!RANDOM_>
add(CapPtr<FreeObject, CBAlloc> n, const FreeListKey& key)
add(capptr::AllocFull<FreeObject> n, const FreeListKey& key)
{
static_assert(RANDOM_ == RANDOM, "Don't set template parameter");
end[0] = FreeObject::store_next(end[0], n, key);
@@ -372,7 +375,7 @@ namespace snmalloc
* and is thus subject to encoding if the next_object pointers
* encoded.
*/
CapPtr<FreeObject, CBAlloc>
capptr::AllocFull<FreeObject>
read_head(uint32_t index, const FreeListKey& key)
{
return FreeObject::encode_next(
@@ -444,7 +447,7 @@ namespace snmalloc
template<bool RANDOM_ = RANDOM>
std::enable_if_t<
!RANDOM_,
std::pair<CapPtr<FreeObject, CBAlloc>, CapPtr<FreeObject, CBAlloc>>>
std::pair<capptr::AllocFull<FreeObject>, capptr::AllocFull<FreeObject>>>
extract_segment(const FreeListKey& key)
{
static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!");
@@ -456,7 +459,7 @@ namespace snmalloc
// to the actual object. This isn't true if the builder is
// empty, but you are not allowed to call this in the empty case.
auto last =
CapPtr<FreeObject, CBAlloc>(reinterpret_cast<FreeObject*>(end[0]));
capptr::AllocFull<FreeObject>(reinterpret_cast<FreeObject*>(end[0]));
init();
return {first, last};
}

8
src/mem/localalloc.h
View File

@@ -263,7 +263,9 @@ namespace snmalloc
MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry(
core_alloc->backend_state_ptr(), address_cast(p));
local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
entry.get_remote()->trunc_id(), CapPtr<void, CBAlloc>(p), key_global);
entry.get_remote()->trunc_id(),
capptr::AllocFull<void>(p),
key_global);
post_remote_cache();
return;
}
@@ -469,7 +471,7 @@ namespace snmalloc
{
local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
entry.get_remote()->trunc_id(),
CapPtr<void, CBAlloc>(p),
capptr::AllocFull<void>(p),
key_global);
# ifdef SNMALLOC_TRACING
std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p)
@@ -501,7 +503,7 @@ namespace snmalloc
# endif
ChunkRecord* slab_record =
reinterpret_cast<ChunkRecord*>(entry.get_metaslab());
slab_record->chunk = CapPtr<void, CBChunk>(p);
slab_record->chunk = capptr::Chunk<void>(p);
check_init(
[](
CoreAlloc* core_alloc,

6
src/mem/localcache.h
View File

@@ -13,19 +13,19 @@ namespace snmalloc
using Stats = AllocStats<NUM_SIZECLASSES, NUM_LARGE_CLASSES>;
inline static SNMALLOC_FAST_PATH void*
finish_alloc_no_zero(CapPtr<FreeObject, CBAlloc> p, sizeclass_t sizeclass)
finish_alloc_no_zero(capptr::AllocFull<FreeObject> p, sizeclass_t sizeclass)
{
SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p)));
UNUSED(sizeclass);
auto r = capptr_reveal(capptr_export(p.as_void()));
auto r = capptr_reveal(capptr_to_user_address_control(p.as_void()));
return r;
}
template<ZeroMem zero_mem, typename SharedStateHandle>
inline static SNMALLOC_FAST_PATH void*
finish_alloc(CapPtr<FreeObject, CBAlloc> p, sizeclass_t sizeclass)
finish_alloc(capptr::AllocFull<FreeObject> p, sizeclass_t sizeclass)
{
auto r = finish_alloc_no_zero(p, sizeclass);

3
src/mem/metaslab.h
View File

@@ -5,7 +5,6 @@
#include "../ds/seqqueue.h"
#include "../mem/remoteallocator.h"
#include "freelist.h"
#include "ptrhelpers.h"
#include "sizeclasstable.h"
namespace snmalloc
@@ -153,7 +152,7 @@ namespace snmalloc
* component, but with randomisation, it may only return part of the
* available objects for this metaslab.
*/
static SNMALLOC_FAST_PATH std::pair<CapPtr<FreeObject, CBAlloc>, bool>
static SNMALLOC_FAST_PATH std::pair<capptr::AllocFull<FreeObject>, bool>
alloc_free_list(
Metaslab* meta,
FreeListIter& fast_free_list,

98
src/mem/ptrhelpers.h
View File

@@ -1,98 +0,0 @@
#pragma once
#include "../aal/aal.h"
#include "../ds/ptrwrap.h"
#include "allocconfig.h"
namespace snmalloc
{
/*
* At various points, we do pointer math on high-authority pointers to find
* some metadata. `capptr_bound_chunkd` and `capptr_chunk_from_chunkd`
* encapsulate the notion that the result of these accesses is left unbounded
* in non-debug builds, because most codepaths do not reveal these pointers or
* any progeny to the application. However, in some cases we have already
* (partially) bounded these high-authority pointers (to CBChunk) and wish to
* preserve this annotation (rather than always returning a CBChunkD-annotated
* pointer); `capptr_bound_chunkd_bounds` does the computation for us and is
* used in the signatures of below and in those of wrappers around them.
*/
template<capptr_bounds B>
constexpr capptr_bounds capptr_bound_chunkd_bounds()
{
switch (B)
{
case CBArena:
return CBChunkD;
case CBChunkD:
return CBChunkD;
case CBChunk:
return CBChunk;
}
}
/**
* Construct an CapPtr<T, CBChunkD> from an CapPtr<T, CBArena> or
* CapPtr<T, CBChunkD> input. For an CapPtr<T, CBChunk> input, simply pass
* it through (preserving the static notion of bounds).
*
* Applies bounds on debug builds, otherwise is just sleight of hand.
*
* Requires that `p` point at a multiple of `sz` (that is, at the base of a
* highly-aligned object) to avoid representability issues.
*/
template<typename T, capptr_bounds B>
SNMALLOC_FAST_PATH CapPtr<T, capptr_bound_chunkd_bounds<B>()>
capptr_bound_chunkd(CapPtr<T, B> p, size_t sz)
{
static_assert(B == CBArena || B == CBChunkD || B == CBChunk);
SNMALLOC_ASSERT((address_cast(p) % sz) == 0);
#ifndef NDEBUG
// On Debug builds, apply bounds if not already there
if constexpr (B == CBArena)
return Aal::capptr_bound<T, CBChunkD>(p, sz);
else // quiesce MSVC's warnings about unreachable code below
#endif
{
UNUSED(sz);
return CapPtr<T, capptr_bound_chunkd_bounds<B>()>(p.unsafe_ptr());
}
}
/**
* Apply bounds that might not have been applied when constructing an
* CapPtr<T, CBChunkD>. That is, on non-debug builds, apply bounds; debug
* builds have already had them applied.
*
* Requires that `p` point at a multiple of `sz` (that is, at the base of a
* highly-aligned object) to avoid representability issues.
*/
template<typename T>
SNMALLOC_FAST_PATH CapPtr<T, CBChunk>
capptr_chunk_from_chunkd(CapPtr<T, CBChunkD> p, size_t sz)
{
SNMALLOC_ASSERT((address_cast(p) % sz) == 0);
#ifndef NDEBUG
// On debug builds, CBChunkD are already bounded as if CBChunk.
UNUSED(sz);
return CapPtr<T, CBChunk>(p.unsafe_ptr());
#else
// On non-debug builds, apply bounds now, as they haven't been already.
return Aal::capptr_bound<T, CBChunk>(p, sz);
#endif
}
/**
* Very rarely, while debugging, it's both useful and acceptable to forget
* that we have applied chunk bounds to something.
*/
template<typename T>
SNMALLOC_FAST_PATH CapPtr<T, CBChunkD>
capptr_debug_chunkd_from_chunk(CapPtr<T, CBChunk> p)
{
return CapPtr<T, CBChunkD>(p.unsafe_ptr());
}
} // namespace snmalloc

27
src/mem/remoteallocator.h
View File

@@ -35,10 +35,10 @@ namespace snmalloc
// Store the message queue on a separate cacheline. It is mutable data that
// is read by other threads.
alignas(CACHELINE_SIZE) AtomicCapPtr<FreeObject, CBAlloc> back{nullptr};
alignas(CACHELINE_SIZE) capptr::AtomicAllocFull<FreeObject> back{nullptr};
// Store the two ends on different cache lines as access by different
// threads.
alignas(CACHELINE_SIZE) CapPtr<FreeObject, CBAlloc> front{nullptr};
alignas(CACHELINE_SIZE) capptr::AllocFull<FreeObject> front{nullptr};
constexpr RemoteAllocator() = default;
@@ -48,7 +48,7 @@ namespace snmalloc
SNMALLOC_ASSERT(front != nullptr);
}
void init(CapPtr<FreeObject, CBAlloc> stub)
void init(capptr::AllocFull<FreeObject> stub)
{
stub->atomic_store_null(key_global);
front = stub;
@@ -56,9 +56,9 @@ namespace snmalloc
invariant();
}
CapPtr<FreeObject, CBAlloc> destroy()
capptr::AllocFull<FreeObject> destroy()
{
CapPtr<FreeObject, CBAlloc> fnt = front;
capptr::AllocFull<FreeObject> fnt = front;
back.store(nullptr, std::memory_order_relaxed);
front = nullptr;
return fnt;
@@ -66,7 +66,7 @@ namespace snmalloc
inline bool is_empty()
{
CapPtr<FreeObject, CBAlloc> bk = back.load(std::memory_order_relaxed);
capptr::AllocFull<FreeObject> bk = back.load(std::memory_order_relaxed);
return bk == front;
}
@@ -76,21 +76,21 @@ namespace snmalloc
* last should be linked together through their next pointers.
*/
void enqueue(
CapPtr<FreeObject, CBAlloc> first,
CapPtr<FreeObject, CBAlloc> last,
capptr::AllocFull<FreeObject> first,
capptr::AllocFull<FreeObject> last,
const FreeListKey& key)
{
invariant();
last->atomic_store_null(key);
// exchange needs to be a release, so nullptr in next is visible.
CapPtr<FreeObject, CBAlloc> prev =
capptr::AllocFull<FreeObject> prev =
back.exchange(last, std::memory_order_release);
prev->atomic_store_next(first, key);
}
CapPtr<FreeObject, CBAlloc> peek()
capptr::AllocFull<FreeObject> peek()
{
return front;
}
@@ -98,11 +98,12 @@ namespace snmalloc
/**
* Returns the front message, or null if not possible to return a message.
*/
std::pair<CapPtr<FreeObject, CBAlloc>, bool> dequeue(const FreeListKey& key)
std::pair<capptr::AllocFull<FreeObject>, bool>
dequeue(const FreeListKey& key)
{
invariant();
CapPtr<FreeObject, CBAlloc> first = front;
CapPtr<FreeObject, CBAlloc> next = first->atomic_read_next(key);
capptr::AllocFull<FreeObject> first = front;
capptr::AllocFull<FreeObject> next = first->atomic_read_next(key);
if (next != nullptr)
{

2
src/mem/remotecache.h
View File

@@ -66,7 +66,7 @@ namespace snmalloc
template<size_t allocator_size>
SNMALLOC_FAST_PATH void dealloc(
RemoteAllocator::alloc_id_t target_id,
CapPtr<void, CBAlloc> p,
capptr::AllocFull<void> p,
const FreeListKey& key)
{
SNMALLOC_ASSERT(initialised);

6
src/mem/slaballocator.h
View File

@@ -18,7 +18,7 @@ namespace snmalloc
struct ChunkRecord
{
std::atomic<ChunkRecord*> next;
CapPtr<void, CBChunk> chunk;
capptr::Chunk<void> chunk;
};
/**
@@ -76,7 +76,7 @@ namespace snmalloc
{
public:
template<SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle>
static std::pair<CapPtr<void, CBChunk>, Metaslab*> alloc_chunk(
static std::pair<capptr::Chunk<void>, Metaslab*> alloc_chunk(
typename SharedStateHandle::LocalState& local_state,
sizeclass_t sizeclass,
sizeclass_t slab_sizeclass, // TODO sizeclass_t
@@ -163,7 +163,7 @@ namespace snmalloc
// Cache line align
size_t size = bits::align_up(sizeof(U), 64);
CapPtr<void, CBChunk> p =
capptr::Chunk<void> p =
SharedStateHandle::template alloc_meta_data<U>(local_state, size);
if (p == nullptr)