Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Overhaul CapPtr

Browse Source 
* Switch to a multidimensional taxonomy.

  Rather than encoding the abstract bound states in a single enum, move to a
  more algebraic treatment.  The dimensions themselves are within the
  snmalloc::capptr_bounds namespace so that their fairly generic names do not
  conflict with consumer code.  Aliases for many points in the space are
  established outside that namespace for ease of use elsewhere.

* Introduce several new namespaces:

    * snmalloc::capptr::dimension holds each of the dimension enums

    * snmalloc::capptr holds the bound<> type itself and a ConceptBound

    * snmalloc::capptr::bounds gives convenient specializations of bound<>

    * snmalloc::capptr also has aliases for CapPtr<> itself

  All told, rather than `CapPtr<T, CBChunk>`, we now expect client code to read
  `capptr::Chunk<T>` in almost all cases (and this is just an alias for the
  appropriate `CapPtr<T, bounds<...>>` type).  When the bound<>s themselves are
  necessary, as when calling capptr_bound, we expect that they will almost
  always be pronounced using an alias (e.g., `capptr::bounds::Alloc`).

* Chase consequences.

* Prune old taxa and aliases that are no longer in use in snmalloc2.
This commit is contained in:
Nathaniel Wesley Filardo
2021-07-19 09:49:18 +01:00
committed by Nathaniel Wesley Filardo
parent b57390663e
commit 9065893181
17 changed files with 399 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/aal/aal.h
View File

@@ -189,32 +189,33 @@ namespace snmalloc
*/
template<
typename T,
enum capptr_bounds nbounds,
enum capptr_bounds obounds,
SNMALLOC_CONCEPT(capptr::ConceptBound) BOut,
SNMALLOC_CONCEPT(capptr::ConceptBound) BIn,
typename U = T>
static SNMALLOC_FAST_PATH CapPtr<T, nbounds>
capptr_bound(CapPtr<U, obounds> a, size_t size) noexcept
static SNMALLOC_FAST_PATH CapPtr<T, BOut>
capptr_bound(CapPtr<U, BIn> a, size_t size) noexcept
{
// Impose constraints on bounds annotations.
static_assert(
obounds == CBArena || obounds == CBChunkD || obounds == CBChunk ||
obounds == CBChunkE);
static_assert(capptr_is_bounds_refinement<obounds, nbounds>());
static_assert(BIn::spatial >= capptr::dimension::Spatial::Chunk);
static_assert(capptr_is_spatial_refinement<BIn, BOut>());
UNUSED(size);
return CapPtr<T, nbounds>(a.template as_static<T>().unsafe_ptr());
return CapPtr<T, BOut>(a.template as_static<T>().unsafe_capptr);
}
/**
* For architectures which do not enforce StrictProvenance, there's nothing
* to be done, so just return the pointer unmodified with new annotation.
*/
template<typename T, capptr_bounds BOut, capptr_bounds BIn>
template<
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) BOut,
SNMALLOC_CONCEPT(capptr::ConceptBound) BIn>
static SNMALLOC_FAST_PATH CapPtr<T, BOut>
capptr_rebound(CapPtr<void, BOut> a, CapPtr<T, BIn> r) noexcept
{
UNUSED(a);
return CapPtr<T, BOut>(r.unsafe_ptr());
return CapPtr<T, BOut>(r.unsafe_capptr);
}
};
} // namespace snmalloc

9
src/aal/aal_concept.h
View File

@@ -43,21 +43,22 @@ namespace snmalloc
template<typename AAL>
concept ConceptAAL_capptr_methods =
requires(CapPtr<void, CBArena> auth, CapPtr<void, CBAlloc> ret, size_t sz)
requires(capptr::Chunk<void> auth, capptr::AllocFull<void> ret, size_t sz)
{
/**
* Produce a pointer with reduced authority from a more privilged pointer.
* The resulting pointer will have base at auth's address and length of
* exactly sz. auth+sz must not exceed auth's limit.
*/
{ AAL::template capptr_bound<void, CBChunk>(auth, sz) } noexcept
-> ConceptSame<CapPtr<void, CBChunk>>;
{ AAL::template capptr_bound<void, capptr::bounds::Chunk>(auth, sz) }
noexcept
-> ConceptSame<capptr::Chunk<void>>;
/**
* Construct a copy of auth with its target set to that of ret.
*/
{ AAL::capptr_rebound(auth, ret) } noexcept
-> ConceptSame<CapPtr<void, CBArena>>;
-> ConceptSame<capptr::Chunk<void>>;
};
template<typename AAL>

18
src/backend/address_space.h
View File

@@ -43,7 +43,7 @@ namespace snmalloc
* arena_map for use in subsequent amplification.
*/
template<bool committed, SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap>
CapPtr<void, CBChunk>
capptr::Chunk<void>
reserve(typename Pagemap::LocalState* local_state, size_t size)
{
#ifdef SNMALLOC_TRACING
@@ -62,21 +62,21 @@ namespace snmalloc
{
if (size >= PAL::minimum_alloc_size)
{
auto base = CapPtr<void, CBChunk>(
PAL::template reserve_aligned<committed>(size));
auto base =
capptr::Chunk<void>(PAL::template reserve_aligned<committed>(size));
Pagemap::register_range(local_state, address_cast(base), size);
return base;
}
}
CapPtr<void, CBChunk> res;
capptr::Chunk<void> res;
{
FlagLock lock(spin_lock);
res = core.template reserve<PAL, Pagemap>(local_state, size);
if (res == nullptr)
{
// Allocation failed ask OS for more memory
CapPtr<void, CBChunk> block = nullptr;
capptr::Chunk<void> block = nullptr;
size_t block_size = 0;
if constexpr (pal_supports<AlignedAllocation, PAL>)
{
@@ -92,7 +92,7 @@ namespace snmalloc
// It's a bit of a lie to convert without applying bounds, but the
// platform will have bounded block for us and it's better that
// the rest of our internals expect CBChunk bounds.
block = CapPtr<void, CBChunk>(block_raw);
block = capptr::Chunk<void>(block_raw);
}
else if constexpr (!pal_supports<NoAllocation, PAL>)
{
@@ -110,7 +110,7 @@ namespace snmalloc
size_request >= needed_size;
size_request = size_request / 2)
{
block = CapPtr<void, CBChunk>(PAL::reserve(size_request));
block = capptr::Chunk<void>(PAL::reserve(size_request));
if (block != nullptr)
{
block_size = size_request;
@@ -158,7 +158,7 @@ namespace snmalloc
* used, by smaller objects.
*/
template<bool committed, SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap>
CapPtr<void, CBChunk> reserve_with_left_over(
capptr::Chunk<void> reserve_with_left_over(
typename Pagemap::LocalState* local_state, size_t size)
{
SNMALLOC_ASSERT(size >= sizeof(void*));
@@ -198,7 +198,7 @@ namespace snmalloc
template<SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap>
void add_range(
typename Pagemap::LocalState* local_state,
CapPtr<void, CBChunk> base,
capptr::Chunk<void> base,
size_t length)
{
FlagLock lock(spin_lock);

35
src/backend/address_space_core.h
View File

@@ -25,7 +25,7 @@ namespace snmalloc
{
struct FreeChunk
{
CapPtr<FreeChunk, CBChunk> next;
capptr::Chunk<FreeChunk> next;
};
/**
@@ -43,12 +43,12 @@ namespace snmalloc
* bits::BITS is used for simplicity, we do not use below the pointer size,
* and large entries will be unlikely to be supported by the platform.
*/
std::array<CapPtr<FreeChunk, CBChunk>, bits::BITS> ranges = {};
std::array<capptr::Chunk<FreeChunk>, bits::BITS> ranges = {};
/**
* Checks a block satisfies its invariant.
*/
inline void check_block(CapPtr<FreeChunk, CBChunk> base, size_t align_bits)
inline void check_block(capptr::Chunk<FreeChunk> base, size_t align_bits)
{
SNMALLOC_ASSERT(
address_cast(base) ==
@@ -72,8 +72,8 @@ namespace snmalloc
void set_next(
typename Pagemap::LocalState* local_state,
size_t align_bits,
CapPtr<FreeChunk, CBChunk> base,
CapPtr<FreeChunk, CBChunk> next)
capptr::Chunk<FreeChunk> base,
capptr::Chunk<FreeChunk> next)
{
if (align_bits >= MIN_CHUNK_BITS)
{
@@ -102,16 +102,16 @@ namespace snmalloc
* particular size.
*/
template<SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap>
CapPtr<FreeChunk, CBChunk> get_next(
capptr::Chunk<FreeChunk> get_next(
typename Pagemap::LocalState* local_state,
size_t align_bits,
CapPtr<FreeChunk, CBChunk> base)
capptr::Chunk<FreeChunk> base)
{
if (align_bits >= MIN_CHUNK_BITS)
{
const MetaEntry& t = Pagemap::template get_metaentry<false>(
local_state, address_cast(base));
return CapPtr<FreeChunk, CBChunk>(
return capptr::Chunk<FreeChunk>(
reinterpret_cast<FreeChunk*>(t.get_metaslab()));
}
@@ -127,7 +127,7 @@ namespace snmalloc
void add_block(
typename Pagemap::LocalState* local_state,
size_t align_bits,
CapPtr<FreeChunk, CBChunk> base)
capptr::Chunk<FreeChunk> base)
{
check_block(base, align_bits);
SNMALLOC_ASSERT(align_bits < 64);
@@ -143,10 +143,10 @@ namespace snmalloc
template<
SNMALLOC_CONCEPT(ConceptPAL) PAL,
SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap>
CapPtr<void, CBChunk>
capptr::Chunk<void>
remove_block(typename Pagemap::LocalState* local_state, size_t align_bits)
{
CapPtr<FreeChunk, CBChunk> first = ranges[align_bits];
capptr::Chunk<FreeChunk> first = ranges[align_bits];
if (first == nullptr)
{
if (align_bits == (bits::BITS - 1))
@@ -156,7 +156,7 @@ namespace snmalloc
}
// Look for larger block and split up recursively
CapPtr<void, CBChunk> bigger =
capptr::Chunk<void> bigger =
remove_block<PAL, Pagemap>(local_state, align_bits + 1);
if (bigger != nullptr)
{
@@ -174,7 +174,8 @@ namespace snmalloc
add_block<PAL, Pagemap>(
local_state,
align_bits,
Aal::capptr_bound<FreeChunk, CBChunk>(left_over, left_over_size));
Aal::capptr_bound<FreeChunk, capptr::bounds::Chunk>(
left_over, left_over_size));
check_block(left_over.as_static<FreeChunk>(), align_bits);
}
check_block(bigger.as_static<FreeChunk>(), align_bits + 1);
@@ -196,7 +197,7 @@ namespace snmalloc
SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap>
void add_range(
typename Pagemap::LocalState* local_state,
CapPtr<void, CBChunk> base,
capptr::Chunk<void> base,
size_t length)
{
// For start and end that are not chunk sized, we need to
@@ -233,7 +234,7 @@ namespace snmalloc
* Commit a block of memory
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
void commit_block(CapPtr<void, CBChunk> base, size_t size)
void commit_block(capptr::Chunk<void> base, size_t size)
{
// Rounding required for sub-page allocations.
auto page_start = pointer_align_down<OS_PAGE_SIZE, char>(base);
@@ -257,7 +258,7 @@ namespace snmalloc
template<
SNMALLOC_CONCEPT(ConceptPAL) PAL,
SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap>
CapPtr<void, CBChunk>
capptr::Chunk<void>
reserve(typename Pagemap::LocalState* local_state, size_t size)
{
#ifdef SNMALLOC_TRACING
@@ -281,7 +282,7 @@ namespace snmalloc
template<
SNMALLOC_CONCEPT(ConceptPAL) PAL,
SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap>
CapPtr<void, CBChunk> reserve_with_left_over(
capptr::Chunk<void> reserve_with_left_over(
typename Pagemap::LocalState* local_state, size_t size)
{
SNMALLOC_ASSERT(size >= sizeof(void*));

20
src/backend/backend.h
View File

@@ -39,7 +39,7 @@ namespace snmalloc
* Backend allocator may use guard pages and separate area of
* address space to protect this from corruption.
*/
static CapPtr<void, CBChunk> alloc_meta_data(
static capptr::Chunk<void> alloc_meta_data(
AddressSpaceManager<PAL>& global, LocalState* local_state, size_t size)
{
return reserve<true>(global, local_state, size);
@@ -54,7 +54,7 @@ namespace snmalloc
* (remote, sizeclass, metaslab)
* where metaslab, is the second element of the pair return.
*/
static std::pair<CapPtr<void, CBChunk>, Metaslab*> alloc_chunk(
static std::pair<capptr::Chunk<void>, Metaslab*> alloc_chunk(
AddressSpaceManager<PAL>& global,
LocalState* local_state,
size_t size,
@@ -70,7 +70,7 @@ namespace snmalloc
if (meta == nullptr)
return {nullptr, nullptr};
CapPtr<void, CBChunk> p = reserve<false>(global, local_state, size);
capptr::Chunk<void> p = reserve<false>(global, local_state, size);
#ifdef SNMALLOC_TRACING
std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")"
@@ -98,7 +98,7 @@ namespace snmalloc
* space managers.
*/
template<bool is_meta>
static CapPtr<void, CBChunk> reserve(
static capptr::Chunk<void> reserve(
AddressSpaceManager<PAL>& global, LocalState* local_state, size_t size)
{
#ifdef SNMALLOC_CHECK_CLIENT
@@ -108,7 +108,7 @@ namespace snmalloc
constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK;
#endif
CapPtr<void, CBChunk> p;
capptr::Chunk<void> p;
if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE))
{
#ifdef SNMALLOC_CHECK_CLIENT
@@ -182,8 +182,8 @@ namespace snmalloc
* the range [base, base+full_size]. The first and last slot are not used
* so that the edges can be used for guard pages.
*/
static CapPtr<void, CBChunk>
sub_range(CapPtr<void, CBChunk> base, size_t full_size, size_t sub_size)
static capptr::Chunk<void>
sub_range(capptr::Chunk<void> base, size_t full_size, size_t sub_size)
{
SNMALLOC_ASSERT(bits::is_pow2(full_size));
SNMALLOC_ASSERT(bits::is_pow2(sub_size));
@@ -318,7 +318,7 @@ namespace snmalloc
auto [heap_base, heap_length] =
Pagemap::concretePagemap.init(base, length);
address_space.template add_range<Pagemap>(
local_state, CapPtr<void, CBChunk>(heap_base), heap_length);
local_state, capptr::Chunk<void>(heap_base), heap_length);
}
/**
@@ -332,7 +332,7 @@ namespace snmalloc
* places or with different policies.
*/
template<typename T>
static CapPtr<void, CBChunk>
static capptr::Chunk<void>
alloc_meta_data(LocalState* local_state, size_t size)
{
return AddressSpaceAllocator::alloc_meta_data(
@@ -348,7 +348,7 @@ namespace snmalloc
* (remote, sizeclass, metaslab)
* where metaslab, is the second element of the pair return.
*/
static std::pair<CapPtr<void, CBChunk>, Metaslab*> alloc_chunk(
static std::pair<capptr::Chunk<void>, Metaslab*> alloc_chunk(
LocalState* local_state,
size_t size,
RemoteAllocator* remote,

28
src/ds/address.h
View File

@@ -35,7 +35,7 @@ namespace snmalloc
reinterpret_cast<uintptr_t>(base) + static_cast<uintptr_t>(diff));
}
template<enum capptr_bounds bounds, typename T>
template<SNMALLOC_CONCEPT(capptr::ConceptBound) bounds, typename T>
inline CapPtr<void, bounds>
pointer_offset(CapPtr<T, bounds> base, size_t diff)
{
@@ -52,7 +52,7 @@ namespace snmalloc
return reinterpret_cast<U*>(reinterpret_cast<char*>(base) + diff);
}
template<enum capptr_bounds bounds, typename T>
template<SNMALLOC_CONCEPT(capptr::ConceptBound) bounds, typename T>
inline CapPtr<void, bounds>
pointer_offset_signed(CapPtr<T, bounds> base, ptrdiff_t diff)
{
@@ -76,7 +76,7 @@ namespace snmalloc
* capptr_bound.
*/
template<typename T, enum capptr_bounds bounds>
template<typename T, SNMALLOC_CONCEPT(capptr::ConceptBound) bounds>
inline SNMALLOC_FAST_PATH address_t address_cast(CapPtr<T, bounds> a)
{
return address_cast(a.unsafe_ptr());
@@ -132,7 +132,10 @@ namespace snmalloc
pointer_align_down<alignment>(reinterpret_cast<uintptr_t>(p)));
}
template<size_t alignment, typename T, capptr_bounds bounds>
template<
size_t alignment,
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) bounds>
inline CapPtr<T, bounds> pointer_align_down(CapPtr<void, bounds> p)
{
return CapPtr<T, bounds>(pointer_align_down<alignment, T>(p.unsafe_ptr()));
@@ -166,7 +169,10 @@ namespace snmalloc
}
}
template<size_t alignment, typename T = void, enum capptr_bounds bounds>
template<
size_t alignment,
typename T = void,
SNMALLOC_CONCEPT(capptr::ConceptBound) bounds>
inline CapPtr<T, bounds> pointer_align_up(CapPtr<void, bounds> p)
{
return CapPtr<T, bounds>(pointer_align_up<alignment, T>(p.unsafe_ptr()));
@@ -195,7 +201,7 @@ namespace snmalloc
#endif
}
template<typename T = void, enum capptr_bounds bounds>
template<typename T = void, SNMALLOC_CONCEPT(capptr::ConceptBound) bounds>
inline CapPtr<T, bounds>
pointer_align_down(CapPtr<void, bounds> p, size_t alignment)
{
@@ -219,7 +225,7 @@ namespace snmalloc
#endif
}
template<typename T = void, enum capptr_bounds bounds>
template<typename T = void, SNMALLOC_CONCEPT(capptr::ConceptBound) bounds>
inline CapPtr<T, bounds>
pointer_align_up(CapPtr<void, bounds> p, size_t alignment)
{
@@ -241,8 +247,8 @@ namespace snmalloc
template<
typename T = void,
typename U = void,
enum capptr_bounds Tbounds,
enum capptr_bounds Ubounds>
SNMALLOC_CONCEPT(capptr::ConceptBound) Tbounds,
SNMALLOC_CONCEPT(capptr::ConceptBound) Ubounds>
inline size_t pointer_diff(CapPtr<T, Tbounds> base, CapPtr<U, Ubounds> cursor)
{
return pointer_diff(base.unsafe_ptr(), cursor.unsafe_ptr());
@@ -261,8 +267,8 @@ namespace snmalloc
template<
typename T = void,
typename U = void,
enum capptr_bounds Tbounds,
enum capptr_bounds Ubounds>
SNMALLOC_CONCEPT(capptr::ConceptBound) Tbounds,
SNMALLOC_CONCEPT(capptr::ConceptBound) Ubounds>
inline ptrdiff_t
pointer_diff_signed(CapPtr<T, Tbounds> base, CapPtr<U, Ubounds> cursor)
{

339
src/ds/ptrwrap.h
View File

@@ -1,6 +1,7 @@
#pragma once
#include "defines.h"
#include "../ds/concept.h"
#include "../ds/defines.h"
#include <atomic>
@@ -21,78 +22,150 @@ namespace snmalloc
/**
* Summaries of StrictProvenance metadata. We abstract away the particular
* size and any offset into the bounds.
*
* CBArena is as powerful as our pointers get: they're results from mmap(),
* and so confer as much authority as the kernel has given us.
*
* CBChunk is restricted to either a single chunk (SUPERSLAB_SIZE) or perhaps
* to several if we've requesed a large allocation (see capptr_chunk_is_alloc
* and its uses).
*
* CBChunkD is curious: we often use CBArena-bounded pointers to derive
* pointers to Allocslab metadata, and on most fast paths these pointers end
* up being ephemeral. As such, on NDEBUG builds, we elide the capptr_bounds
* that would bound these to chunks and instead just unsafely inherit the
* CBArena bounds. The use of CBChunkD thus helps to ensure that we
* eventually do invoke capptr_bounds when these pointers end up being longer
* lived!
*
* *E forms are "exported" and have had platform constraints applied. That
* means, for example, on CheriBSD, that they have had their VMMAP permission
* stripped.
*
* Yes, I wish the start-of-comment characters were aligned below as well.
* I blame clang format.
*/
enum capptr_bounds
namespace capptr
{
/* Spatial Notes */
CBArena, /* Arena */
CBChunkD, /* Arena Chunk-bounded in debug; internal use only! */
CBChunk, /* Chunk */
CBChunkE, /* Chunk (+ platform constraints) */
CBAlloc, /* Alloc */
CBAllocE /* Alloc (+ platform constraints) */
};
namespace dimension
{
/*
* Describe the spatial extent (intended to be) authorized by a pointer.
*
* Bounds dimensions are sorted so that < reflects authority.
*/
enum class Spatial
{
/**
* Bounded to a particular allocation (which might be Large!)
*/
Alloc,
/**
* Bounded to one or more particular chunk granules
*/
Chunk,
};
/**
* On some platforms (e.g., CHERI), pointers can be checked to see whether
* they authorize control of the address space. See the PAL's
* capptr_export().
*/
enum class AddressSpaceControl
{
/**
* All intended control constraints have been applied. For example, on
* CheriBSD, the VMMAP permission has been stripped and so this CapPtr<>
* cannot authorize manipulation of the address space itself, though it
* continues to authorize loads and stores.
*/
User,
/**
* No control constraints have been applied. On CheriBSD, specifically,
* this grants control of the address space (via mmap and friends) and
* in Cornucopia exempts the pointer from revocation (as long as the
* mapping remains in place, but snmalloc does not presently tear down
* its own mappings.)
*/
Full
};
} // namespace dimension
/**
* The aggregate type of a bound: a Cartesian product of the individual
* dimensions, above.
*/
template<dimension::Spatial S, dimension::AddressSpaceControl AS>
struct bound
{
static constexpr enum dimension::Spatial spatial = S;
static constexpr enum dimension::AddressSpaceControl
address_space_control = AS;
/**
* Set just the spatial component of the bounds
*/
template<enum dimension::Spatial SO>
using with_spatial = bound<SO, AS>;
/**
* Set just the address space control component of the bounds
*/
template<enum dimension::AddressSpaceControl ASO>
using with_address_space_control = bound<S, ASO>;
};
// clang-format off
#ifdef __cpp_concepts
/*
* This is spelled a little differently from our other concepts because GCC
* treats "{ T::spatial }" as generating a reference and then complains that
* it isn't "ConceptSame<const Spatial>", though clang is perfectly happy
* with that spelling. Both seem happy with this formulation.
*/
template<typename T>
concept ConceptBound =
ConceptSame<decltype(T::spatial), const capptr::dimension::Spatial> &&
ConceptSame<decltype(T::address_space_control),
const capptr::dimension::AddressSpaceControl>;
#endif
// clang-format on
/*
* Several combinations are used often enough that we give convenient
* aliases for them.
*/
namespace bounds
{
/**
* Internal access to a Chunk of memory. These flow between the ASM and
* the slab allocators, for example.
*/
using Chunk =
bound<dimension::Spatial::Chunk, dimension::AddressSpaceControl::Full>;
/**
* User access to an entire Chunk. Used as an ephemeral state when
* returning a large allocation. See capptr_chunk_is_alloc.
*/
using ChunkUser =
Chunk::with_address_space_control<dimension::AddressSpaceControl::User>;
/**
* Internal access to just one allocation (usually, within a slab).
*/
using AllocFull = Chunk::with_spatial<dimension::Spatial::Alloc>;
/**
* User access to just one allocation (usually, within a slab).
*/
using Alloc = AllocFull::with_address_space_control<
dimension::AddressSpaceControl::User>;
} // namespace bounds
} // namespace capptr
/**
* Compute the "exported" variant of a capptr_bounds annotation. This is
* used by the PAL's capptr_export function to compute its return value's
* annotation.
* Determine whether BI is a spatial refinement of BO.
* Chunk and ChunkD are considered eqivalent here.
*/
template<capptr_bounds B>
SNMALLOC_CONSTEVAL capptr_bounds capptr_export_type()
template<
SNMALLOC_CONCEPT(capptr::ConceptBound) BI,
SNMALLOC_CONCEPT(capptr::ConceptBound) BO>
SNMALLOC_CONSTEVAL bool capptr_is_spatial_refinement()
{
static_assert(
(B == CBChunk) || (B == CBAlloc), "capptr_export_type of bad type");
switch (B)
if (BI::address_space_control != BO::address_space_control)
{
case CBChunk:
return CBChunkE;
case CBAlloc:
return CBAllocE;
return false;
}
}
template<capptr_bounds BI, capptr_bounds BO>
SNMALLOC_CONSTEVAL bool capptr_is_bounds_refinement()
{
switch (BI)
switch (BI::spatial)
{
case CBAllocE:
return BO == CBAllocE;
case CBAlloc:
return BO == CBAlloc;
case CBChunkE:
return BO == CBAllocE || BO == CBChunkE;
case CBChunk:
return BO == CBAlloc || BO == CBChunk || BO == CBChunkD;
case CBChunkD:
return BO == CBAlloc || BO == CBChunk || BO == CBChunkD;
case CBArena:
return BO == CBAlloc || BO == CBChunk || BO == CBChunkD ||
BO == CBArena;
using namespace capptr::dimension;
case Spatial::Chunk:
return true;
case Spatial::Alloc:
return BO::spatial == Spatial::Alloc;
}
}
@@ -100,18 +173,19 @@ namespace snmalloc
* A pointer annotated with a "phantom type parameter" carrying a static
* summary of its StrictProvenance metadata.
*/
template<typename T, capptr_bounds bounds>
class CapPtr
template<typename T, SNMALLOC_CONCEPT(capptr::ConceptBound) bounds>
struct CapPtr
{
uintptr_t unsafe_capptr;
T* unsafe_capptr;
public:
/**
* nullptr is implicitly constructable at any bounds type
*/
constexpr CapPtr(const std::nullptr_t) : unsafe_capptr(0) {}
constexpr SNMALLOC_FAST_PATH CapPtr(const std::nullptr_t n)
: unsafe_capptr(n)
{}
constexpr CapPtr() : CapPtr(nullptr){};
constexpr SNMALLOC_FAST_PATH CapPtr() : CapPtr(nullptr) {}
/**
* all other constructions must be explicit
@@ -127,23 +201,21 @@ namespace snmalloc
# pragma warning(push)
# pragma warning(disable : 4702)
#endif
constexpr explicit CapPtr(uintptr_t p) : unsafe_capptr(p) {}
constexpr explicit SNMALLOC_FAST_PATH CapPtr(T* p) : unsafe_capptr(p) {}
#ifdef _MSC_VER
# pragma warning(pop)
#endif
explicit CapPtr(T* p) : unsafe_capptr(reinterpret_cast<uintptr_t>(p)) {}
/**
* Allow static_cast<>-s that preserve bounds but vary the target type.
*/
template<typename U>
SNMALLOC_FAST_PATH CapPtr<U, bounds> as_static()
[[nodiscard]] SNMALLOC_FAST_PATH CapPtr<U, bounds> as_static() const
{
return CapPtr<U, bounds>(this->unsafe_capptr);
return CapPtr<U, bounds>(static_cast<U*>(this->unsafe_capptr));
}
SNMALLOC_FAST_PATH CapPtr<void, bounds> as_void()
[[nodiscard]] SNMALLOC_FAST_PATH CapPtr<void, bounds> as_void() const
{
return this->as_static<void>();
}
@@ -152,9 +224,9 @@ namespace snmalloc
* A more aggressive bounds-preserving cast, using reinterpret_cast
*/
template<typename U>
SNMALLOC_FAST_PATH CapPtr<U, bounds> as_reinterpret()
[[nodiscard]] SNMALLOC_FAST_PATH CapPtr<U, bounds> as_reinterpret() const
{
return CapPtr<U, bounds>(this->unsafe_capptr);
return CapPtr<U, bounds>(reinterpret_cast<U*>(this->unsafe_capptr));
}
SNMALLOC_FAST_PATH bool operator==(const CapPtr& rhs) const
@@ -172,60 +244,82 @@ namespace snmalloc
return this->unsafe_capptr < rhs.unsafe_capptr;
}
[[nodiscard]] SNMALLOC_FAST_PATH T* unsafe_ptr() const
SNMALLOC_FAST_PATH T* operator->() const
{
return reinterpret_cast<T*>(this->unsafe_capptr);
/*
* Alloc-bounded, platform-constrained pointers are associated with
* objects coming from or going to the client; we should be doing nothing
* with them.
*/
static_assert(
(bounds::spatial != capptr::dimension::Spatial::Alloc) ||
(bounds::address_space_control !=
capptr::dimension::AddressSpaceControl::User));
return this->unsafe_capptr;
}
[[nodiscard]] SNMALLOC_FAST_PATH uintptr_t unsafe_uintptr() const
[[nodiscard]] SNMALLOC_FAST_PATH T* unsafe_ptr() const
{
return this->unsafe_capptr;
}
SNMALLOC_FAST_PATH T* operator->() const
[[nodiscard]] SNMALLOC_FAST_PATH uintptr_t unsafe_uintptr() const
{
/*
* CBAllocE bounds are associated with objects coming from or going to the
* client; we should be doing nothing with them.
*/
static_assert(bounds != CBAllocE);
return unsafe_ptr();
return reinterpret_cast<uintptr_t>(this->unsafe_capptr);
}
};
static_assert(sizeof(CapPtr<void, CBArena>) == sizeof(void*));
static_assert(alignof(CapPtr<void, CBArena>) == alignof(void*));
namespace capptr
{
/*
* Aliases for CapPtr<> types with particular bounds.
*/
template<typename T>
using CapPtrCBArena = CapPtr<T, CBArena>;
template<typename T>
using Chunk = CapPtr<T, bounds::Chunk>;
template<typename T>
using CapPtrCBChunk = CapPtr<T, CBChunk>;
template<typename T>
using ChunkUser = CapPtr<T, bounds::ChunkUser>;
template<typename T>
using CapPtrCBChunkE = CapPtr<T, CBChunkE>;
template<typename T>
using AllocFull = CapPtr<T, bounds::AllocFull>;
template<typename T>
using CapPtrCBAlloc = CapPtr<T, CBAlloc>;
template<typename T>
using Alloc = CapPtr<T, bounds::Alloc>;
} // namespace capptr
static_assert(sizeof(capptr::Chunk<void>) == sizeof(void*));
static_assert(alignof(capptr::Chunk<void>) == alignof(void*));
/**
* Sometimes (with large allocations) we really mean the entire chunk (or even
* several chunks) to be the allocation.
*/
template<typename T>
inline SNMALLOC_FAST_PATH CapPtr<T, CBAllocE>
capptr_chunk_is_alloc(CapPtr<T, CBChunkE> p)
inline SNMALLOC_FAST_PATH capptr::Alloc<T>
capptr_chunk_is_alloc(capptr::ChunkUser<T> p)
{
return CapPtr<T, CBAlloc>(p.unsafe_capptr);
return capptr::Alloc<T>(p.unsafe_capptr);
}
/**
* With all the bounds and constraints in place, it's safe to extract a void
* pointer (to reveal to the client).
* pointer (to reveal to the client). Roughly dual to capptr_from_client().
*/
inline SNMALLOC_FAST_PATH void* capptr_reveal(CapPtr<void, CBAllocE> p)
inline SNMALLOC_FAST_PATH void* capptr_reveal(capptr::Alloc<void> p)
{
return p.unsafe_ptr();
return p.unsafe_capptr;
}
/**
* Given a void* from the client, it's fine to call it Alloc. Roughly
* dual to capptr_reveal().
*/
static inline SNMALLOC_FAST_PATH capptr::Alloc<void>
capptr_from_client(void* p)
{
return capptr::Alloc<void>(p);
}
/**
@@ -237,7 +331,7 @@ namespace snmalloc
* annotations around an un-annotated std::atomic<T*>, to appease C++, yet
* will expose or consume only CapPtr<T> with the same bounds annotation.
*/
template<typename T, capptr_bounds bounds>
template<typename T, SNMALLOC_CONCEPT(capptr::ConceptBound) bounds>
struct AtomicCapPtr
{
std::atomic<T*> unsafe_capptr;
@@ -245,12 +339,16 @@ namespace snmalloc
/**
* nullptr is constructable at any bounds type
*/
constexpr AtomicCapPtr(const std::nullptr_t n) : unsafe_capptr(n) {}
constexpr SNMALLOC_FAST_PATH AtomicCapPtr(const std::nullptr_t n)
: unsafe_capptr(n)
{}
/**
* Interconversion with CapPtr
*/
AtomicCapPtr(CapPtr<T, bounds> p) : unsafe_capptr(p.unsafe_ptr()) {}
constexpr SNMALLOC_FAST_PATH AtomicCapPtr(CapPtr<T, bounds> p)
: unsafe_capptr(p.unsafe_capptr)
{}
operator CapPtr<T, bounds>() const noexcept
{
@@ -260,7 +358,7 @@ namespace snmalloc
// Our copy-assignment operator follows std::atomic and returns a copy of
// the RHS. Clang finds this surprising; we suppress the warning.
// NOLINTNEXTLINE(misc-unconventional-assign-operator)
CapPtr<T, bounds> operator=(CapPtr<T, bounds> p) noexcept
SNMALLOC_FAST_PATH CapPtr<T, bounds> operator=(CapPtr<T, bounds> p) noexcept
{
this->store(p);
return p;
@@ -276,7 +374,7 @@ namespace snmalloc
CapPtr<T, bounds> desired,
std::memory_order order = std::memory_order_seq_cst) noexcept
{
this->unsafe_capptr.store(desired.unsafe_ptr(), order);
this->unsafe_capptr.store(desired.unsafe_capptr, order);
}
SNMALLOC_FAST_PATH CapPtr<T, bounds> exchange(
@@ -284,7 +382,7 @@ namespace snmalloc
std::memory_order order = std::memory_order_seq_cst) noexcept
{
return CapPtr<T, bounds>(
this->unsafe_capptr.exchange(desired.unsafe_ptr(), order));
this->unsafe_capptr.exchange(desired.unsafe_capptr, order));
}
SNMALLOC_FAST_PATH bool operator==(const AtomicCapPtr& rhs) const
@@ -303,13 +401,24 @@ namespace snmalloc
}
};
template<typename T>
using AtomicCapPtrCBArena = AtomicCapPtr<T, CBArena>;
namespace capptr
{
/*
* Aliases for AtomicCapPtr<> types with particular bounds.
*/
template<typename T>
using AtomicCapPtrCBChunk = AtomicCapPtr<T, CBChunk>;
template<typename T>
using AtomicChunk = AtomicCapPtr<T, bounds::Chunk>;
template<typename T>
using AtomicCapPtrCBAlloc = AtomicCapPtr<T, CBAlloc>;
template<typename T>
using AtomicChunkUser = AtomicCapPtr<T, bounds::ChunkUser>;
template<typename T>
using AtomicAllocFull = AtomicCapPtr<T, bounds::AllocFull>;
template<typename T>
using AtomicAlloc = AtomicCapPtr<T, bounds::Alloc>;
} // namespace capptr
} // namespace snmalloc

26
src/mem/corealloc.h
View File

@@ -161,7 +161,7 @@ namespace snmalloc
{
// Manufacture an allocation to prime the queue
// Using an actual allocation removes a conditional from a critical path.
auto dummy = CapPtr<void, CBAlloc>(small_alloc_one(MIN_ALLOC_SIZE))
auto dummy = capptr::AllocFull<void>(small_alloc_one(MIN_ALLOC_SIZE))
.template as_static<FreeObject>();
if (dummy == nullptr)
{
@@ -187,7 +187,7 @@ namespace snmalloc
}
static SNMALLOC_FAST_PATH void alloc_new_list(
CapPtr<void, CBChunk>& bumpptr,
capptr::Chunk<void>& bumpptr,
Metaslab* meta,
size_t rsize,
size_t slab_size,
@@ -203,7 +203,7 @@ namespace snmalloc
// Structure to represent the temporary list elements
struct PreAllocObject
{
CapPtr<PreAllocObject, CBAlloc> next;
capptr::AllocFull<PreAllocObject> next;
};
// The following code implements Sattolo's algorithm for generating
// random cyclic permutations. This implementation is in the opposite
@@ -213,9 +213,10 @@ namespace snmalloc
// Note the wide bounds on curr relative to each of the ->next fields;
// curr is not persisted once the list is built.
CapPtr<PreAllocObject, CBChunk> curr =
capptr::Chunk<PreAllocObject> curr =
pointer_offset(bumpptr, 0).template as_static<PreAllocObject>();
curr->next = Aal::capptr_bound<PreAllocObject, CBAlloc>(curr, rsize);
curr->next = Aal::capptr_bound<PreAllocObject, capptr::bounds::AllocFull>(
curr, rsize);
uint16_t count = 1;
for (curr =
@@ -229,12 +230,13 @@ namespace snmalloc
pointer_offset(bumpptr, insert_index * rsize)
.template as_static<PreAllocObject>()
->next,
Aal::capptr_bound<PreAllocObject, CBAlloc>(curr, rsize));
Aal::capptr_bound<PreAllocObject, capptr::bounds::AllocFull>(
curr, rsize));
count++;
}
// Pick entry into space, and then build linked list by traversing cycle
// to the start. Use ->next to jump from CBArena to CBAlloc.
// to the start. Use ->next to jump from Chunk to Alloc.
auto start_index = entropy.sample(count);
auto start_ptr = pointer_offset(bumpptr, start_index * rsize)
.template as_static<PreAllocObject>()
@@ -249,7 +251,9 @@ namespace snmalloc
auto p = bumpptr;
do
{
b.add(Aal::capptr_bound<FreeObject, CBAlloc>(p, rsize), key);
b.add(
Aal::capptr_bound<FreeObject, capptr::bounds::AllocFull>(p, rsize),
key);
p = pointer_offset(p, rsize);
} while (p < slab_end);
#endif
@@ -299,7 +303,7 @@ namespace snmalloc
auto start_of_slab = pointer_align_down<void>(
p, snmalloc::sizeclass_to_slab_size(sizeclass));
// TODO Add bounds correctly here
chunk_record->chunk = CapPtr<void, CBChunk>(start_of_slab);
chunk_record->chunk = capptr::Chunk<void>(start_of_slab);
#ifdef SNMALLOC_TRACING
std::cout << "Slab " << start_of_slab << " is unused, Object sizeclass "
@@ -422,7 +426,7 @@ namespace snmalloc
* need_post will be set to true, if capacity is exceeded.
*/
void handle_dealloc_remote(
const MetaEntry& entry, CapPtr<FreeObject, CBAlloc> p, bool& need_post)
const MetaEntry& entry, capptr::AllocFull<FreeObject> p, bool& need_post)
{
// TODO this needs to not double count stats
// TODO this needs to not double revoke if using MTE
@@ -576,7 +580,7 @@ namespace snmalloc
Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)),
"Not deallocating start of an object");
auto cp = CapPtr<FreeObject, CBAlloc>(reinterpret_cast<FreeObject*>(p));
auto cp = capptr::AllocFull<FreeObject>(reinterpret_cast<FreeObject*>(p));
auto& key = entropy.get_free_list_key();

53
src/mem/freelist.h
View File

@@ -66,9 +66,9 @@ namespace snmalloc
{
union
{
CapPtr<FreeObject, CBAlloc> next_object;
capptr::AllocFull<FreeObject> next_object;
// TODO: Should really use C++20 atomic_ref rather than a union.
AtomicCapPtr<FreeObject, CBAlloc> atomic_next_object;
capptr::AtomicAllocFull<FreeObject> atomic_next_object;
};
#ifdef SNMALLOC_CHECK_CLIENT
// Encoded representation of a back pointer.
@@ -78,7 +78,7 @@ namespace snmalloc
#endif
public:
static CapPtr<FreeObject, CBAlloc> make(CapPtr<void, CBAlloc> p)
static capptr::AllocFull<FreeObject> make(capptr::AllocFull<void> p)
{
return p.template as_static<FreeObject>();
}
@@ -86,8 +86,10 @@ namespace snmalloc
/**
* Encode next
*/
inline static CapPtr<FreeObject, CBAlloc> encode_next(
address_t curr, CapPtr<FreeObject, CBAlloc> next, const FreeListKey& key)
inline static capptr::AllocFull<FreeObject> encode_next(
address_t curr,
capptr::AllocFull<FreeObject> next,
const FreeListKey& key)
{
// Note we can consider other encoding schemes here.
// * XORing curr and next. This doesn't require any key material
@@ -98,7 +100,8 @@ namespace snmalloc
if constexpr (CHECK_CLIENT && !aal_supports<StrictProvenance>)
{
return CapPtr<FreeObject, CBAlloc>(address_cast(next) ^ key.key_next);
return capptr::AllocFull<FreeObject>(reinterpret_cast<FreeObject*>(
reinterpret_cast<uintptr_t>(next.unsafe_ptr()) ^ key.key_next));
}
else
{
@@ -115,9 +118,9 @@ namespace snmalloc
* optimization for repeated snoc operations (in which
* next->next_object is nullptr).
*/
static CapPtr<FreeObject, CBAlloc>* store_next(
CapPtr<FreeObject, CBAlloc>* curr,
CapPtr<FreeObject, CBAlloc> next,
static capptr::AllocFull<FreeObject>* store_next(
capptr::AllocFull<FreeObject>* curr,
capptr::AllocFull<FreeObject> next,
const FreeListKey& key)
{
#ifdef SNMALLOC_CHECK_CLIENT
@@ -131,7 +134,7 @@ namespace snmalloc
}
static void
store_null(CapPtr<FreeObject, CBAlloc>* curr, const FreeListKey& key)
store_null(capptr::AllocFull<FreeObject>* curr, const FreeListKey& key)
{
*curr = encode_next(address_cast(curr), nullptr, key);
}
@@ -141,8 +144,8 @@ namespace snmalloc
*
* Uses the atomic view of next, so can be used in the message queues.
*/
void
atomic_store_next(CapPtr<FreeObject, CBAlloc> next, const FreeListKey& key)
void atomic_store_next(
capptr::AllocFull<FreeObject> next, const FreeListKey& key)
{
#ifdef SNMALLOC_CHECK_CLIENT
next->prev_encoded =
@@ -164,7 +167,7 @@ namespace snmalloc
std::memory_order_relaxed);
}
CapPtr<FreeObject, CBAlloc> atomic_read_next(const FreeListKey& key)
capptr::AllocFull<FreeObject> atomic_read_next(const FreeListKey& key)
{
auto n = encode_next(
address_cast(&next_object),
@@ -194,7 +197,7 @@ namespace snmalloc
/**
* Read the next pointer
*/
CapPtr<FreeObject, CBAlloc> read_next(const FreeListKey& key)
capptr::AllocFull<FreeObject> read_next(const FreeListKey& key)
{
return encode_next(address_cast(&next_object), next_object, key);
}
@@ -211,14 +214,14 @@ namespace snmalloc
*/
class FreeListIter
{
CapPtr<FreeObject, CBAlloc> curr{nullptr};
capptr::AllocFull<FreeObject> curr{nullptr};
#ifdef SNMALLOC_CHECK_CLIENT
address_t prev{0};
#endif
public:
constexpr FreeListIter(
CapPtr<FreeObject, CBAlloc> head, address_t prev_value)
capptr::AllocFull<FreeObject> head, address_t prev_value)
: curr(head)
{
#ifdef SNMALLOC_CHECK_CLIENT
@@ -240,7 +243,7 @@ namespace snmalloc
/**
* Returns current head without affecting the iterator.
*/
CapPtr<FreeObject, CBAlloc> peek()
capptr::AllocFull<FreeObject> peek()
{
return curr;
}
@@ -248,7 +251,7 @@ namespace snmalloc
/**
* Moves the iterator on, and returns the current value.
*/
CapPtr<FreeObject, CBAlloc> take(const FreeListKey& key)
capptr::AllocFull<FreeObject> take(const FreeListKey& key)
{
auto c = curr;
auto next = curr->read_next(key);
@@ -289,11 +292,11 @@ namespace snmalloc
static constexpr size_t LENGTH = RANDOM ? 2 : 1;
// Pointer to the first element.
std::array<CapPtr<FreeObject, CBAlloc>, LENGTH> head;
std::array<capptr::AllocFull<FreeObject>, LENGTH> head;
// Pointer to the reference to the last element.
// In the empty case end[i] == &head[i]
// This enables branch free enqueuing.
std::array<CapPtr<FreeObject, CBAlloc>*, LENGTH> end{nullptr};
std::array<capptr::AllocFull<FreeObject>*, LENGTH> end{nullptr};
std::array<uint16_t, RANDOM ? 2 : 0> length{};
@@ -321,7 +324,7 @@ namespace snmalloc
* Adds an element to the builder
*/
void add(
CapPtr<FreeObject, CBAlloc> n,
capptr::AllocFull<FreeObject> n,
const FreeListKey& key,
LocalEntropy& entropy)
{
@@ -348,7 +351,7 @@ namespace snmalloc
*/
template<bool RANDOM_ = RANDOM>
std::enable_if_t<!RANDOM_>
add(CapPtr<FreeObject, CBAlloc> n, const FreeListKey& key)
add(capptr::AllocFull<FreeObject> n, const FreeListKey& key)
{
static_assert(RANDOM_ == RANDOM, "Don't set template parameter");
end[0] = FreeObject::store_next(end[0], n, key);
@@ -372,7 +375,7 @@ namespace snmalloc
* and is thus subject to encoding if the next_object pointers
* encoded.
*/
CapPtr<FreeObject, CBAlloc>
capptr::AllocFull<FreeObject>
read_head(uint32_t index, const FreeListKey& key)
{
return FreeObject::encode_next(
@@ -444,7 +447,7 @@ namespace snmalloc
template<bool RANDOM_ = RANDOM>
std::enable_if_t<
!RANDOM_,
std::pair<CapPtr<FreeObject, CBAlloc>, CapPtr<FreeObject, CBAlloc>>>
std::pair<capptr::AllocFull<FreeObject>, capptr::AllocFull<FreeObject>>>
extract_segment(const FreeListKey& key)
{
static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!");
@@ -456,7 +459,7 @@ namespace snmalloc
// to the actual object. This isn't true if the builder is
// empty, but you are not allowed to call this in the empty case.
auto last =
CapPtr<FreeObject, CBAlloc>(reinterpret_cast<FreeObject*>(end[0]));
capptr::AllocFull<FreeObject>(reinterpret_cast<FreeObject*>(end[0]));
init();
return {first, last};
}

8
src/mem/localalloc.h
View File

@@ -263,7 +263,9 @@ namespace snmalloc
MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry(
core_alloc->backend_state_ptr(), address_cast(p));
local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
entry.get_remote()->trunc_id(), CapPtr<void, CBAlloc>(p), key_global);
entry.get_remote()->trunc_id(),
capptr::AllocFull<void>(p),
key_global);
post_remote_cache();
return;
}
@@ -469,7 +471,7 @@ namespace snmalloc
{
local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
entry.get_remote()->trunc_id(),
CapPtr<void, CBAlloc>(p),
capptr::AllocFull<void>(p),
key_global);
# ifdef SNMALLOC_TRACING
std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p)
@@ -501,7 +503,7 @@ namespace snmalloc
# endif
ChunkRecord* slab_record =
reinterpret_cast<ChunkRecord*>(entry.get_metaslab());
slab_record->chunk = CapPtr<void, CBChunk>(p);
slab_record->chunk = capptr::Chunk<void>(p);
check_init(
[](
CoreAlloc* core_alloc,

6
src/mem/localcache.h
View File

@@ -13,19 +13,19 @@ namespace snmalloc
using Stats = AllocStats<NUM_SIZECLASSES, NUM_LARGE_CLASSES>;
inline static SNMALLOC_FAST_PATH void*
finish_alloc_no_zero(CapPtr<FreeObject, CBAlloc> p, sizeclass_t sizeclass)
finish_alloc_no_zero(capptr::AllocFull<FreeObject> p, sizeclass_t sizeclass)
{
SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p)));
UNUSED(sizeclass);
auto r = capptr_reveal(capptr_export(p.as_void()));
auto r = capptr_reveal(capptr_to_user_address_control(p.as_void()));
return r;
}
template<ZeroMem zero_mem, typename SharedStateHandle>
inline static SNMALLOC_FAST_PATH void*
finish_alloc(CapPtr<FreeObject, CBAlloc> p, sizeclass_t sizeclass)
finish_alloc(capptr::AllocFull<FreeObject> p, sizeclass_t sizeclass)
{
auto r = finish_alloc_no_zero(p, sizeclass);

3
src/mem/metaslab.h
View File

@@ -5,7 +5,6 @@
#include "../ds/seqqueue.h"
#include "../mem/remoteallocator.h"
#include "freelist.h"
#include "ptrhelpers.h"
#include "sizeclasstable.h"
namespace snmalloc
@@ -153,7 +152,7 @@ namespace snmalloc
* component, but with randomisation, it may only return part of the
* available objects for this metaslab.
*/
static SNMALLOC_FAST_PATH std::pair<CapPtr<FreeObject, CBAlloc>, bool>
static SNMALLOC_FAST_PATH std::pair<capptr::AllocFull<FreeObject>, bool>
alloc_free_list(
Metaslab* meta,
FreeListIter& fast_free_list,

98
src/mem/ptrhelpers.h
View File

@@ -1,98 +0,0 @@
#pragma once
#include "../aal/aal.h"
#include "../ds/ptrwrap.h"
#include "allocconfig.h"
namespace snmalloc
{
/*
* At various points, we do pointer math on high-authority pointers to find
* some metadata. `capptr_bound_chunkd` and `capptr_chunk_from_chunkd`
* encapsulate the notion that the result of these accesses is left unbounded
* in non-debug builds, because most codepaths do not reveal these pointers or
* any progeny to the application. However, in some cases we have already
* (partially) bounded these high-authority pointers (to CBChunk) and wish to
* preserve this annotation (rather than always returning a CBChunkD-annotated
* pointer); `capptr_bound_chunkd_bounds` does the computation for us and is
* used in the signatures of below and in those of wrappers around them.
*/
template<capptr_bounds B>
constexpr capptr_bounds capptr_bound_chunkd_bounds()
{
switch (B)
{
case CBArena:
return CBChunkD;
case CBChunkD:
return CBChunkD;
case CBChunk:
return CBChunk;
}
}
/**
* Construct an CapPtr<T, CBChunkD> from an CapPtr<T, CBArena> or
* CapPtr<T, CBChunkD> input. For an CapPtr<T, CBChunk> input, simply pass
* it through (preserving the static notion of bounds).
*
* Applies bounds on debug builds, otherwise is just sleight of hand.
*
* Requires that `p` point at a multiple of `sz` (that is, at the base of a
* highly-aligned object) to avoid representability issues.
*/
template<typename T, capptr_bounds B>
SNMALLOC_FAST_PATH CapPtr<T, capptr_bound_chunkd_bounds<B>()>
capptr_bound_chunkd(CapPtr<T, B> p, size_t sz)
{
static_assert(B == CBArena || B == CBChunkD || B == CBChunk);
SNMALLOC_ASSERT((address_cast(p) % sz) == 0);
#ifndef NDEBUG
// On Debug builds, apply bounds if not already there
if constexpr (B == CBArena)
return Aal::capptr_bound<T, CBChunkD>(p, sz);
else // quiesce MSVC's warnings about unreachable code below
#endif
{
UNUSED(sz);
return CapPtr<T, capptr_bound_chunkd_bounds<B>()>(p.unsafe_ptr());
}
}
/**
* Apply bounds that might not have been applied when constructing an
* CapPtr<T, CBChunkD>. That is, on non-debug builds, apply bounds; debug
* builds have already had them applied.
*
* Requires that `p` point at a multiple of `sz` (that is, at the base of a
* highly-aligned object) to avoid representability issues.
*/
template<typename T>
SNMALLOC_FAST_PATH CapPtr<T, CBChunk>
capptr_chunk_from_chunkd(CapPtr<T, CBChunkD> p, size_t sz)
{
SNMALLOC_ASSERT((address_cast(p) % sz) == 0);
#ifndef NDEBUG
// On debug builds, CBChunkD are already bounded as if CBChunk.
UNUSED(sz);
return CapPtr<T, CBChunk>(p.unsafe_ptr());
#else
// On non-debug builds, apply bounds now, as they haven't been already.
return Aal::capptr_bound<T, CBChunk>(p, sz);
#endif
}
/**
* Very rarely, while debugging, it's both useful and acceptable to forget
* that we have applied chunk bounds to something.
*/
template<typename T>
SNMALLOC_FAST_PATH CapPtr<T, CBChunkD>
capptr_debug_chunkd_from_chunk(CapPtr<T, CBChunk> p)
{
return CapPtr<T, CBChunkD>(p.unsafe_ptr());
}
} // namespace snmalloc

27
src/mem/remoteallocator.h
View File

@@ -35,10 +35,10 @@ namespace snmalloc
// Store the message queue on a separate cacheline. It is mutable data that
// is read by other threads.
alignas(CACHELINE_SIZE) AtomicCapPtr<FreeObject, CBAlloc> back{nullptr};
alignas(CACHELINE_SIZE) capptr::AtomicAllocFull<FreeObject> back{nullptr};
// Store the two ends on different cache lines as access by different
// threads.
alignas(CACHELINE_SIZE) CapPtr<FreeObject, CBAlloc> front{nullptr};
alignas(CACHELINE_SIZE) capptr::AllocFull<FreeObject> front{nullptr};
constexpr RemoteAllocator() = default;
@@ -48,7 +48,7 @@ namespace snmalloc
SNMALLOC_ASSERT(front != nullptr);
}
void init(CapPtr<FreeObject, CBAlloc> stub)
void init(capptr::AllocFull<FreeObject> stub)
{
stub->atomic_store_null(key_global);
front = stub;
@@ -56,9 +56,9 @@ namespace snmalloc
invariant();
}
CapPtr<FreeObject, CBAlloc> destroy()
capptr::AllocFull<FreeObject> destroy()
{
CapPtr<FreeObject, CBAlloc> fnt = front;
capptr::AllocFull<FreeObject> fnt = front;
back.store(nullptr, std::memory_order_relaxed);
front = nullptr;
return fnt;
@@ -66,7 +66,7 @@ namespace snmalloc
inline bool is_empty()
{
CapPtr<FreeObject, CBAlloc> bk = back.load(std::memory_order_relaxed);
capptr::AllocFull<FreeObject> bk = back.load(std::memory_order_relaxed);
return bk == front;
}
@@ -76,21 +76,21 @@ namespace snmalloc
* last should be linked together through their next pointers.
*/
void enqueue(
CapPtr<FreeObject, CBAlloc> first,
CapPtr<FreeObject, CBAlloc> last,
capptr::AllocFull<FreeObject> first,
capptr::AllocFull<FreeObject> last,
const FreeListKey& key)
{
invariant();
last->atomic_store_null(key);
// exchange needs to be a release, so nullptr in next is visible.
CapPtr<FreeObject, CBAlloc> prev =
capptr::AllocFull<FreeObject> prev =
back.exchange(last, std::memory_order_release);
prev->atomic_store_next(first, key);
}
CapPtr<FreeObject, CBAlloc> peek()
capptr::AllocFull<FreeObject> peek()
{
return front;
}
@@ -98,11 +98,12 @@ namespace snmalloc
/**
* Returns the front message, or null if not possible to return a message.
*/
std::pair<CapPtr<FreeObject, CBAlloc>, bool> dequeue(const FreeListKey& key)
std::pair<capptr::AllocFull<FreeObject>, bool>
dequeue(const FreeListKey& key)
{
invariant();
CapPtr<FreeObject, CBAlloc> first = front;
CapPtr<FreeObject, CBAlloc> next = first->atomic_read_next(key);
capptr::AllocFull<FreeObject> first = front;
capptr::AllocFull<FreeObject> next = first->atomic_read_next(key);
if (next != nullptr)
{

2
src/mem/remotecache.h
View File

@@ -66,7 +66,7 @@ namespace snmalloc
template<size_t allocator_size>
SNMALLOC_FAST_PATH void dealloc(
RemoteAllocator::alloc_id_t target_id,
CapPtr<void, CBAlloc> p,
capptr::AllocFull<void> p,
const FreeListKey& key)
{
SNMALLOC_ASSERT(initialised);

6
src/mem/slaballocator.h
View File

@@ -18,7 +18,7 @@ namespace snmalloc
struct ChunkRecord
{
std::atomic<ChunkRecord*> next;
CapPtr<void, CBChunk> chunk;
capptr::Chunk<void> chunk;
};
/**
@@ -76,7 +76,7 @@ namespace snmalloc
{
public:
template<SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle>
static std::pair<CapPtr<void, CBChunk>, Metaslab*> alloc_chunk(
static std::pair<capptr::Chunk<void>, Metaslab*> alloc_chunk(
typename SharedStateHandle::LocalState& local_state,
sizeclass_t sizeclass,
sizeclass_t slab_sizeclass, // TODO sizeclass_t
@@ -163,7 +163,7 @@ namespace snmalloc
// Cache line align
size_t size = bits::align_up(sizeof(U), 64);
CapPtr<void, CBChunk> p =
capptr::Chunk<void> p =
SharedStateHandle::template alloc_meta_data<U>(local_state, size);
if (p == nullptr)

46
src/pal/pal.h
View File

@@ -69,28 +69,46 @@ namespace snmalloc
// Used to keep Superslab metadata committed.
static constexpr size_t OS_PAGE_SIZE = Pal::page_size;
/**
* Compute the AddressSpaceControl::User variant of a capptr::bound
* annotation. This is used by the PAL's capptr_export function to compute
* its return value's annotation.
*/
template<SNMALLOC_CONCEPT(capptr::ConceptBound) B>
using capptr_user_address_control_type =
typename B::template with_address_space_control<
capptr::dimension::AddressSpaceControl::User>;
/**
* Perform platform-specific adjustment of return pointers.
*
* This is here, rather than in every PAL proper, merely to minimize
* disruption to PALs for platforms that do not support StrictProvenance AALs.
*/
template<typename PAL = Pal, typename AAL = Aal, typename T, capptr_bounds B>
template<
typename PAL = Pal,
typename AAL = Aal,
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) B>
static inline typename std::enable_if_t<
!aal_supports<StrictProvenance, AAL>,
CapPtr<T, capptr_export_type<B>()>>
capptr_export(CapPtr<T, B> p)
CapPtr<T, capptr_user_address_control_type<B>>>
capptr_to_user_address_control(CapPtr<T, B> p)
{
return CapPtr<T, capptr_export_type<B>()>(p.unsafe_ptr());
return CapPtr<T, capptr_user_address_control_type<B>>(p.unsafe_capptr);
}
template<typename PAL = Pal, typename AAL = Aal, typename T, capptr_bounds B>
static inline typename std::enable_if_t<
template<
typename PAL = Pal,
typename AAL = Aal,
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) B>
static SNMALLOC_FAST_PATH typename std::enable_if_t<
aal_supports<StrictProvenance, AAL>,
CapPtr<T, capptr_export_type<B>()>>
capptr_export(CapPtr<T, B> p)
CapPtr<T, capptr_user_address_control_type<B>>>
capptr_to_user_address_control(CapPtr<T, B> p)
{
return PAL::capptr_export(p);
return PAL::capptr_to_user_address_control(p);
}
/**
@@ -101,12 +119,16 @@ namespace snmalloc
* disruption and avoid code bloat. This wrapper ought to compile down to
* nothing if SROA is doing its job.
*/
template<typename PAL, bool page_aligned = false, typename T, capptr_bounds B>
template<
typename PAL,
bool page_aligned = false,
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) B>
static SNMALLOC_FAST_PATH void pal_zero(CapPtr<T, B> p, size_t sz)
{
static_assert(
!page_aligned || B == CBArena || B == CBChunkD || B == CBChunk);
PAL::template zero<page_aligned>(p.unsafe_ptr(), sz);
!page_aligned || B::spatial >= capptr::dimension::Spatial::Chunk);
PAL::template zero<page_aligned>(p.unsafe_capptr, sz);
}
static_assert(