diff --git a/src/aal/aal.h b/src/aal/aal.h index 00f6362..7e2af37 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -189,32 +189,33 @@ namespace snmalloc */ template< typename T, - enum capptr_bounds nbounds, - enum capptr_bounds obounds, + SNMALLOC_CONCEPT(capptr::ConceptBound) BOut, + SNMALLOC_CONCEPT(capptr::ConceptBound) BIn, typename U = T> - static SNMALLOC_FAST_PATH CapPtr - capptr_bound(CapPtr a, size_t size) noexcept + static SNMALLOC_FAST_PATH CapPtr + capptr_bound(CapPtr a, size_t size) noexcept { // Impose constraints on bounds annotations. - static_assert( - obounds == CBArena || obounds == CBChunkD || obounds == CBChunk || - obounds == CBChunkE); - static_assert(capptr_is_bounds_refinement()); + static_assert(BIn::spatial >= capptr::dimension::Spatial::Chunk); + static_assert(capptr_is_spatial_refinement()); UNUSED(size); - return CapPtr(a.template as_static().unsafe_ptr()); + return CapPtr(a.template as_static().unsafe_capptr); } /** * For architectures which do not enforce StrictProvenance, there's nothing * to be done, so just return the pointer unmodified with new annotation. */ - template + template< + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) BOut, + SNMALLOC_CONCEPT(capptr::ConceptBound) BIn> static SNMALLOC_FAST_PATH CapPtr capptr_rebound(CapPtr a, CapPtr r) noexcept { UNUSED(a); - return CapPtr(r.unsafe_ptr()); + return CapPtr(r.unsafe_capptr); } }; } // namespace snmalloc diff --git a/src/aal/aal_concept.h b/src/aal/aal_concept.h index 180e893..2a14a94 100644 --- a/src/aal/aal_concept.h +++ b/src/aal/aal_concept.h @@ -43,21 +43,22 @@ namespace snmalloc template concept ConceptAAL_capptr_methods = - requires(CapPtr auth, CapPtr ret, size_t sz) + requires(capptr::Chunk auth, capptr::AllocFull ret, size_t sz) { /** * Produce a pointer with reduced authority from a more privilged pointer. * The resulting pointer will have base at auth's address and length of * exactly sz. auth+sz must not exceed auth's limit. */ - { AAL::template capptr_bound(auth, sz) } noexcept - -> ConceptSame>; + { AAL::template capptr_bound(auth, sz) } + noexcept + -> ConceptSame>; /** * Construct a copy of auth with its target set to that of ret. */ { AAL::capptr_rebound(auth, ret) } noexcept - -> ConceptSame>; + -> ConceptSame>; }; template diff --git a/src/backend/address_space.h b/src/backend/address_space.h index ada5295..e197e77 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -43,7 +43,7 @@ namespace snmalloc * arena_map for use in subsequent amplification. */ template - CapPtr + capptr::Chunk reserve(typename Pagemap::LocalState* local_state, size_t size) { #ifdef SNMALLOC_TRACING @@ -62,21 +62,21 @@ namespace snmalloc { if (size >= PAL::minimum_alloc_size) { - auto base = CapPtr( - PAL::template reserve_aligned(size)); + auto base = + capptr::Chunk(PAL::template reserve_aligned(size)); Pagemap::register_range(local_state, address_cast(base), size); return base; } } - CapPtr res; + capptr::Chunk res; { FlagLock lock(spin_lock); res = core.template reserve(local_state, size); if (res == nullptr) { // Allocation failed ask OS for more memory - CapPtr block = nullptr; + capptr::Chunk block = nullptr; size_t block_size = 0; if constexpr (pal_supports) { @@ -92,7 +92,7 @@ namespace snmalloc // It's a bit of a lie to convert without applying bounds, but the // platform will have bounded block for us and it's better that // the rest of our internals expect CBChunk bounds. - block = CapPtr(block_raw); + block = capptr::Chunk(block_raw); } else if constexpr (!pal_supports) { @@ -110,7 +110,7 @@ namespace snmalloc size_request >= needed_size; size_request = size_request / 2) { - block = CapPtr(PAL::reserve(size_request)); + block = capptr::Chunk(PAL::reserve(size_request)); if (block != nullptr) { block_size = size_request; @@ -158,7 +158,7 @@ namespace snmalloc * used, by smaller objects. */ template - CapPtr reserve_with_left_over( + capptr::Chunk reserve_with_left_over( typename Pagemap::LocalState* local_state, size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -198,7 +198,7 @@ namespace snmalloc template void add_range( typename Pagemap::LocalState* local_state, - CapPtr base, + capptr::Chunk base, size_t length) { FlagLock lock(spin_lock); diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 0aa5872..d3088c0 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -25,7 +25,7 @@ namespace snmalloc { struct FreeChunk { - CapPtr next; + capptr::Chunk next; }; /** @@ -43,12 +43,12 @@ namespace snmalloc * bits::BITS is used for simplicity, we do not use below the pointer size, * and large entries will be unlikely to be supported by the platform. */ - std::array, bits::BITS> ranges = {}; + std::array, bits::BITS> ranges = {}; /** * Checks a block satisfies its invariant. */ - inline void check_block(CapPtr base, size_t align_bits) + inline void check_block(capptr::Chunk base, size_t align_bits) { SNMALLOC_ASSERT( address_cast(base) == @@ -72,8 +72,8 @@ namespace snmalloc void set_next( typename Pagemap::LocalState* local_state, size_t align_bits, - CapPtr base, - CapPtr next) + capptr::Chunk base, + capptr::Chunk next) { if (align_bits >= MIN_CHUNK_BITS) { @@ -102,16 +102,16 @@ namespace snmalloc * particular size. */ template - CapPtr get_next( + capptr::Chunk get_next( typename Pagemap::LocalState* local_state, size_t align_bits, - CapPtr base) + capptr::Chunk base) { if (align_bits >= MIN_CHUNK_BITS) { const MetaEntry& t = Pagemap::template get_metaentry( local_state, address_cast(base)); - return CapPtr( + return capptr::Chunk( reinterpret_cast(t.get_metaslab())); } @@ -127,7 +127,7 @@ namespace snmalloc void add_block( typename Pagemap::LocalState* local_state, size_t align_bits, - CapPtr base) + capptr::Chunk base) { check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); @@ -143,10 +143,10 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr + capptr::Chunk remove_block(typename Pagemap::LocalState* local_state, size_t align_bits) { - CapPtr first = ranges[align_bits]; + capptr::Chunk first = ranges[align_bits]; if (first == nullptr) { if (align_bits == (bits::BITS - 1)) @@ -156,7 +156,7 @@ namespace snmalloc } // Look for larger block and split up recursively - CapPtr bigger = + capptr::Chunk bigger = remove_block(local_state, align_bits + 1); if (bigger != nullptr) { @@ -174,7 +174,8 @@ namespace snmalloc add_block( local_state, align_bits, - Aal::capptr_bound(left_over, left_over_size)); + Aal::capptr_bound( + left_over, left_over_size)); check_block(left_over.as_static(), align_bits); } check_block(bigger.as_static(), align_bits + 1); @@ -196,7 +197,7 @@ namespace snmalloc SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> void add_range( typename Pagemap::LocalState* local_state, - CapPtr base, + capptr::Chunk base, size_t length) { // For start and end that are not chunk sized, we need to @@ -233,7 +234,7 @@ namespace snmalloc * Commit a block of memory */ template - void commit_block(CapPtr base, size_t size) + void commit_block(capptr::Chunk base, size_t size) { // Rounding required for sub-page allocations. auto page_start = pointer_align_down(base); @@ -257,7 +258,7 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr + capptr::Chunk reserve(typename Pagemap::LocalState* local_state, size_t size) { #ifdef SNMALLOC_TRACING @@ -281,7 +282,7 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr reserve_with_left_over( + capptr::Chunk reserve_with_left_over( typename Pagemap::LocalState* local_state, size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); diff --git a/src/backend/backend.h b/src/backend/backend.h index 8f36b7e..daf0363 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -39,7 +39,7 @@ namespace snmalloc * Backend allocator may use guard pages and separate area of * address space to protect this from corruption. */ - static CapPtr alloc_meta_data( + static capptr::Chunk alloc_meta_data( AddressSpaceManager& global, LocalState* local_state, size_t size) { return reserve(global, local_state, size); @@ -54,7 +54,7 @@ namespace snmalloc * (remote, sizeclass, metaslab) * where metaslab, is the second element of the pair return. */ - static std::pair, Metaslab*> alloc_chunk( + static std::pair, Metaslab*> alloc_chunk( AddressSpaceManager& global, LocalState* local_state, size_t size, @@ -70,7 +70,7 @@ namespace snmalloc if (meta == nullptr) return {nullptr, nullptr}; - CapPtr p = reserve(global, local_state, size); + capptr::Chunk p = reserve(global, local_state, size); #ifdef SNMALLOC_TRACING std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" @@ -98,7 +98,7 @@ namespace snmalloc * space managers. */ template - static CapPtr reserve( + static capptr::Chunk reserve( AddressSpaceManager& global, LocalState* local_state, size_t size) { #ifdef SNMALLOC_CHECK_CLIENT @@ -108,7 +108,7 @@ namespace snmalloc constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK; #endif - CapPtr p; + capptr::Chunk p; if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) { #ifdef SNMALLOC_CHECK_CLIENT @@ -182,8 +182,8 @@ namespace snmalloc * the range [base, base+full_size]. The first and last slot are not used * so that the edges can be used for guard pages. */ - static CapPtr - sub_range(CapPtr base, size_t full_size, size_t sub_size) + static capptr::Chunk + sub_range(capptr::Chunk base, size_t full_size, size_t sub_size) { SNMALLOC_ASSERT(bits::is_pow2(full_size)); SNMALLOC_ASSERT(bits::is_pow2(sub_size)); @@ -318,7 +318,7 @@ namespace snmalloc auto [heap_base, heap_length] = Pagemap::concretePagemap.init(base, length); address_space.template add_range( - local_state, CapPtr(heap_base), heap_length); + local_state, capptr::Chunk(heap_base), heap_length); } /** @@ -332,7 +332,7 @@ namespace snmalloc * places or with different policies. */ template - static CapPtr + static capptr::Chunk alloc_meta_data(LocalState* local_state, size_t size) { return AddressSpaceAllocator::alloc_meta_data( @@ -348,7 +348,7 @@ namespace snmalloc * (remote, sizeclass, metaslab) * where metaslab, is the second element of the pair return. */ - static std::pair, Metaslab*> alloc_chunk( + static std::pair, Metaslab*> alloc_chunk( LocalState* local_state, size_t size, RemoteAllocator* remote, diff --git a/src/ds/address.h b/src/ds/address.h index ae1c553..90b0bb4 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -35,7 +35,7 @@ namespace snmalloc reinterpret_cast(base) + static_cast(diff)); } - template + template inline CapPtr pointer_offset(CapPtr base, size_t diff) { @@ -52,7 +52,7 @@ namespace snmalloc return reinterpret_cast(reinterpret_cast(base) + diff); } - template + template inline CapPtr pointer_offset_signed(CapPtr base, ptrdiff_t diff) { @@ -76,7 +76,7 @@ namespace snmalloc * capptr_bound. */ - template + template inline SNMALLOC_FAST_PATH address_t address_cast(CapPtr a) { return address_cast(a.unsafe_ptr()); @@ -132,7 +132,10 @@ namespace snmalloc pointer_align_down(reinterpret_cast(p))); } - template + template< + size_t alignment, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) bounds> inline CapPtr pointer_align_down(CapPtr p) { return CapPtr(pointer_align_down(p.unsafe_ptr())); @@ -166,7 +169,10 @@ namespace snmalloc } } - template + template< + size_t alignment, + typename T = void, + SNMALLOC_CONCEPT(capptr::ConceptBound) bounds> inline CapPtr pointer_align_up(CapPtr p) { return CapPtr(pointer_align_up(p.unsafe_ptr())); @@ -195,7 +201,7 @@ namespace snmalloc #endif } - template + template inline CapPtr pointer_align_down(CapPtr p, size_t alignment) { @@ -219,7 +225,7 @@ namespace snmalloc #endif } - template + template inline CapPtr pointer_align_up(CapPtr p, size_t alignment) { @@ -241,8 +247,8 @@ namespace snmalloc template< typename T = void, typename U = void, - enum capptr_bounds Tbounds, - enum capptr_bounds Ubounds> + SNMALLOC_CONCEPT(capptr::ConceptBound) Tbounds, + SNMALLOC_CONCEPT(capptr::ConceptBound) Ubounds> inline size_t pointer_diff(CapPtr base, CapPtr cursor) { return pointer_diff(base.unsafe_ptr(), cursor.unsafe_ptr()); @@ -261,8 +267,8 @@ namespace snmalloc template< typename T = void, typename U = void, - enum capptr_bounds Tbounds, - enum capptr_bounds Ubounds> + SNMALLOC_CONCEPT(capptr::ConceptBound) Tbounds, + SNMALLOC_CONCEPT(capptr::ConceptBound) Ubounds> inline ptrdiff_t pointer_diff_signed(CapPtr base, CapPtr cursor) { diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index ff073e9..11c9926 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -1,6 +1,7 @@ #pragma once -#include "defines.h" +#include "../ds/concept.h" +#include "../ds/defines.h" #include @@ -21,78 +22,150 @@ namespace snmalloc /** * Summaries of StrictProvenance metadata. We abstract away the particular * size and any offset into the bounds. - * - * CBArena is as powerful as our pointers get: they're results from mmap(), - * and so confer as much authority as the kernel has given us. - * - * CBChunk is restricted to either a single chunk (SUPERSLAB_SIZE) or perhaps - * to several if we've requesed a large allocation (see capptr_chunk_is_alloc - * and its uses). - * - * CBChunkD is curious: we often use CBArena-bounded pointers to derive - * pointers to Allocslab metadata, and on most fast paths these pointers end - * up being ephemeral. As such, on NDEBUG builds, we elide the capptr_bounds - * that would bound these to chunks and instead just unsafely inherit the - * CBArena bounds. The use of CBChunkD thus helps to ensure that we - * eventually do invoke capptr_bounds when these pointers end up being longer - * lived! - * - * *E forms are "exported" and have had platform constraints applied. That - * means, for example, on CheriBSD, that they have had their VMMAP permission - * stripped. - * - * Yes, I wish the start-of-comment characters were aligned below as well. - * I blame clang format. */ - enum capptr_bounds + + namespace capptr { - /* Spatial Notes */ - CBArena, /* Arena */ - CBChunkD, /* Arena Chunk-bounded in debug; internal use only! */ - CBChunk, /* Chunk */ - CBChunkE, /* Chunk (+ platform constraints) */ - CBAlloc, /* Alloc */ - CBAllocE /* Alloc (+ platform constraints) */ - }; + namespace dimension + { + /* + * Describe the spatial extent (intended to be) authorized by a pointer. + * + * Bounds dimensions are sorted so that < reflects authority. + */ + enum class Spatial + { + /** + * Bounded to a particular allocation (which might be Large!) + */ + Alloc, + /** + * Bounded to one or more particular chunk granules + */ + Chunk, + }; + + /** + * On some platforms (e.g., CHERI), pointers can be checked to see whether + * they authorize control of the address space. See the PAL's + * capptr_export(). + */ + enum class AddressSpaceControl + { + /** + * All intended control constraints have been applied. For example, on + * CheriBSD, the VMMAP permission has been stripped and so this CapPtr<> + * cannot authorize manipulation of the address space itself, though it + * continues to authorize loads and stores. + */ + User, + /** + * No control constraints have been applied. On CheriBSD, specifically, + * this grants control of the address space (via mmap and friends) and + * in Cornucopia exempts the pointer from revocation (as long as the + * mapping remains in place, but snmalloc does not presently tear down + * its own mappings.) + */ + Full + }; + + } // namespace dimension + + /** + * The aggregate type of a bound: a Cartesian product of the individual + * dimensions, above. + */ + template + struct bound + { + static constexpr enum dimension::Spatial spatial = S; + static constexpr enum dimension::AddressSpaceControl + address_space_control = AS; + + /** + * Set just the spatial component of the bounds + */ + template + using with_spatial = bound; + + /** + * Set just the address space control component of the bounds + */ + template + using with_address_space_control = bound; + }; + + // clang-format off +#ifdef __cpp_concepts + /* + * This is spelled a little differently from our other concepts because GCC + * treats "{ T::spatial }" as generating a reference and then complains that + * it isn't "ConceptSame", though clang is perfectly happy + * with that spelling. Both seem happy with this formulation. + */ + template + concept ConceptBound = + ConceptSame && + ConceptSame; +#endif + // clang-format on + + /* + * Several combinations are used often enough that we give convenient + * aliases for them. + */ + namespace bounds + { + /** + * Internal access to a Chunk of memory. These flow between the ASM and + * the slab allocators, for example. + */ + using Chunk = + bound; + + /** + * User access to an entire Chunk. Used as an ephemeral state when + * returning a large allocation. See capptr_chunk_is_alloc. + */ + using ChunkUser = + Chunk::with_address_space_control; + + /** + * Internal access to just one allocation (usually, within a slab). + */ + using AllocFull = Chunk::with_spatial; + + /** + * User access to just one allocation (usually, within a slab). + */ + using Alloc = AllocFull::with_address_space_control< + dimension::AddressSpaceControl::User>; + } // namespace bounds + } // namespace capptr /** - * Compute the "exported" variant of a capptr_bounds annotation. This is - * used by the PAL's capptr_export function to compute its return value's - * annotation. + * Determine whether BI is a spatial refinement of BO. + * Chunk and ChunkD are considered eqivalent here. */ - template - SNMALLOC_CONSTEVAL capptr_bounds capptr_export_type() + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BI, + SNMALLOC_CONCEPT(capptr::ConceptBound) BO> + SNMALLOC_CONSTEVAL bool capptr_is_spatial_refinement() { - static_assert( - (B == CBChunk) || (B == CBAlloc), "capptr_export_type of bad type"); - - switch (B) + if (BI::address_space_control != BO::address_space_control) { - case CBChunk: - return CBChunkE; - case CBAlloc: - return CBAllocE; + return false; } - } - template - SNMALLOC_CONSTEVAL bool capptr_is_bounds_refinement() - { - switch (BI) + switch (BI::spatial) { - case CBAllocE: - return BO == CBAllocE; - case CBAlloc: - return BO == CBAlloc; - case CBChunkE: - return BO == CBAllocE || BO == CBChunkE; - case CBChunk: - return BO == CBAlloc || BO == CBChunk || BO == CBChunkD; - case CBChunkD: - return BO == CBAlloc || BO == CBChunk || BO == CBChunkD; - case CBArena: - return BO == CBAlloc || BO == CBChunk || BO == CBChunkD || - BO == CBArena; + using namespace capptr::dimension; + case Spatial::Chunk: + return true; + + case Spatial::Alloc: + return BO::spatial == Spatial::Alloc; } } @@ -100,18 +173,19 @@ namespace snmalloc * A pointer annotated with a "phantom type parameter" carrying a static * summary of its StrictProvenance metadata. */ - template - class CapPtr + template + struct CapPtr { - uintptr_t unsafe_capptr; + T* unsafe_capptr; - public: /** * nullptr is implicitly constructable at any bounds type */ - constexpr CapPtr(const std::nullptr_t) : unsafe_capptr(0) {} + constexpr SNMALLOC_FAST_PATH CapPtr(const std::nullptr_t n) + : unsafe_capptr(n) + {} - constexpr CapPtr() : CapPtr(nullptr){}; + constexpr SNMALLOC_FAST_PATH CapPtr() : CapPtr(nullptr) {} /** * all other constructions must be explicit @@ -127,23 +201,21 @@ namespace snmalloc # pragma warning(push) # pragma warning(disable : 4702) #endif - constexpr explicit CapPtr(uintptr_t p) : unsafe_capptr(p) {} + constexpr explicit SNMALLOC_FAST_PATH CapPtr(T* p) : unsafe_capptr(p) {} #ifdef _MSC_VER # pragma warning(pop) #endif - explicit CapPtr(T* p) : unsafe_capptr(reinterpret_cast(p)) {} - /** * Allow static_cast<>-s that preserve bounds but vary the target type. */ template - SNMALLOC_FAST_PATH CapPtr as_static() + [[nodiscard]] SNMALLOC_FAST_PATH CapPtr as_static() const { - return CapPtr(this->unsafe_capptr); + return CapPtr(static_cast(this->unsafe_capptr)); } - SNMALLOC_FAST_PATH CapPtr as_void() + [[nodiscard]] SNMALLOC_FAST_PATH CapPtr as_void() const { return this->as_static(); } @@ -152,9 +224,9 @@ namespace snmalloc * A more aggressive bounds-preserving cast, using reinterpret_cast */ template - SNMALLOC_FAST_PATH CapPtr as_reinterpret() + [[nodiscard]] SNMALLOC_FAST_PATH CapPtr as_reinterpret() const { - return CapPtr(this->unsafe_capptr); + return CapPtr(reinterpret_cast(this->unsafe_capptr)); } SNMALLOC_FAST_PATH bool operator==(const CapPtr& rhs) const @@ -172,60 +244,82 @@ namespace snmalloc return this->unsafe_capptr < rhs.unsafe_capptr; } - [[nodiscard]] SNMALLOC_FAST_PATH T* unsafe_ptr() const + SNMALLOC_FAST_PATH T* operator->() const { - return reinterpret_cast(this->unsafe_capptr); + /* + * Alloc-bounded, platform-constrained pointers are associated with + * objects coming from or going to the client; we should be doing nothing + * with them. + */ + static_assert( + (bounds::spatial != capptr::dimension::Spatial::Alloc) || + (bounds::address_space_control != + capptr::dimension::AddressSpaceControl::User)); + return this->unsafe_capptr; } - [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t unsafe_uintptr() const + [[nodiscard]] SNMALLOC_FAST_PATH T* unsafe_ptr() const { return this->unsafe_capptr; } - SNMALLOC_FAST_PATH T* operator->() const + [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t unsafe_uintptr() const { - /* - * CBAllocE bounds are associated with objects coming from or going to the - * client; we should be doing nothing with them. - */ - static_assert(bounds != CBAllocE); - return unsafe_ptr(); + return reinterpret_cast(this->unsafe_capptr); } }; - static_assert(sizeof(CapPtr) == sizeof(void*)); - static_assert(alignof(CapPtr) == alignof(void*)); + namespace capptr + { + /* + * Aliases for CapPtr<> types with particular bounds. + */ - template - using CapPtrCBArena = CapPtr; + template + using Chunk = CapPtr; - template - using CapPtrCBChunk = CapPtr; + template + using ChunkUser = CapPtr; - template - using CapPtrCBChunkE = CapPtr; + template + using AllocFull = CapPtr; - template - using CapPtrCBAlloc = CapPtr; + template + using Alloc = CapPtr; + + } // namespace capptr + + static_assert(sizeof(capptr::Chunk) == sizeof(void*)); + static_assert(alignof(capptr::Chunk) == alignof(void*)); /** * Sometimes (with large allocations) we really mean the entire chunk (or even * several chunks) to be the allocation. */ template - inline SNMALLOC_FAST_PATH CapPtr - capptr_chunk_is_alloc(CapPtr p) + inline SNMALLOC_FAST_PATH capptr::Alloc + capptr_chunk_is_alloc(capptr::ChunkUser p) { - return CapPtr(p.unsafe_capptr); + return capptr::Alloc(p.unsafe_capptr); } /** * With all the bounds and constraints in place, it's safe to extract a void - * pointer (to reveal to the client). + * pointer (to reveal to the client). Roughly dual to capptr_from_client(). */ - inline SNMALLOC_FAST_PATH void* capptr_reveal(CapPtr p) + inline SNMALLOC_FAST_PATH void* capptr_reveal(capptr::Alloc p) { - return p.unsafe_ptr(); + return p.unsafe_capptr; + } + + /** + * Given a void* from the client, it's fine to call it Alloc. Roughly + * dual to capptr_reveal(). + */ + static inline SNMALLOC_FAST_PATH capptr::Alloc + capptr_from_client(void* p) + { + return capptr::Alloc(p); } /** @@ -237,7 +331,7 @@ namespace snmalloc * annotations around an un-annotated std::atomic, to appease C++, yet * will expose or consume only CapPtr with the same bounds annotation. */ - template + template struct AtomicCapPtr { std::atomic unsafe_capptr; @@ -245,12 +339,16 @@ namespace snmalloc /** * nullptr is constructable at any bounds type */ - constexpr AtomicCapPtr(const std::nullptr_t n) : unsafe_capptr(n) {} + constexpr SNMALLOC_FAST_PATH AtomicCapPtr(const std::nullptr_t n) + : unsafe_capptr(n) + {} /** * Interconversion with CapPtr */ - AtomicCapPtr(CapPtr p) : unsafe_capptr(p.unsafe_ptr()) {} + constexpr SNMALLOC_FAST_PATH AtomicCapPtr(CapPtr p) + : unsafe_capptr(p.unsafe_capptr) + {} operator CapPtr() const noexcept { @@ -260,7 +358,7 @@ namespace snmalloc // Our copy-assignment operator follows std::atomic and returns a copy of // the RHS. Clang finds this surprising; we suppress the warning. // NOLINTNEXTLINE(misc-unconventional-assign-operator) - CapPtr operator=(CapPtr p) noexcept + SNMALLOC_FAST_PATH CapPtr operator=(CapPtr p) noexcept { this->store(p); return p; @@ -276,7 +374,7 @@ namespace snmalloc CapPtr desired, std::memory_order order = std::memory_order_seq_cst) noexcept { - this->unsafe_capptr.store(desired.unsafe_ptr(), order); + this->unsafe_capptr.store(desired.unsafe_capptr, order); } SNMALLOC_FAST_PATH CapPtr exchange( @@ -284,7 +382,7 @@ namespace snmalloc std::memory_order order = std::memory_order_seq_cst) noexcept { return CapPtr( - this->unsafe_capptr.exchange(desired.unsafe_ptr(), order)); + this->unsafe_capptr.exchange(desired.unsafe_capptr, order)); } SNMALLOC_FAST_PATH bool operator==(const AtomicCapPtr& rhs) const @@ -303,13 +401,24 @@ namespace snmalloc } }; - template - using AtomicCapPtrCBArena = AtomicCapPtr; + namespace capptr + { + /* + * Aliases for AtomicCapPtr<> types with particular bounds. + */ - template - using AtomicCapPtrCBChunk = AtomicCapPtr; + template + using AtomicChunk = AtomicCapPtr; - template - using AtomicCapPtrCBAlloc = AtomicCapPtr; + template + using AtomicChunkUser = AtomicCapPtr; + + template + using AtomicAllocFull = AtomicCapPtr; + + template + using AtomicAlloc = AtomicCapPtr; + + } // namespace capptr } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index cb3869d..881ce10 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -161,7 +161,7 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - auto dummy = CapPtr(small_alloc_one(MIN_ALLOC_SIZE)) + auto dummy = capptr::AllocFull(small_alloc_one(MIN_ALLOC_SIZE)) .template as_static(); if (dummy == nullptr) { @@ -187,7 +187,7 @@ namespace snmalloc } static SNMALLOC_FAST_PATH void alloc_new_list( - CapPtr& bumpptr, + capptr::Chunk& bumpptr, Metaslab* meta, size_t rsize, size_t slab_size, @@ -203,7 +203,7 @@ namespace snmalloc // Structure to represent the temporary list elements struct PreAllocObject { - CapPtr next; + capptr::AllocFull next; }; // The following code implements Sattolo's algorithm for generating // random cyclic permutations. This implementation is in the opposite @@ -213,9 +213,10 @@ namespace snmalloc // Note the wide bounds on curr relative to each of the ->next fields; // curr is not persisted once the list is built. - CapPtr curr = + capptr::Chunk curr = pointer_offset(bumpptr, 0).template as_static(); - curr->next = Aal::capptr_bound(curr, rsize); + curr->next = Aal::capptr_bound( + curr, rsize); uint16_t count = 1; for (curr = @@ -229,12 +230,13 @@ namespace snmalloc pointer_offset(bumpptr, insert_index * rsize) .template as_static() ->next, - Aal::capptr_bound(curr, rsize)); + Aal::capptr_bound( + curr, rsize)); count++; } // Pick entry into space, and then build linked list by traversing cycle - // to the start. Use ->next to jump from CBArena to CBAlloc. + // to the start. Use ->next to jump from Chunk to Alloc. auto start_index = entropy.sample(count); auto start_ptr = pointer_offset(bumpptr, start_index * rsize) .template as_static() @@ -249,7 +251,9 @@ namespace snmalloc auto p = bumpptr; do { - b.add(Aal::capptr_bound(p, rsize), key); + b.add( + Aal::capptr_bound(p, rsize), + key); p = pointer_offset(p, rsize); } while (p < slab_end); #endif @@ -299,7 +303,7 @@ namespace snmalloc auto start_of_slab = pointer_align_down( p, snmalloc::sizeclass_to_slab_size(sizeclass)); // TODO Add bounds correctly here - chunk_record->chunk = CapPtr(start_of_slab); + chunk_record->chunk = capptr::Chunk(start_of_slab); #ifdef SNMALLOC_TRACING std::cout << "Slab " << start_of_slab << " is unused, Object sizeclass " @@ -422,7 +426,7 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaEntry& entry, CapPtr p, bool& need_post) + const MetaEntry& entry, capptr::AllocFull p, bool& need_post) { // TODO this needs to not double count stats // TODO this needs to not double revoke if using MTE @@ -576,7 +580,7 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = CapPtr(reinterpret_cast(p)); + auto cp = capptr::AllocFull(reinterpret_cast(p)); auto& key = entropy.get_free_list_key(); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index eab71eb..09db138 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -66,9 +66,9 @@ namespace snmalloc { union { - CapPtr next_object; + capptr::AllocFull next_object; // TODO: Should really use C++20 atomic_ref rather than a union. - AtomicCapPtr atomic_next_object; + capptr::AtomicAllocFull atomic_next_object; }; #ifdef SNMALLOC_CHECK_CLIENT // Encoded representation of a back pointer. @@ -78,7 +78,7 @@ namespace snmalloc #endif public: - static CapPtr make(CapPtr p) + static capptr::AllocFull make(capptr::AllocFull p) { return p.template as_static(); } @@ -86,8 +86,10 @@ namespace snmalloc /** * Encode next */ - inline static CapPtr encode_next( - address_t curr, CapPtr next, const FreeListKey& key) + inline static capptr::AllocFull encode_next( + address_t curr, + capptr::AllocFull next, + const FreeListKey& key) { // Note we can consider other encoding schemes here. // * XORing curr and next. This doesn't require any key material @@ -98,7 +100,8 @@ namespace snmalloc if constexpr (CHECK_CLIENT && !aal_supports) { - return CapPtr(address_cast(next) ^ key.key_next); + return capptr::AllocFull(reinterpret_cast( + reinterpret_cast(next.unsafe_ptr()) ^ key.key_next)); } else { @@ -115,9 +118,9 @@ namespace snmalloc * optimization for repeated snoc operations (in which * next->next_object is nullptr). */ - static CapPtr* store_next( - CapPtr* curr, - CapPtr next, + static capptr::AllocFull* store_next( + capptr::AllocFull* curr, + capptr::AllocFull next, const FreeListKey& key) { #ifdef SNMALLOC_CHECK_CLIENT @@ -131,7 +134,7 @@ namespace snmalloc } static void - store_null(CapPtr* curr, const FreeListKey& key) + store_null(capptr::AllocFull* curr, const FreeListKey& key) { *curr = encode_next(address_cast(curr), nullptr, key); } @@ -141,8 +144,8 @@ namespace snmalloc * * Uses the atomic view of next, so can be used in the message queues. */ - void - atomic_store_next(CapPtr next, const FreeListKey& key) + void atomic_store_next( + capptr::AllocFull next, const FreeListKey& key) { #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = @@ -164,7 +167,7 @@ namespace snmalloc std::memory_order_relaxed); } - CapPtr atomic_read_next(const FreeListKey& key) + capptr::AllocFull atomic_read_next(const FreeListKey& key) { auto n = encode_next( address_cast(&next_object), @@ -194,7 +197,7 @@ namespace snmalloc /** * Read the next pointer */ - CapPtr read_next(const FreeListKey& key) + capptr::AllocFull read_next(const FreeListKey& key) { return encode_next(address_cast(&next_object), next_object, key); } @@ -211,14 +214,14 @@ namespace snmalloc */ class FreeListIter { - CapPtr curr{nullptr}; + capptr::AllocFull curr{nullptr}; #ifdef SNMALLOC_CHECK_CLIENT address_t prev{0}; #endif public: constexpr FreeListIter( - CapPtr head, address_t prev_value) + capptr::AllocFull head, address_t prev_value) : curr(head) { #ifdef SNMALLOC_CHECK_CLIENT @@ -240,7 +243,7 @@ namespace snmalloc /** * Returns current head without affecting the iterator. */ - CapPtr peek() + capptr::AllocFull peek() { return curr; } @@ -248,7 +251,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - CapPtr take(const FreeListKey& key) + capptr::AllocFull take(const FreeListKey& key) { auto c = curr; auto next = curr->read_next(key); @@ -289,11 +292,11 @@ namespace snmalloc static constexpr size_t LENGTH = RANDOM ? 2 : 1; // Pointer to the first element. - std::array, LENGTH> head; + std::array, LENGTH> head; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - std::array*, LENGTH> end{nullptr}; + std::array*, LENGTH> end{nullptr}; std::array length{}; @@ -321,7 +324,7 @@ namespace snmalloc * Adds an element to the builder */ void add( - CapPtr n, + capptr::AllocFull n, const FreeListKey& key, LocalEntropy& entropy) { @@ -348,7 +351,7 @@ namespace snmalloc */ template std::enable_if_t - add(CapPtr n, const FreeListKey& key) + add(capptr::AllocFull n, const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); end[0] = FreeObject::store_next(end[0], n, key); @@ -372,7 +375,7 @@ namespace snmalloc * and is thus subject to encoding if the next_object pointers * encoded. */ - CapPtr + capptr::AllocFull read_head(uint32_t index, const FreeListKey& key) { return FreeObject::encode_next( @@ -444,7 +447,7 @@ namespace snmalloc template std::enable_if_t< !RANDOM_, - std::pair, CapPtr>> + std::pair, capptr::AllocFull>> extract_segment(const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); @@ -456,7 +459,7 @@ namespace snmalloc // to the actual object. This isn't true if the builder is // empty, but you are not allowed to call this in the empty case. auto last = - CapPtr(reinterpret_cast(end[0])); + capptr::AllocFull(reinterpret_cast(end[0])); init(); return {first, last}; } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 761754e..b20b3d3 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -263,7 +263,9 @@ namespace snmalloc MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), CapPtr(p), key_global); + entry.get_remote()->trunc_id(), + capptr::AllocFull(p), + key_global); post_remote_cache(); return; } @@ -469,7 +471,7 @@ namespace snmalloc { local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), - CapPtr(p), + capptr::AllocFull(p), key_global); # ifdef SNMALLOC_TRACING std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p) @@ -501,7 +503,7 @@ namespace snmalloc # endif ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); - slab_record->chunk = CapPtr(p); + slab_record->chunk = capptr::Chunk(p); check_init( []( CoreAlloc* core_alloc, diff --git a/src/mem/localcache.h b/src/mem/localcache.h index eaaf6c1..a655cfb 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -13,19 +13,19 @@ namespace snmalloc using Stats = AllocStats; inline static SNMALLOC_FAST_PATH void* - finish_alloc_no_zero(CapPtr p, sizeclass_t sizeclass) + finish_alloc_no_zero(capptr::AllocFull p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); - auto r = capptr_reveal(capptr_export(p.as_void())); + auto r = capptr_reveal(capptr_to_user_address_control(p.as_void())); return r; } template inline static SNMALLOC_FAST_PATH void* - finish_alloc(CapPtr p, sizeclass_t sizeclass) + finish_alloc(capptr::AllocFull p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 66dbecb..a6015b2 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -5,7 +5,6 @@ #include "../ds/seqqueue.h" #include "../mem/remoteallocator.h" #include "freelist.h" -#include "ptrhelpers.h" #include "sizeclasstable.h" namespace snmalloc @@ -153,7 +152,7 @@ namespace snmalloc * component, but with randomisation, it may only return part of the * available objects for this metaslab. */ - static SNMALLOC_FAST_PATH std::pair, bool> + static SNMALLOC_FAST_PATH std::pair, bool> alloc_free_list( Metaslab* meta, FreeListIter& fast_free_list, diff --git a/src/mem/ptrhelpers.h b/src/mem/ptrhelpers.h deleted file mode 100644 index 8b0cee9..0000000 --- a/src/mem/ptrhelpers.h +++ /dev/null @@ -1,98 +0,0 @@ -#pragma once - -#include "../aal/aal.h" -#include "../ds/ptrwrap.h" -#include "allocconfig.h" - -namespace snmalloc -{ - /* - * At various points, we do pointer math on high-authority pointers to find - * some metadata. `capptr_bound_chunkd` and `capptr_chunk_from_chunkd` - * encapsulate the notion that the result of these accesses is left unbounded - * in non-debug builds, because most codepaths do not reveal these pointers or - * any progeny to the application. However, in some cases we have already - * (partially) bounded these high-authority pointers (to CBChunk) and wish to - * preserve this annotation (rather than always returning a CBChunkD-annotated - * pointer); `capptr_bound_chunkd_bounds` does the computation for us and is - * used in the signatures of below and in those of wrappers around them. - */ - - template - constexpr capptr_bounds capptr_bound_chunkd_bounds() - { - switch (B) - { - case CBArena: - return CBChunkD; - case CBChunkD: - return CBChunkD; - case CBChunk: - return CBChunk; - } - } - - /** - * Construct an CapPtr from an CapPtr or - * CapPtr input. For an CapPtr input, simply pass - * it through (preserving the static notion of bounds). - * - * Applies bounds on debug builds, otherwise is just sleight of hand. - * - * Requires that `p` point at a multiple of `sz` (that is, at the base of a - * highly-aligned object) to avoid representability issues. - */ - template - SNMALLOC_FAST_PATH CapPtr()> - capptr_bound_chunkd(CapPtr p, size_t sz) - { - static_assert(B == CBArena || B == CBChunkD || B == CBChunk); - SNMALLOC_ASSERT((address_cast(p) % sz) == 0); - -#ifndef NDEBUG - // On Debug builds, apply bounds if not already there - if constexpr (B == CBArena) - return Aal::capptr_bound(p, sz); - else // quiesce MSVC's warnings about unreachable code below -#endif - { - UNUSED(sz); - return CapPtr()>(p.unsafe_ptr()); - } - } - - /** - * Apply bounds that might not have been applied when constructing an - * CapPtr. That is, on non-debug builds, apply bounds; debug - * builds have already had them applied. - * - * Requires that `p` point at a multiple of `sz` (that is, at the base of a - * highly-aligned object) to avoid representability issues. - */ - template - SNMALLOC_FAST_PATH CapPtr - capptr_chunk_from_chunkd(CapPtr p, size_t sz) - { - SNMALLOC_ASSERT((address_cast(p) % sz) == 0); - -#ifndef NDEBUG - // On debug builds, CBChunkD are already bounded as if CBChunk. - UNUSED(sz); - return CapPtr(p.unsafe_ptr()); -#else - // On non-debug builds, apply bounds now, as they haven't been already. - return Aal::capptr_bound(p, sz); -#endif - } - - /** - * Very rarely, while debugging, it's both useful and acceptable to forget - * that we have applied chunk bounds to something. - */ - template - SNMALLOC_FAST_PATH CapPtr - capptr_debug_chunkd_from_chunk(CapPtr p) - { - return CapPtr(p.unsafe_ptr()); - } -} // namespace snmalloc diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index e02c2c8..f9596c7 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -35,10 +35,10 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) AtomicCapPtr back{nullptr}; + alignas(CACHELINE_SIZE) capptr::AtomicAllocFull back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) CapPtr front{nullptr}; + alignas(CACHELINE_SIZE) capptr::AllocFull front{nullptr}; constexpr RemoteAllocator() = default; @@ -48,7 +48,7 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void init(CapPtr stub) + void init(capptr::AllocFull stub) { stub->atomic_store_null(key_global); front = stub; @@ -56,9 +56,9 @@ namespace snmalloc invariant(); } - CapPtr destroy() + capptr::AllocFull destroy() { - CapPtr fnt = front; + capptr::AllocFull fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -66,7 +66,7 @@ namespace snmalloc inline bool is_empty() { - CapPtr bk = back.load(std::memory_order_relaxed); + capptr::AllocFull bk = back.load(std::memory_order_relaxed); return bk == front; } @@ -76,21 +76,21 @@ namespace snmalloc * last should be linked together through their next pointers. */ void enqueue( - CapPtr first, - CapPtr last, + capptr::AllocFull first, + capptr::AllocFull last, const FreeListKey& key) { invariant(); last->atomic_store_null(key); // exchange needs to be a release, so nullptr in next is visible. - CapPtr prev = + capptr::AllocFull prev = back.exchange(last, std::memory_order_release); prev->atomic_store_next(first, key); } - CapPtr peek() + capptr::AllocFull peek() { return front; } @@ -98,11 +98,12 @@ namespace snmalloc /** * Returns the front message, or null if not possible to return a message. */ - std::pair, bool> dequeue(const FreeListKey& key) + std::pair, bool> + dequeue(const FreeListKey& key) { invariant(); - CapPtr first = front; - CapPtr next = first->atomic_read_next(key); + capptr::AllocFull first = front; + capptr::AllocFull next = first->atomic_read_next(key); if (next != nullptr) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 4b88d23..0bf11ed 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -66,7 +66,7 @@ namespace snmalloc template SNMALLOC_FAST_PATH void dealloc( RemoteAllocator::alloc_id_t target_id, - CapPtr p, + capptr::AllocFull p, const FreeListKey& key) { SNMALLOC_ASSERT(initialised); diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 5c91b44..aa4d47f 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -18,7 +18,7 @@ namespace snmalloc struct ChunkRecord { std::atomic next; - CapPtr chunk; + capptr::Chunk chunk; }; /** @@ -76,7 +76,7 @@ namespace snmalloc { public: template - static std::pair, Metaslab*> alloc_chunk( + static std::pair, Metaslab*> alloc_chunk( typename SharedStateHandle::LocalState& local_state, sizeclass_t sizeclass, sizeclass_t slab_sizeclass, // TODO sizeclass_t @@ -163,7 +163,7 @@ namespace snmalloc // Cache line align size_t size = bits::align_up(sizeof(U), 64); - CapPtr p = + capptr::Chunk p = SharedStateHandle::template alloc_meta_data(local_state, size); if (p == nullptr) diff --git a/src/pal/pal.h b/src/pal/pal.h index d95ef0f..a2ad115 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -69,28 +69,46 @@ namespace snmalloc // Used to keep Superslab metadata committed. static constexpr size_t OS_PAGE_SIZE = Pal::page_size; + /** + * Compute the AddressSpaceControl::User variant of a capptr::bound + * annotation. This is used by the PAL's capptr_export function to compute + * its return value's annotation. + */ + template + using capptr_user_address_control_type = + typename B::template with_address_space_control< + capptr::dimension::AddressSpaceControl::User>; + /** * Perform platform-specific adjustment of return pointers. * * This is here, rather than in every PAL proper, merely to minimize * disruption to PALs for platforms that do not support StrictProvenance AALs. */ - template + template< + typename PAL = Pal, + typename AAL = Aal, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> static inline typename std::enable_if_t< !aal_supports, - CapPtr()>> - capptr_export(CapPtr p) + CapPtr>> + capptr_to_user_address_control(CapPtr p) { - return CapPtr()>(p.unsafe_ptr()); + return CapPtr>(p.unsafe_capptr); } - template - static inline typename std::enable_if_t< + template< + typename PAL = Pal, + typename AAL = Aal, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + static SNMALLOC_FAST_PATH typename std::enable_if_t< aal_supports, - CapPtr()>> - capptr_export(CapPtr p) + CapPtr>> + capptr_to_user_address_control(CapPtr p) { - return PAL::capptr_export(p); + return PAL::capptr_to_user_address_control(p); } /** @@ -101,12 +119,16 @@ namespace snmalloc * disruption and avoid code bloat. This wrapper ought to compile down to * nothing if SROA is doing its job. */ - template + template< + typename PAL, + bool page_aligned = false, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> static SNMALLOC_FAST_PATH void pal_zero(CapPtr p, size_t sz) { static_assert( - !page_aligned || B == CBArena || B == CBChunkD || B == CBChunk); - PAL::template zero(p.unsafe_ptr(), sz); + !page_aligned || B::spatial >= capptr::dimension::Spatial::Chunk); + PAL::template zero(p.unsafe_capptr, sz); } static_assert(