From 906589318126acd285f60d82e7cca035a9d522b9 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 19 Jul 2021 09:49:18 +0100 Subject: [PATCH] Overhaul CapPtr * Switch to a multidimensional taxonomy. Rather than encoding the abstract bound states in a single enum, move to a more algebraic treatment. The dimensions themselves are within the snmalloc::capptr_bounds namespace so that their fairly generic names do not conflict with consumer code. Aliases for many points in the space are established outside that namespace for ease of use elsewhere. * Introduce several new namespaces: * snmalloc::capptr::dimension holds each of the dimension enums * snmalloc::capptr holds the bound<> type itself and a ConceptBound * snmalloc::capptr::bounds gives convenient specializations of bound<> * snmalloc::capptr also has aliases for CapPtr<> itself All told, rather than `CapPtr`, we now expect client code to read `capptr::Chunk` in almost all cases (and this is just an alias for the appropriate `CapPtr>` type). When the bound<>s themselves are necessary, as when calling capptr_bound, we expect that they will almost always be pronounced using an alias (e.g., `capptr::bounds::Alloc`). * Chase consequences. * Prune old taxa and aliases that are no longer in use in snmalloc2. --- src/aal/aal.h | 23 ++- src/aal/aal_concept.h | 9 +- src/backend/address_space.h | 18 +- src/backend/address_space_core.h | 35 ++-- src/backend/backend.h | 20 +- src/ds/address.h | 28 ++- src/ds/ptrwrap.h | 339 ++++++++++++++++++++----------- src/mem/corealloc.h | 26 ++- src/mem/freelist.h | 53 ++--- src/mem/localalloc.h | 8 +- src/mem/localcache.h | 6 +- src/mem/metaslab.h | 3 +- src/mem/ptrhelpers.h | 98 --------- src/mem/remoteallocator.h | 27 +-- src/mem/remotecache.h | 2 +- src/mem/slaballocator.h | 6 +- src/pal/pal.h | 46 +++-- 17 files changed, 399 insertions(+), 348 deletions(-) delete mode 100644 src/mem/ptrhelpers.h diff --git a/src/aal/aal.h b/src/aal/aal.h index 00f6362..7e2af37 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -189,32 +189,33 @@ namespace snmalloc */ template< typename T, - enum capptr_bounds nbounds, - enum capptr_bounds obounds, + SNMALLOC_CONCEPT(capptr::ConceptBound) BOut, + SNMALLOC_CONCEPT(capptr::ConceptBound) BIn, typename U = T> - static SNMALLOC_FAST_PATH CapPtr - capptr_bound(CapPtr a, size_t size) noexcept + static SNMALLOC_FAST_PATH CapPtr + capptr_bound(CapPtr a, size_t size) noexcept { // Impose constraints on bounds annotations. - static_assert( - obounds == CBArena || obounds == CBChunkD || obounds == CBChunk || - obounds == CBChunkE); - static_assert(capptr_is_bounds_refinement()); + static_assert(BIn::spatial >= capptr::dimension::Spatial::Chunk); + static_assert(capptr_is_spatial_refinement()); UNUSED(size); - return CapPtr(a.template as_static().unsafe_ptr()); + return CapPtr(a.template as_static().unsafe_capptr); } /** * For architectures which do not enforce StrictProvenance, there's nothing * to be done, so just return the pointer unmodified with new annotation. */ - template + template< + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) BOut, + SNMALLOC_CONCEPT(capptr::ConceptBound) BIn> static SNMALLOC_FAST_PATH CapPtr capptr_rebound(CapPtr a, CapPtr r) noexcept { UNUSED(a); - return CapPtr(r.unsafe_ptr()); + return CapPtr(r.unsafe_capptr); } }; } // namespace snmalloc diff --git a/src/aal/aal_concept.h b/src/aal/aal_concept.h index 180e893..2a14a94 100644 --- a/src/aal/aal_concept.h +++ b/src/aal/aal_concept.h @@ -43,21 +43,22 @@ namespace snmalloc template concept ConceptAAL_capptr_methods = - requires(CapPtr auth, CapPtr ret, size_t sz) + requires(capptr::Chunk auth, capptr::AllocFull ret, size_t sz) { /** * Produce a pointer with reduced authority from a more privilged pointer. * The resulting pointer will have base at auth's address and length of * exactly sz. auth+sz must not exceed auth's limit. */ - { AAL::template capptr_bound(auth, sz) } noexcept - -> ConceptSame>; + { AAL::template capptr_bound(auth, sz) } + noexcept + -> ConceptSame>; /** * Construct a copy of auth with its target set to that of ret. */ { AAL::capptr_rebound(auth, ret) } noexcept - -> ConceptSame>; + -> ConceptSame>; }; template diff --git a/src/backend/address_space.h b/src/backend/address_space.h index ada5295..e197e77 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -43,7 +43,7 @@ namespace snmalloc * arena_map for use in subsequent amplification. */ template - CapPtr + capptr::Chunk reserve(typename Pagemap::LocalState* local_state, size_t size) { #ifdef SNMALLOC_TRACING @@ -62,21 +62,21 @@ namespace snmalloc { if (size >= PAL::minimum_alloc_size) { - auto base = CapPtr( - PAL::template reserve_aligned(size)); + auto base = + capptr::Chunk(PAL::template reserve_aligned(size)); Pagemap::register_range(local_state, address_cast(base), size); return base; } } - CapPtr res; + capptr::Chunk res; { FlagLock lock(spin_lock); res = core.template reserve(local_state, size); if (res == nullptr) { // Allocation failed ask OS for more memory - CapPtr block = nullptr; + capptr::Chunk block = nullptr; size_t block_size = 0; if constexpr (pal_supports) { @@ -92,7 +92,7 @@ namespace snmalloc // It's a bit of a lie to convert without applying bounds, but the // platform will have bounded block for us and it's better that // the rest of our internals expect CBChunk bounds. - block = CapPtr(block_raw); + block = capptr::Chunk(block_raw); } else if constexpr (!pal_supports) { @@ -110,7 +110,7 @@ namespace snmalloc size_request >= needed_size; size_request = size_request / 2) { - block = CapPtr(PAL::reserve(size_request)); + block = capptr::Chunk(PAL::reserve(size_request)); if (block != nullptr) { block_size = size_request; @@ -158,7 +158,7 @@ namespace snmalloc * used, by smaller objects. */ template - CapPtr reserve_with_left_over( + capptr::Chunk reserve_with_left_over( typename Pagemap::LocalState* local_state, size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -198,7 +198,7 @@ namespace snmalloc template void add_range( typename Pagemap::LocalState* local_state, - CapPtr base, + capptr::Chunk base, size_t length) { FlagLock lock(spin_lock); diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 0aa5872..d3088c0 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -25,7 +25,7 @@ namespace snmalloc { struct FreeChunk { - CapPtr next; + capptr::Chunk next; }; /** @@ -43,12 +43,12 @@ namespace snmalloc * bits::BITS is used for simplicity, we do not use below the pointer size, * and large entries will be unlikely to be supported by the platform. */ - std::array, bits::BITS> ranges = {}; + std::array, bits::BITS> ranges = {}; /** * Checks a block satisfies its invariant. */ - inline void check_block(CapPtr base, size_t align_bits) + inline void check_block(capptr::Chunk base, size_t align_bits) { SNMALLOC_ASSERT( address_cast(base) == @@ -72,8 +72,8 @@ namespace snmalloc void set_next( typename Pagemap::LocalState* local_state, size_t align_bits, - CapPtr base, - CapPtr next) + capptr::Chunk base, + capptr::Chunk next) { if (align_bits >= MIN_CHUNK_BITS) { @@ -102,16 +102,16 @@ namespace snmalloc * particular size. */ template - CapPtr get_next( + capptr::Chunk get_next( typename Pagemap::LocalState* local_state, size_t align_bits, - CapPtr base) + capptr::Chunk base) { if (align_bits >= MIN_CHUNK_BITS) { const MetaEntry& t = Pagemap::template get_metaentry( local_state, address_cast(base)); - return CapPtr( + return capptr::Chunk( reinterpret_cast(t.get_metaslab())); } @@ -127,7 +127,7 @@ namespace snmalloc void add_block( typename Pagemap::LocalState* local_state, size_t align_bits, - CapPtr base) + capptr::Chunk base) { check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); @@ -143,10 +143,10 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr + capptr::Chunk remove_block(typename Pagemap::LocalState* local_state, size_t align_bits) { - CapPtr first = ranges[align_bits]; + capptr::Chunk first = ranges[align_bits]; if (first == nullptr) { if (align_bits == (bits::BITS - 1)) @@ -156,7 +156,7 @@ namespace snmalloc } // Look for larger block and split up recursively - CapPtr bigger = + capptr::Chunk bigger = remove_block(local_state, align_bits + 1); if (bigger != nullptr) { @@ -174,7 +174,8 @@ namespace snmalloc add_block( local_state, align_bits, - Aal::capptr_bound(left_over, left_over_size)); + Aal::capptr_bound( + left_over, left_over_size)); check_block(left_over.as_static(), align_bits); } check_block(bigger.as_static(), align_bits + 1); @@ -196,7 +197,7 @@ namespace snmalloc SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> void add_range( typename Pagemap::LocalState* local_state, - CapPtr base, + capptr::Chunk base, size_t length) { // For start and end that are not chunk sized, we need to @@ -233,7 +234,7 @@ namespace snmalloc * Commit a block of memory */ template - void commit_block(CapPtr base, size_t size) + void commit_block(capptr::Chunk base, size_t size) { // Rounding required for sub-page allocations. auto page_start = pointer_align_down(base); @@ -257,7 +258,7 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr + capptr::Chunk reserve(typename Pagemap::LocalState* local_state, size_t size) { #ifdef SNMALLOC_TRACING @@ -281,7 +282,7 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr reserve_with_left_over( + capptr::Chunk reserve_with_left_over( typename Pagemap::LocalState* local_state, size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); diff --git a/src/backend/backend.h b/src/backend/backend.h index 8f36b7e..daf0363 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -39,7 +39,7 @@ namespace snmalloc * Backend allocator may use guard pages and separate area of * address space to protect this from corruption. */ - static CapPtr alloc_meta_data( + static capptr::Chunk alloc_meta_data( AddressSpaceManager& global, LocalState* local_state, size_t size) { return reserve(global, local_state, size); @@ -54,7 +54,7 @@ namespace snmalloc * (remote, sizeclass, metaslab) * where metaslab, is the second element of the pair return. */ - static std::pair, Metaslab*> alloc_chunk( + static std::pair, Metaslab*> alloc_chunk( AddressSpaceManager& global, LocalState* local_state, size_t size, @@ -70,7 +70,7 @@ namespace snmalloc if (meta == nullptr) return {nullptr, nullptr}; - CapPtr p = reserve(global, local_state, size); + capptr::Chunk p = reserve(global, local_state, size); #ifdef SNMALLOC_TRACING std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" @@ -98,7 +98,7 @@ namespace snmalloc * space managers. */ template - static CapPtr reserve( + static capptr::Chunk reserve( AddressSpaceManager& global, LocalState* local_state, size_t size) { #ifdef SNMALLOC_CHECK_CLIENT @@ -108,7 +108,7 @@ namespace snmalloc constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK; #endif - CapPtr p; + capptr::Chunk p; if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) { #ifdef SNMALLOC_CHECK_CLIENT @@ -182,8 +182,8 @@ namespace snmalloc * the range [base, base+full_size]. The first and last slot are not used * so that the edges can be used for guard pages. */ - static CapPtr - sub_range(CapPtr base, size_t full_size, size_t sub_size) + static capptr::Chunk + sub_range(capptr::Chunk base, size_t full_size, size_t sub_size) { SNMALLOC_ASSERT(bits::is_pow2(full_size)); SNMALLOC_ASSERT(bits::is_pow2(sub_size)); @@ -318,7 +318,7 @@ namespace snmalloc auto [heap_base, heap_length] = Pagemap::concretePagemap.init(base, length); address_space.template add_range( - local_state, CapPtr(heap_base), heap_length); + local_state, capptr::Chunk(heap_base), heap_length); } /** @@ -332,7 +332,7 @@ namespace snmalloc * places or with different policies. */ template - static CapPtr + static capptr::Chunk alloc_meta_data(LocalState* local_state, size_t size) { return AddressSpaceAllocator::alloc_meta_data( @@ -348,7 +348,7 @@ namespace snmalloc * (remote, sizeclass, metaslab) * where metaslab, is the second element of the pair return. */ - static std::pair, Metaslab*> alloc_chunk( + static std::pair, Metaslab*> alloc_chunk( LocalState* local_state, size_t size, RemoteAllocator* remote, diff --git a/src/ds/address.h b/src/ds/address.h index ae1c553..90b0bb4 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -35,7 +35,7 @@ namespace snmalloc reinterpret_cast(base) + static_cast(diff)); } - template + template inline CapPtr pointer_offset(CapPtr base, size_t diff) { @@ -52,7 +52,7 @@ namespace snmalloc return reinterpret_cast(reinterpret_cast(base) + diff); } - template + template inline CapPtr pointer_offset_signed(CapPtr base, ptrdiff_t diff) { @@ -76,7 +76,7 @@ namespace snmalloc * capptr_bound. */ - template + template inline SNMALLOC_FAST_PATH address_t address_cast(CapPtr a) { return address_cast(a.unsafe_ptr()); @@ -132,7 +132,10 @@ namespace snmalloc pointer_align_down(reinterpret_cast(p))); } - template + template< + size_t alignment, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) bounds> inline CapPtr pointer_align_down(CapPtr p) { return CapPtr(pointer_align_down(p.unsafe_ptr())); @@ -166,7 +169,10 @@ namespace snmalloc } } - template + template< + size_t alignment, + typename T = void, + SNMALLOC_CONCEPT(capptr::ConceptBound) bounds> inline CapPtr pointer_align_up(CapPtr p) { return CapPtr(pointer_align_up(p.unsafe_ptr())); @@ -195,7 +201,7 @@ namespace snmalloc #endif } - template + template inline CapPtr pointer_align_down(CapPtr p, size_t alignment) { @@ -219,7 +225,7 @@ namespace snmalloc #endif } - template + template inline CapPtr pointer_align_up(CapPtr p, size_t alignment) { @@ -241,8 +247,8 @@ namespace snmalloc template< typename T = void, typename U = void, - enum capptr_bounds Tbounds, - enum capptr_bounds Ubounds> + SNMALLOC_CONCEPT(capptr::ConceptBound) Tbounds, + SNMALLOC_CONCEPT(capptr::ConceptBound) Ubounds> inline size_t pointer_diff(CapPtr base, CapPtr cursor) { return pointer_diff(base.unsafe_ptr(), cursor.unsafe_ptr()); @@ -261,8 +267,8 @@ namespace snmalloc template< typename T = void, typename U = void, - enum capptr_bounds Tbounds, - enum capptr_bounds Ubounds> + SNMALLOC_CONCEPT(capptr::ConceptBound) Tbounds, + SNMALLOC_CONCEPT(capptr::ConceptBound) Ubounds> inline ptrdiff_t pointer_diff_signed(CapPtr base, CapPtr cursor) { diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index ff073e9..11c9926 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -1,6 +1,7 @@ #pragma once -#include "defines.h" +#include "../ds/concept.h" +#include "../ds/defines.h" #include @@ -21,78 +22,150 @@ namespace snmalloc /** * Summaries of StrictProvenance metadata. We abstract away the particular * size and any offset into the bounds. - * - * CBArena is as powerful as our pointers get: they're results from mmap(), - * and so confer as much authority as the kernel has given us. - * - * CBChunk is restricted to either a single chunk (SUPERSLAB_SIZE) or perhaps - * to several if we've requesed a large allocation (see capptr_chunk_is_alloc - * and its uses). - * - * CBChunkD is curious: we often use CBArena-bounded pointers to derive - * pointers to Allocslab metadata, and on most fast paths these pointers end - * up being ephemeral. As such, on NDEBUG builds, we elide the capptr_bounds - * that would bound these to chunks and instead just unsafely inherit the - * CBArena bounds. The use of CBChunkD thus helps to ensure that we - * eventually do invoke capptr_bounds when these pointers end up being longer - * lived! - * - * *E forms are "exported" and have had platform constraints applied. That - * means, for example, on CheriBSD, that they have had their VMMAP permission - * stripped. - * - * Yes, I wish the start-of-comment characters were aligned below as well. - * I blame clang format. */ - enum capptr_bounds + + namespace capptr { - /* Spatial Notes */ - CBArena, /* Arena */ - CBChunkD, /* Arena Chunk-bounded in debug; internal use only! */ - CBChunk, /* Chunk */ - CBChunkE, /* Chunk (+ platform constraints) */ - CBAlloc, /* Alloc */ - CBAllocE /* Alloc (+ platform constraints) */ - }; + namespace dimension + { + /* + * Describe the spatial extent (intended to be) authorized by a pointer. + * + * Bounds dimensions are sorted so that < reflects authority. + */ + enum class Spatial + { + /** + * Bounded to a particular allocation (which might be Large!) + */ + Alloc, + /** + * Bounded to one or more particular chunk granules + */ + Chunk, + }; + + /** + * On some platforms (e.g., CHERI), pointers can be checked to see whether + * they authorize control of the address space. See the PAL's + * capptr_export(). + */ + enum class AddressSpaceControl + { + /** + * All intended control constraints have been applied. For example, on + * CheriBSD, the VMMAP permission has been stripped and so this CapPtr<> + * cannot authorize manipulation of the address space itself, though it + * continues to authorize loads and stores. + */ + User, + /** + * No control constraints have been applied. On CheriBSD, specifically, + * this grants control of the address space (via mmap and friends) and + * in Cornucopia exempts the pointer from revocation (as long as the + * mapping remains in place, but snmalloc does not presently tear down + * its own mappings.) + */ + Full + }; + + } // namespace dimension + + /** + * The aggregate type of a bound: a Cartesian product of the individual + * dimensions, above. + */ + template + struct bound + { + static constexpr enum dimension::Spatial spatial = S; + static constexpr enum dimension::AddressSpaceControl + address_space_control = AS; + + /** + * Set just the spatial component of the bounds + */ + template + using with_spatial = bound; + + /** + * Set just the address space control component of the bounds + */ + template + using with_address_space_control = bound; + }; + + // clang-format off +#ifdef __cpp_concepts + /* + * This is spelled a little differently from our other concepts because GCC + * treats "{ T::spatial }" as generating a reference and then complains that + * it isn't "ConceptSame", though clang is perfectly happy + * with that spelling. Both seem happy with this formulation. + */ + template + concept ConceptBound = + ConceptSame && + ConceptSame; +#endif + // clang-format on + + /* + * Several combinations are used often enough that we give convenient + * aliases for them. + */ + namespace bounds + { + /** + * Internal access to a Chunk of memory. These flow between the ASM and + * the slab allocators, for example. + */ + using Chunk = + bound; + + /** + * User access to an entire Chunk. Used as an ephemeral state when + * returning a large allocation. See capptr_chunk_is_alloc. + */ + using ChunkUser = + Chunk::with_address_space_control; + + /** + * Internal access to just one allocation (usually, within a slab). + */ + using AllocFull = Chunk::with_spatial; + + /** + * User access to just one allocation (usually, within a slab). + */ + using Alloc = AllocFull::with_address_space_control< + dimension::AddressSpaceControl::User>; + } // namespace bounds + } // namespace capptr /** - * Compute the "exported" variant of a capptr_bounds annotation. This is - * used by the PAL's capptr_export function to compute its return value's - * annotation. + * Determine whether BI is a spatial refinement of BO. + * Chunk and ChunkD are considered eqivalent here. */ - template - SNMALLOC_CONSTEVAL capptr_bounds capptr_export_type() + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BI, + SNMALLOC_CONCEPT(capptr::ConceptBound) BO> + SNMALLOC_CONSTEVAL bool capptr_is_spatial_refinement() { - static_assert( - (B == CBChunk) || (B == CBAlloc), "capptr_export_type of bad type"); - - switch (B) + if (BI::address_space_control != BO::address_space_control) { - case CBChunk: - return CBChunkE; - case CBAlloc: - return CBAllocE; + return false; } - } - template - SNMALLOC_CONSTEVAL bool capptr_is_bounds_refinement() - { - switch (BI) + switch (BI::spatial) { - case CBAllocE: - return BO == CBAllocE; - case CBAlloc: - return BO == CBAlloc; - case CBChunkE: - return BO == CBAllocE || BO == CBChunkE; - case CBChunk: - return BO == CBAlloc || BO == CBChunk || BO == CBChunkD; - case CBChunkD: - return BO == CBAlloc || BO == CBChunk || BO == CBChunkD; - case CBArena: - return BO == CBAlloc || BO == CBChunk || BO == CBChunkD || - BO == CBArena; + using namespace capptr::dimension; + case Spatial::Chunk: + return true; + + case Spatial::Alloc: + return BO::spatial == Spatial::Alloc; } } @@ -100,18 +173,19 @@ namespace snmalloc * A pointer annotated with a "phantom type parameter" carrying a static * summary of its StrictProvenance metadata. */ - template - class CapPtr + template + struct CapPtr { - uintptr_t unsafe_capptr; + T* unsafe_capptr; - public: /** * nullptr is implicitly constructable at any bounds type */ - constexpr CapPtr(const std::nullptr_t) : unsafe_capptr(0) {} + constexpr SNMALLOC_FAST_PATH CapPtr(const std::nullptr_t n) + : unsafe_capptr(n) + {} - constexpr CapPtr() : CapPtr(nullptr){}; + constexpr SNMALLOC_FAST_PATH CapPtr() : CapPtr(nullptr) {} /** * all other constructions must be explicit @@ -127,23 +201,21 @@ namespace snmalloc # pragma warning(push) # pragma warning(disable : 4702) #endif - constexpr explicit CapPtr(uintptr_t p) : unsafe_capptr(p) {} + constexpr explicit SNMALLOC_FAST_PATH CapPtr(T* p) : unsafe_capptr(p) {} #ifdef _MSC_VER # pragma warning(pop) #endif - explicit CapPtr(T* p) : unsafe_capptr(reinterpret_cast(p)) {} - /** * Allow static_cast<>-s that preserve bounds but vary the target type. */ template - SNMALLOC_FAST_PATH CapPtr as_static() + [[nodiscard]] SNMALLOC_FAST_PATH CapPtr as_static() const { - return CapPtr(this->unsafe_capptr); + return CapPtr(static_cast(this->unsafe_capptr)); } - SNMALLOC_FAST_PATH CapPtr as_void() + [[nodiscard]] SNMALLOC_FAST_PATH CapPtr as_void() const { return this->as_static(); } @@ -152,9 +224,9 @@ namespace snmalloc * A more aggressive bounds-preserving cast, using reinterpret_cast */ template - SNMALLOC_FAST_PATH CapPtr as_reinterpret() + [[nodiscard]] SNMALLOC_FAST_PATH CapPtr as_reinterpret() const { - return CapPtr(this->unsafe_capptr); + return CapPtr(reinterpret_cast(this->unsafe_capptr)); } SNMALLOC_FAST_PATH bool operator==(const CapPtr& rhs) const @@ -172,60 +244,82 @@ namespace snmalloc return this->unsafe_capptr < rhs.unsafe_capptr; } - [[nodiscard]] SNMALLOC_FAST_PATH T* unsafe_ptr() const + SNMALLOC_FAST_PATH T* operator->() const { - return reinterpret_cast(this->unsafe_capptr); + /* + * Alloc-bounded, platform-constrained pointers are associated with + * objects coming from or going to the client; we should be doing nothing + * with them. + */ + static_assert( + (bounds::spatial != capptr::dimension::Spatial::Alloc) || + (bounds::address_space_control != + capptr::dimension::AddressSpaceControl::User)); + return this->unsafe_capptr; } - [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t unsafe_uintptr() const + [[nodiscard]] SNMALLOC_FAST_PATH T* unsafe_ptr() const { return this->unsafe_capptr; } - SNMALLOC_FAST_PATH T* operator->() const + [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t unsafe_uintptr() const { - /* - * CBAllocE bounds are associated with objects coming from or going to the - * client; we should be doing nothing with them. - */ - static_assert(bounds != CBAllocE); - return unsafe_ptr(); + return reinterpret_cast(this->unsafe_capptr); } }; - static_assert(sizeof(CapPtr) == sizeof(void*)); - static_assert(alignof(CapPtr) == alignof(void*)); + namespace capptr + { + /* + * Aliases for CapPtr<> types with particular bounds. + */ - template - using CapPtrCBArena = CapPtr; + template + using Chunk = CapPtr; - template - using CapPtrCBChunk = CapPtr; + template + using ChunkUser = CapPtr; - template - using CapPtrCBChunkE = CapPtr; + template + using AllocFull = CapPtr; - template - using CapPtrCBAlloc = CapPtr; + template + using Alloc = CapPtr; + + } // namespace capptr + + static_assert(sizeof(capptr::Chunk) == sizeof(void*)); + static_assert(alignof(capptr::Chunk) == alignof(void*)); /** * Sometimes (with large allocations) we really mean the entire chunk (or even * several chunks) to be the allocation. */ template - inline SNMALLOC_FAST_PATH CapPtr - capptr_chunk_is_alloc(CapPtr p) + inline SNMALLOC_FAST_PATH capptr::Alloc + capptr_chunk_is_alloc(capptr::ChunkUser p) { - return CapPtr(p.unsafe_capptr); + return capptr::Alloc(p.unsafe_capptr); } /** * With all the bounds and constraints in place, it's safe to extract a void - * pointer (to reveal to the client). + * pointer (to reveal to the client). Roughly dual to capptr_from_client(). */ - inline SNMALLOC_FAST_PATH void* capptr_reveal(CapPtr p) + inline SNMALLOC_FAST_PATH void* capptr_reveal(capptr::Alloc p) { - return p.unsafe_ptr(); + return p.unsafe_capptr; + } + + /** + * Given a void* from the client, it's fine to call it Alloc. Roughly + * dual to capptr_reveal(). + */ + static inline SNMALLOC_FAST_PATH capptr::Alloc + capptr_from_client(void* p) + { + return capptr::Alloc(p); } /** @@ -237,7 +331,7 @@ namespace snmalloc * annotations around an un-annotated std::atomic, to appease C++, yet * will expose or consume only CapPtr with the same bounds annotation. */ - template + template struct AtomicCapPtr { std::atomic unsafe_capptr; @@ -245,12 +339,16 @@ namespace snmalloc /** * nullptr is constructable at any bounds type */ - constexpr AtomicCapPtr(const std::nullptr_t n) : unsafe_capptr(n) {} + constexpr SNMALLOC_FAST_PATH AtomicCapPtr(const std::nullptr_t n) + : unsafe_capptr(n) + {} /** * Interconversion with CapPtr */ - AtomicCapPtr(CapPtr p) : unsafe_capptr(p.unsafe_ptr()) {} + constexpr SNMALLOC_FAST_PATH AtomicCapPtr(CapPtr p) + : unsafe_capptr(p.unsafe_capptr) + {} operator CapPtr() const noexcept { @@ -260,7 +358,7 @@ namespace snmalloc // Our copy-assignment operator follows std::atomic and returns a copy of // the RHS. Clang finds this surprising; we suppress the warning. // NOLINTNEXTLINE(misc-unconventional-assign-operator) - CapPtr operator=(CapPtr p) noexcept + SNMALLOC_FAST_PATH CapPtr operator=(CapPtr p) noexcept { this->store(p); return p; @@ -276,7 +374,7 @@ namespace snmalloc CapPtr desired, std::memory_order order = std::memory_order_seq_cst) noexcept { - this->unsafe_capptr.store(desired.unsafe_ptr(), order); + this->unsafe_capptr.store(desired.unsafe_capptr, order); } SNMALLOC_FAST_PATH CapPtr exchange( @@ -284,7 +382,7 @@ namespace snmalloc std::memory_order order = std::memory_order_seq_cst) noexcept { return CapPtr( - this->unsafe_capptr.exchange(desired.unsafe_ptr(), order)); + this->unsafe_capptr.exchange(desired.unsafe_capptr, order)); } SNMALLOC_FAST_PATH bool operator==(const AtomicCapPtr& rhs) const @@ -303,13 +401,24 @@ namespace snmalloc } }; - template - using AtomicCapPtrCBArena = AtomicCapPtr; + namespace capptr + { + /* + * Aliases for AtomicCapPtr<> types with particular bounds. + */ - template - using AtomicCapPtrCBChunk = AtomicCapPtr; + template + using AtomicChunk = AtomicCapPtr; - template - using AtomicCapPtrCBAlloc = AtomicCapPtr; + template + using AtomicChunkUser = AtomicCapPtr; + + template + using AtomicAllocFull = AtomicCapPtr; + + template + using AtomicAlloc = AtomicCapPtr; + + } // namespace capptr } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index cb3869d..881ce10 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -161,7 +161,7 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - auto dummy = CapPtr(small_alloc_one(MIN_ALLOC_SIZE)) + auto dummy = capptr::AllocFull(small_alloc_one(MIN_ALLOC_SIZE)) .template as_static(); if (dummy == nullptr) { @@ -187,7 +187,7 @@ namespace snmalloc } static SNMALLOC_FAST_PATH void alloc_new_list( - CapPtr& bumpptr, + capptr::Chunk& bumpptr, Metaslab* meta, size_t rsize, size_t slab_size, @@ -203,7 +203,7 @@ namespace snmalloc // Structure to represent the temporary list elements struct PreAllocObject { - CapPtr next; + capptr::AllocFull next; }; // The following code implements Sattolo's algorithm for generating // random cyclic permutations. This implementation is in the opposite @@ -213,9 +213,10 @@ namespace snmalloc // Note the wide bounds on curr relative to each of the ->next fields; // curr is not persisted once the list is built. - CapPtr curr = + capptr::Chunk curr = pointer_offset(bumpptr, 0).template as_static(); - curr->next = Aal::capptr_bound(curr, rsize); + curr->next = Aal::capptr_bound( + curr, rsize); uint16_t count = 1; for (curr = @@ -229,12 +230,13 @@ namespace snmalloc pointer_offset(bumpptr, insert_index * rsize) .template as_static() ->next, - Aal::capptr_bound(curr, rsize)); + Aal::capptr_bound( + curr, rsize)); count++; } // Pick entry into space, and then build linked list by traversing cycle - // to the start. Use ->next to jump from CBArena to CBAlloc. + // to the start. Use ->next to jump from Chunk to Alloc. auto start_index = entropy.sample(count); auto start_ptr = pointer_offset(bumpptr, start_index * rsize) .template as_static() @@ -249,7 +251,9 @@ namespace snmalloc auto p = bumpptr; do { - b.add(Aal::capptr_bound(p, rsize), key); + b.add( + Aal::capptr_bound(p, rsize), + key); p = pointer_offset(p, rsize); } while (p < slab_end); #endif @@ -299,7 +303,7 @@ namespace snmalloc auto start_of_slab = pointer_align_down( p, snmalloc::sizeclass_to_slab_size(sizeclass)); // TODO Add bounds correctly here - chunk_record->chunk = CapPtr(start_of_slab); + chunk_record->chunk = capptr::Chunk(start_of_slab); #ifdef SNMALLOC_TRACING std::cout << "Slab " << start_of_slab << " is unused, Object sizeclass " @@ -422,7 +426,7 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaEntry& entry, CapPtr p, bool& need_post) + const MetaEntry& entry, capptr::AllocFull p, bool& need_post) { // TODO this needs to not double count stats // TODO this needs to not double revoke if using MTE @@ -576,7 +580,7 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = CapPtr(reinterpret_cast(p)); + auto cp = capptr::AllocFull(reinterpret_cast(p)); auto& key = entropy.get_free_list_key(); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index eab71eb..09db138 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -66,9 +66,9 @@ namespace snmalloc { union { - CapPtr next_object; + capptr::AllocFull next_object; // TODO: Should really use C++20 atomic_ref rather than a union. - AtomicCapPtr atomic_next_object; + capptr::AtomicAllocFull atomic_next_object; }; #ifdef SNMALLOC_CHECK_CLIENT // Encoded representation of a back pointer. @@ -78,7 +78,7 @@ namespace snmalloc #endif public: - static CapPtr make(CapPtr p) + static capptr::AllocFull make(capptr::AllocFull p) { return p.template as_static(); } @@ -86,8 +86,10 @@ namespace snmalloc /** * Encode next */ - inline static CapPtr encode_next( - address_t curr, CapPtr next, const FreeListKey& key) + inline static capptr::AllocFull encode_next( + address_t curr, + capptr::AllocFull next, + const FreeListKey& key) { // Note we can consider other encoding schemes here. // * XORing curr and next. This doesn't require any key material @@ -98,7 +100,8 @@ namespace snmalloc if constexpr (CHECK_CLIENT && !aal_supports) { - return CapPtr(address_cast(next) ^ key.key_next); + return capptr::AllocFull(reinterpret_cast( + reinterpret_cast(next.unsafe_ptr()) ^ key.key_next)); } else { @@ -115,9 +118,9 @@ namespace snmalloc * optimization for repeated snoc operations (in which * next->next_object is nullptr). */ - static CapPtr* store_next( - CapPtr* curr, - CapPtr next, + static capptr::AllocFull* store_next( + capptr::AllocFull* curr, + capptr::AllocFull next, const FreeListKey& key) { #ifdef SNMALLOC_CHECK_CLIENT @@ -131,7 +134,7 @@ namespace snmalloc } static void - store_null(CapPtr* curr, const FreeListKey& key) + store_null(capptr::AllocFull* curr, const FreeListKey& key) { *curr = encode_next(address_cast(curr), nullptr, key); } @@ -141,8 +144,8 @@ namespace snmalloc * * Uses the atomic view of next, so can be used in the message queues. */ - void - atomic_store_next(CapPtr next, const FreeListKey& key) + void atomic_store_next( + capptr::AllocFull next, const FreeListKey& key) { #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = @@ -164,7 +167,7 @@ namespace snmalloc std::memory_order_relaxed); } - CapPtr atomic_read_next(const FreeListKey& key) + capptr::AllocFull atomic_read_next(const FreeListKey& key) { auto n = encode_next( address_cast(&next_object), @@ -194,7 +197,7 @@ namespace snmalloc /** * Read the next pointer */ - CapPtr read_next(const FreeListKey& key) + capptr::AllocFull read_next(const FreeListKey& key) { return encode_next(address_cast(&next_object), next_object, key); } @@ -211,14 +214,14 @@ namespace snmalloc */ class FreeListIter { - CapPtr curr{nullptr}; + capptr::AllocFull curr{nullptr}; #ifdef SNMALLOC_CHECK_CLIENT address_t prev{0}; #endif public: constexpr FreeListIter( - CapPtr head, address_t prev_value) + capptr::AllocFull head, address_t prev_value) : curr(head) { #ifdef SNMALLOC_CHECK_CLIENT @@ -240,7 +243,7 @@ namespace snmalloc /** * Returns current head without affecting the iterator. */ - CapPtr peek() + capptr::AllocFull peek() { return curr; } @@ -248,7 +251,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - CapPtr take(const FreeListKey& key) + capptr::AllocFull take(const FreeListKey& key) { auto c = curr; auto next = curr->read_next(key); @@ -289,11 +292,11 @@ namespace snmalloc static constexpr size_t LENGTH = RANDOM ? 2 : 1; // Pointer to the first element. - std::array, LENGTH> head; + std::array, LENGTH> head; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - std::array*, LENGTH> end{nullptr}; + std::array*, LENGTH> end{nullptr}; std::array length{}; @@ -321,7 +324,7 @@ namespace snmalloc * Adds an element to the builder */ void add( - CapPtr n, + capptr::AllocFull n, const FreeListKey& key, LocalEntropy& entropy) { @@ -348,7 +351,7 @@ namespace snmalloc */ template std::enable_if_t - add(CapPtr n, const FreeListKey& key) + add(capptr::AllocFull n, const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); end[0] = FreeObject::store_next(end[0], n, key); @@ -372,7 +375,7 @@ namespace snmalloc * and is thus subject to encoding if the next_object pointers * encoded. */ - CapPtr + capptr::AllocFull read_head(uint32_t index, const FreeListKey& key) { return FreeObject::encode_next( @@ -444,7 +447,7 @@ namespace snmalloc template std::enable_if_t< !RANDOM_, - std::pair, CapPtr>> + std::pair, capptr::AllocFull>> extract_segment(const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); @@ -456,7 +459,7 @@ namespace snmalloc // to the actual object. This isn't true if the builder is // empty, but you are not allowed to call this in the empty case. auto last = - CapPtr(reinterpret_cast(end[0])); + capptr::AllocFull(reinterpret_cast(end[0])); init(); return {first, last}; } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 761754e..b20b3d3 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -263,7 +263,9 @@ namespace snmalloc MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), CapPtr(p), key_global); + entry.get_remote()->trunc_id(), + capptr::AllocFull(p), + key_global); post_remote_cache(); return; } @@ -469,7 +471,7 @@ namespace snmalloc { local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), - CapPtr(p), + capptr::AllocFull(p), key_global); # ifdef SNMALLOC_TRACING std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p) @@ -501,7 +503,7 @@ namespace snmalloc # endif ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); - slab_record->chunk = CapPtr(p); + slab_record->chunk = capptr::Chunk(p); check_init( []( CoreAlloc* core_alloc, diff --git a/src/mem/localcache.h b/src/mem/localcache.h index eaaf6c1..a655cfb 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -13,19 +13,19 @@ namespace snmalloc using Stats = AllocStats; inline static SNMALLOC_FAST_PATH void* - finish_alloc_no_zero(CapPtr p, sizeclass_t sizeclass) + finish_alloc_no_zero(capptr::AllocFull p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); - auto r = capptr_reveal(capptr_export(p.as_void())); + auto r = capptr_reveal(capptr_to_user_address_control(p.as_void())); return r; } template inline static SNMALLOC_FAST_PATH void* - finish_alloc(CapPtr p, sizeclass_t sizeclass) + finish_alloc(capptr::AllocFull p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 66dbecb..a6015b2 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -5,7 +5,6 @@ #include "../ds/seqqueue.h" #include "../mem/remoteallocator.h" #include "freelist.h" -#include "ptrhelpers.h" #include "sizeclasstable.h" namespace snmalloc @@ -153,7 +152,7 @@ namespace snmalloc * component, but with randomisation, it may only return part of the * available objects for this metaslab. */ - static SNMALLOC_FAST_PATH std::pair, bool> + static SNMALLOC_FAST_PATH std::pair, bool> alloc_free_list( Metaslab* meta, FreeListIter& fast_free_list, diff --git a/src/mem/ptrhelpers.h b/src/mem/ptrhelpers.h deleted file mode 100644 index 8b0cee9..0000000 --- a/src/mem/ptrhelpers.h +++ /dev/null @@ -1,98 +0,0 @@ -#pragma once - -#include "../aal/aal.h" -#include "../ds/ptrwrap.h" -#include "allocconfig.h" - -namespace snmalloc -{ - /* - * At various points, we do pointer math on high-authority pointers to find - * some metadata. `capptr_bound_chunkd` and `capptr_chunk_from_chunkd` - * encapsulate the notion that the result of these accesses is left unbounded - * in non-debug builds, because most codepaths do not reveal these pointers or - * any progeny to the application. However, in some cases we have already - * (partially) bounded these high-authority pointers (to CBChunk) and wish to - * preserve this annotation (rather than always returning a CBChunkD-annotated - * pointer); `capptr_bound_chunkd_bounds` does the computation for us and is - * used in the signatures of below and in those of wrappers around them. - */ - - template - constexpr capptr_bounds capptr_bound_chunkd_bounds() - { - switch (B) - { - case CBArena: - return CBChunkD; - case CBChunkD: - return CBChunkD; - case CBChunk: - return CBChunk; - } - } - - /** - * Construct an CapPtr from an CapPtr or - * CapPtr input. For an CapPtr input, simply pass - * it through (preserving the static notion of bounds). - * - * Applies bounds on debug builds, otherwise is just sleight of hand. - * - * Requires that `p` point at a multiple of `sz` (that is, at the base of a - * highly-aligned object) to avoid representability issues. - */ - template - SNMALLOC_FAST_PATH CapPtr()> - capptr_bound_chunkd(CapPtr p, size_t sz) - { - static_assert(B == CBArena || B == CBChunkD || B == CBChunk); - SNMALLOC_ASSERT((address_cast(p) % sz) == 0); - -#ifndef NDEBUG - // On Debug builds, apply bounds if not already there - if constexpr (B == CBArena) - return Aal::capptr_bound(p, sz); - else // quiesce MSVC's warnings about unreachable code below -#endif - { - UNUSED(sz); - return CapPtr()>(p.unsafe_ptr()); - } - } - - /** - * Apply bounds that might not have been applied when constructing an - * CapPtr. That is, on non-debug builds, apply bounds; debug - * builds have already had them applied. - * - * Requires that `p` point at a multiple of `sz` (that is, at the base of a - * highly-aligned object) to avoid representability issues. - */ - template - SNMALLOC_FAST_PATH CapPtr - capptr_chunk_from_chunkd(CapPtr p, size_t sz) - { - SNMALLOC_ASSERT((address_cast(p) % sz) == 0); - -#ifndef NDEBUG - // On debug builds, CBChunkD are already bounded as if CBChunk. - UNUSED(sz); - return CapPtr(p.unsafe_ptr()); -#else - // On non-debug builds, apply bounds now, as they haven't been already. - return Aal::capptr_bound(p, sz); -#endif - } - - /** - * Very rarely, while debugging, it's both useful and acceptable to forget - * that we have applied chunk bounds to something. - */ - template - SNMALLOC_FAST_PATH CapPtr - capptr_debug_chunkd_from_chunk(CapPtr p) - { - return CapPtr(p.unsafe_ptr()); - } -} // namespace snmalloc diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index e02c2c8..f9596c7 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -35,10 +35,10 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) AtomicCapPtr back{nullptr}; + alignas(CACHELINE_SIZE) capptr::AtomicAllocFull back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) CapPtr front{nullptr}; + alignas(CACHELINE_SIZE) capptr::AllocFull front{nullptr}; constexpr RemoteAllocator() = default; @@ -48,7 +48,7 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void init(CapPtr stub) + void init(capptr::AllocFull stub) { stub->atomic_store_null(key_global); front = stub; @@ -56,9 +56,9 @@ namespace snmalloc invariant(); } - CapPtr destroy() + capptr::AllocFull destroy() { - CapPtr fnt = front; + capptr::AllocFull fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -66,7 +66,7 @@ namespace snmalloc inline bool is_empty() { - CapPtr bk = back.load(std::memory_order_relaxed); + capptr::AllocFull bk = back.load(std::memory_order_relaxed); return bk == front; } @@ -76,21 +76,21 @@ namespace snmalloc * last should be linked together through their next pointers. */ void enqueue( - CapPtr first, - CapPtr last, + capptr::AllocFull first, + capptr::AllocFull last, const FreeListKey& key) { invariant(); last->atomic_store_null(key); // exchange needs to be a release, so nullptr in next is visible. - CapPtr prev = + capptr::AllocFull prev = back.exchange(last, std::memory_order_release); prev->atomic_store_next(first, key); } - CapPtr peek() + capptr::AllocFull peek() { return front; } @@ -98,11 +98,12 @@ namespace snmalloc /** * Returns the front message, or null if not possible to return a message. */ - std::pair, bool> dequeue(const FreeListKey& key) + std::pair, bool> + dequeue(const FreeListKey& key) { invariant(); - CapPtr first = front; - CapPtr next = first->atomic_read_next(key); + capptr::AllocFull first = front; + capptr::AllocFull next = first->atomic_read_next(key); if (next != nullptr) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 4b88d23..0bf11ed 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -66,7 +66,7 @@ namespace snmalloc template SNMALLOC_FAST_PATH void dealloc( RemoteAllocator::alloc_id_t target_id, - CapPtr p, + capptr::AllocFull p, const FreeListKey& key) { SNMALLOC_ASSERT(initialised); diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 5c91b44..aa4d47f 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -18,7 +18,7 @@ namespace snmalloc struct ChunkRecord { std::atomic next; - CapPtr chunk; + capptr::Chunk chunk; }; /** @@ -76,7 +76,7 @@ namespace snmalloc { public: template - static std::pair, Metaslab*> alloc_chunk( + static std::pair, Metaslab*> alloc_chunk( typename SharedStateHandle::LocalState& local_state, sizeclass_t sizeclass, sizeclass_t slab_sizeclass, // TODO sizeclass_t @@ -163,7 +163,7 @@ namespace snmalloc // Cache line align size_t size = bits::align_up(sizeof(U), 64); - CapPtr p = + capptr::Chunk p = SharedStateHandle::template alloc_meta_data(local_state, size); if (p == nullptr) diff --git a/src/pal/pal.h b/src/pal/pal.h index d95ef0f..a2ad115 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -69,28 +69,46 @@ namespace snmalloc // Used to keep Superslab metadata committed. static constexpr size_t OS_PAGE_SIZE = Pal::page_size; + /** + * Compute the AddressSpaceControl::User variant of a capptr::bound + * annotation. This is used by the PAL's capptr_export function to compute + * its return value's annotation. + */ + template + using capptr_user_address_control_type = + typename B::template with_address_space_control< + capptr::dimension::AddressSpaceControl::User>; + /** * Perform platform-specific adjustment of return pointers. * * This is here, rather than in every PAL proper, merely to minimize * disruption to PALs for platforms that do not support StrictProvenance AALs. */ - template + template< + typename PAL = Pal, + typename AAL = Aal, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> static inline typename std::enable_if_t< !aal_supports, - CapPtr()>> - capptr_export(CapPtr p) + CapPtr>> + capptr_to_user_address_control(CapPtr p) { - return CapPtr()>(p.unsafe_ptr()); + return CapPtr>(p.unsafe_capptr); } - template - static inline typename std::enable_if_t< + template< + typename PAL = Pal, + typename AAL = Aal, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + static SNMALLOC_FAST_PATH typename std::enable_if_t< aal_supports, - CapPtr()>> - capptr_export(CapPtr p) + CapPtr>> + capptr_to_user_address_control(CapPtr p) { - return PAL::capptr_export(p); + return PAL::capptr_to_user_address_control(p); } /** @@ -101,12 +119,16 @@ namespace snmalloc * disruption and avoid code bloat. This wrapper ought to compile down to * nothing if SROA is doing its job. */ - template + template< + typename PAL, + bool page_aligned = false, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> static SNMALLOC_FAST_PATH void pal_zero(CapPtr p, size_t sz) { static_assert( - !page_aligned || B == CBArena || B == CBChunkD || B == CBChunk); - PAL::template zero(p.unsafe_ptr(), sz); + !page_aligned || B::spatial >= capptr::dimension::Spatial::Chunk); + PAL::template zero(p.unsafe_capptr, sz); } static_assert(