diff --git a/src/backend/backend.h b/src/backend/backend.h index c84e00c..528abb5 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -1,6 +1,5 @@ #pragma once #include "../mem/allocconfig.h" -#include "../mem/metaslab.h" #include "../pal/pal.h" #include "chunkallocator.h" #include "commitrange.h" @@ -8,6 +7,7 @@ #include "empty_range.h" #include "globalrange.h" #include "largebuddyrange.h" +#include "metatypes.h" #include "pagemap.h" #include "pagemapregisterrange.h" #include "palrange.h" @@ -259,7 +259,7 @@ namespace snmalloc SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); auto meta_cap = - local_state.get_meta_range()->alloc_range(sizeof(Metaslab)); + local_state.get_meta_range()->alloc_range(PAGEMAP_METADATA_STRUCT_SIZE); auto meta = meta_cap.template as_reinterpret().unsafe_ptr(); @@ -277,7 +277,8 @@ namespace snmalloc #endif if (p == nullptr) { - local_state.get_meta_range()->dealloc_range(meta_cap, sizeof(Metaslab)); + local_state.get_meta_range()->dealloc_range( + meta_cap, PAGEMAP_METADATA_STRUCT_SIZE); errno = ENOMEM; #ifdef SNMALLOC_TRACING std::cout << "Out of memory" << std::endl; @@ -300,7 +301,7 @@ namespace snmalloc auto chunk = chunk_record->meta_common.chunk; local_state.get_meta_range()->dealloc_range( - capptr::Chunk(chunk_record), sizeof(Metaslab)); + capptr::Chunk(chunk_record), PAGEMAP_METADATA_STRUCT_SIZE); // TODO, should we set the sizeclass to something specific here? diff --git a/src/backend/chunkallocator.h b/src/backend/chunkallocator.h index 09e4fff..335f4f5 100644 --- a/src/backend/chunkallocator.h +++ b/src/backend/chunkallocator.h @@ -7,6 +7,7 @@ */ #include "../backend/backend_concept.h" +#include "../backend/metatypes.h" #include "../ds/mpmcstack.h" #include "../ds/spmcstack.h" #include "../mem/metaslab.h" @@ -29,22 +30,15 @@ namespace snmalloc MetaCommon meta_common; std::atomic next; }; - static_assert(std::is_standard_layout_v); - static_assert( - offsetof(ChunkRecord, meta_common) == 0, - "ChunkRecord and Metaslab must share a common prefix"); +#if defined(USE_METADATA_CONCEPT) + static_assert(ConceptMetadataStruct); +#endif /** * How many slab sizes that can be provided. */ constexpr size_t NUM_SLAB_SIZES = Pal::address_bits - MIN_CHUNK_BITS; - /** - * Used to ensure the per slab meta data is large enough for both use cases. - */ - static_assert( - sizeof(Metaslab) >= sizeof(ChunkRecord), "We conflate these two types."); - /** * Number of free stacks per chunk size that each allocator will use. * For performance ideally a power of 2. We will return to the central @@ -256,7 +250,7 @@ namespace snmalloc #endif state.add_peak_memory_usage(slab_size); - state.add_peak_memory_usage(sizeof(Metaslab)); + state.add_peak_memory_usage(PAGEMAP_METADATA_STRUCT_SIZE); // TODO handle bounded versus lazy pagemaps in stats state.add_peak_memory_usage( (slab_size / MIN_CHUNK_SIZE) * sizeof(MetaEntry)); diff --git a/src/backend/largebuddyrange.h b/src/backend/largebuddyrange.h index 5f18248..889f7e0 100644 --- a/src/backend/largebuddyrange.h +++ b/src/backend/largebuddyrange.h @@ -3,7 +3,6 @@ #include "../ds/address.h" #include "../ds/bits.h" #include "../mem/allocconfig.h" -#include "../mem/metaslab.h" #include "../pal/pal.h" #include "buddy.h" #include "range_helpers.h" diff --git a/src/backend/metatypes.h b/src/backend/metatypes.h new file mode 100644 index 0000000..1f089e0 --- /dev/null +++ b/src/backend/metatypes.h @@ -0,0 +1,196 @@ +#pragma once +#include "../ds/concept.h" +#include "../mem/allocconfig.h" /* TODO: CACHELINE_SIZE */ +#include "../pal/pal_concept.h" + +namespace snmalloc +{ + /** + * A guaranteed type-stable sub-structure of all metadata referenced by the + * Pagemap. Use-specific structures (Metaslab, ChunkRecord) are expected to + * have this at offset zero so that, even in the face of concurrent mutation + * and reuse of the memory backing that metadata, the types of these fields + * remain fixed. + * + * This class's data is fully private but is friends with the relevant backend + * types and, thus, is "opaque" to the frontend. + */ + class MetaCommon + { + friend class ChunkAllocator; + + template + friend class BackendAllocator; + + capptr::Chunk chunk; + + public: + /** + * Expose the address of, though not the authority to, our corresponding + * chunk. + */ + [[nodiscard]] SNMALLOC_FAST_PATH address_t chunk_address() + { + return address_cast(this->chunk); + } + + /** + * Zero (possibly by unmapping) the memory backing this chunk. We must rely + * on the caller to tell us its size, which is a little unfortunate. + */ + template + SNMALLOC_FAST_PATH void zero_chunk(size_t chunk_size) + { + PAL::zero(this->chunk.unsafe_ptr(), chunk_size); + } + }; + + static const size_t PAGEMAP_METADATA_STRUCT_SIZE = +#ifdef __CHERI_PURE_CAPABILITY__ + 2 * CACHELINE_SIZE +#else + CACHELINE_SIZE +#endif + ; + + // clang-format off + /* This triggers an ICE (C1001) in MSVC, so disable it there */ +#if defined(__cpp_concepts) && !defined(_MSC_VER) + + template + concept ConceptMetadataStruct = + // Metadata structures must be standard layout (for offsetof()), + std::is_standard_layout_v && + // must be of a sufficiently small size, + sizeof(Meta) <= PAGEMAP_METADATA_STRUCT_SIZE && + // and must be pointer-interconvertable with MetaCommon. + ( + ConceptSame || + requires(Meta m) { + // Otherwise, meta must have MetaCommon field named meta_common ... + { &m.meta_common } -> ConceptSame; + // at offset zero. + (offsetof(Meta, meta_common) == 0); + } + ); + + static_assert(ConceptMetadataStruct); +# define USE_METADATA_CONCEPT +#endif + // clang-format on + + struct RemoteAllocator; + class Metaslab; + class sizeclass_t; + + /** + * Entry stored in the pagemap. + */ + class MetaEntry + { + template + friend class BuddyChunkRep; + + /** + * The pointer to the metaslab, the bottom bit is used to indicate if this + * is the first chunk in a PAL allocation, that cannot be combined with + * the preceeding chunk. + */ + uintptr_t meta{0}; + + /** + * Bit used to indicate this should not be considered part of the previous + * PAL allocation. + * + * Some platforms cannot treat different PalAllocs as a single allocation. + * This is true on CHERI as the combined permission might not be + * representable. It is also true on Windows as you cannot Commit across + * multiple continuous VirtualAllocs. + */ + static constexpr address_t BOUNDARY_BIT = 1; + + /** + * A bit-packed pointer to the owning allocator (if any), and the sizeclass + * of this chunk. The sizeclass here is itself a union between two cases: + * + * * log_2(size), at least MIN_CHUNK_BITS, for large allocations. + * + * * a value in [0, NUM_SMALL_SIZECLASSES] for small allocations. These + * may be directly passed to the sizeclass (not slab_sizeclass) functions + * of sizeclasstable.h + * + */ + uintptr_t remote_and_sizeclass{0}; + + public: + constexpr MetaEntry() = default; + + /** + * Constructor, provides the remote and sizeclass embedded in a single + * pointer-sized word. This format is not guaranteed to be stable and so + * the second argument of this must always be the return value from + * `get_remote_and_sizeclass`. + */ + SNMALLOC_FAST_PATH + MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) + : meta(unsafe_to_uintptr(meta)), + remote_and_sizeclass(remote_and_sizeclass) + {} + + /* See mem/metaslab.h */ + SNMALLOC_FAST_PATH + MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass); + + /** + * Return the Metaslab metadata associated with this chunk, guarded by an + * assert that this chunk is being used as a slab (i.e., has an associated + * owning allocator). + */ + [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const + { + SNMALLOC_ASSERT(get_remote() != nullptr); + return unsafe_from_uintptr(meta & ~BOUNDARY_BIT); + } + + /** + * Return the remote and sizeclass in an implementation-defined encoding. + * This is not guaranteed to be stable across snmalloc releases and so the + * only safe use for this is to pass it to the two-argument constructor of + * this class. + */ + [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const + { + return remote_and_sizeclass; + } + + /* See mem/metaslab.h */ + [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const; + [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const; + + MetaEntry(const MetaEntry&) = delete; + + MetaEntry& operator=(const MetaEntry& other) + { + // Don't overwrite the boundary bit with the other's + meta = (other.meta & ~BOUNDARY_BIT) | address_cast(meta & BOUNDARY_BIT); + remote_and_sizeclass = other.remote_and_sizeclass; + return *this; + } + + void set_boundary() + { + meta |= BOUNDARY_BIT; + } + + [[nodiscard]] bool is_boundary() const + { + return meta & BOUNDARY_BIT; + } + + bool clear_boundary_bit() + { + return meta &= ~BOUNDARY_BIT; + } + }; + +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 48d3263..3b8e36a 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -332,7 +332,7 @@ namespace snmalloc SNMALLOC_ASSERT( address_cast(start_of_slab) == - address_cast(chunk_record->meta_common.chunk)); + chunk_record->meta_common.chunk_address()); #if defined(__CHERI_PURE_CAPABILITY__) && !defined(SNMALLOC_CHECK_CLIENT) // Zero the whole slab. For CHERI we at least need to clear the freelist @@ -340,9 +340,9 @@ namespace snmalloc // the freelist order as for SNMALLOC_CHECK_CLIENT. Zeroing the whole slab // may be more friendly to hw because it does not involve pointer chasing // and is amenable to prefetching. - SharedStateHandle::Pal::zero( - chunk_record->meta_common.chunk.unsafe_ptr(), - snmalloc::sizeclass_to_slab_size(sizeclass)); + chunk_record->meta_common + .template zero_chunk( + snmalloc::sizeclass_to_slab_size(sizeclass)); #endif #ifdef SNMALLOC_TRACING diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index e386f16..fdf2577 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -1,5 +1,6 @@ #pragma once +#include "../backend/metatypes.h" #include "../ds/helpers.h" #include "../ds/seqset.h" #include "../mem/remoteallocator.h" @@ -8,18 +9,6 @@ namespace snmalloc { - /** - * A guaranteed type-stable sub-structure of all metadata referenced by the - * Pagemap. Use-specific structures (Metaslab, ChunkRecord) are expected to - * have this at offset zero so that, even in the face of concurrent mutation - * and reuse of the memory backing that metadata, the types of these fields - * remain fixed. - */ - struct MetaCommon - { - capptr::Chunk chunk; - }; - // The Metaslab represent the status of a single slab. class alignas(CACHELINE_SIZE) Metaslab { @@ -216,140 +205,12 @@ namespace snmalloc } }; - static_assert(std::is_standard_layout_v); +#if defined(USE_METADATA_CONCEPT) + static_assert(ConceptMetadataStruct); +#endif static_assert( - offsetof(Metaslab, meta_common) == 0, - "ChunkRecord and Metaslab must share a common prefix"); - - /** - * Entry stored in the pagemap. - */ - class MetaEntry - { - template - friend class BuddyChunkRep; - - /** - * The pointer to the metaslab, the bottom bit is used to indicate if this - * is the first chunk in a PAL allocation, that cannot be combined with - * the preceeding chunk. - */ - uintptr_t meta{0}; - - /** - * Bit used to indicate this should not be considered part of the previous - * PAL allocation. - * - * Some platforms cannot treat different PalAllocs as a single allocation. - * This is true on CHERI as the combined permission might not be - * representable. It is also true on Windows as you cannot Commit across - * multiple continuous VirtualAllocs. - */ - static constexpr address_t BOUNDARY_BIT = 1; - - /** - * A bit-packed pointer to the owning allocator (if any), and the sizeclass - * of this chunk. The sizeclass here is itself a union between two cases: - * - * * log_2(size), at least MIN_CHUNK_BITS, for large allocations. - * - * * a value in [0, NUM_SMALL_SIZECLASSES] for small allocations. These - * may be directly passed to the sizeclass (not slab_sizeclass) functions of - * sizeclasstable.h - * - */ - uintptr_t remote_and_sizeclass{0}; - - public: - constexpr MetaEntry() = default; - - /** - * Constructor, provides the remote and sizeclass embedded in a single - * pointer-sized word. This format is not guaranteed to be stable and so - * the second argument of this must always be the return value from - * `get_remote_and_sizeclass`. - */ - SNMALLOC_FAST_PATH - MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) - : meta(unsafe_to_uintptr(meta)), - remote_and_sizeclass(remote_and_sizeclass) - {} - - SNMALLOC_FAST_PATH - MetaEntry( - Metaslab* meta, - RemoteAllocator* remote, - sizeclass_t sizeclass = sizeclass_t()) - : meta(unsafe_to_uintptr(meta)) - { - /* remote might be nullptr; cast to uintptr_t before offsetting */ - remote_and_sizeclass = pointer_offset( - unsafe_to_uintptr(remote), sizeclass.raw()); - } - - /** - * Return the Metaslab metadata associated with this chunk, guarded by an - * assert that this chunk is being used as a slab (i.e., has an associated - * owning allocator). - */ - [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const - { - SNMALLOC_ASSERT(get_remote() != nullptr); - return unsafe_from_uintptr(meta & ~BOUNDARY_BIT); - } - - /** - * Return the remote and sizeclass in an implementation-defined encoding. - * This is not guaranteed to be stable across snmalloc releases and so the - * only safe use for this is to pass it to the two-argument constructor of - * this class. - */ - [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const - { - return remote_and_sizeclass; - } - - [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const - { - return unsafe_from_uintptr( - pointer_align_down( - remote_and_sizeclass)); - } - - [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const - { - // TODO: perhaps remove static_cast with resolution of - // https://github.com/CTSRD-CHERI/llvm-project/issues/588 - return sizeclass_t::from_raw( - static_cast(remote_and_sizeclass) & - (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); - } - - MetaEntry(const MetaEntry&) = delete; - - MetaEntry& operator=(const MetaEntry& other) - { - // Don't overwrite the boundary bit with the other's - meta = (other.meta & ~BOUNDARY_BIT) | address_cast(meta & BOUNDARY_BIT); - remote_and_sizeclass = other.remote_and_sizeclass; - return *this; - } - - void set_boundary() - { - meta |= BOUNDARY_BIT; - } - - [[nodiscard]] bool is_boundary() const - { - return meta & BOUNDARY_BIT; - } - - bool clear_boundary_bit() - { - return meta &= ~BOUNDARY_BIT; - } - }; + sizeof(Metaslab) == PAGEMAP_METADATA_STRUCT_SIZE, + "Metaslab is expected to be the largest pagemap metadata record"); struct MetaslabCache { @@ -363,4 +224,41 @@ namespace snmalloc uint16_t unused = 0; uint16_t length = 0; }; + + /* + * MetaEntry methods that deal with RemoteAllocator* and sizeclass_t are here, + * so that the backend does not need to know the details and can, instead, + * just provide the storage space. + */ + + SNMALLOC_FAST_PATH_INLINE + MetaEntry::MetaEntry( + Metaslab* meta, + RemoteAllocator* remote, + sizeclass_t sizeclass = sizeclass_t()) + : meta(reinterpret_cast(meta)) + { + /* remote might be nullptr; cast to uintptr_t before offsetting */ + remote_and_sizeclass = + pointer_offset(reinterpret_cast(remote), sizeclass.raw()); + } + + [[nodiscard]] SNMALLOC_FAST_PATH_INLINE RemoteAllocator* + MetaEntry::get_remote() const + { + return reinterpret_cast( + pointer_align_down( + remote_and_sizeclass)); + } + + [[nodiscard]] SNMALLOC_FAST_PATH_INLINE sizeclass_t + MetaEntry::get_sizeclass() const + { + // TODO: perhaps remove static_cast with resolution of + // https://github.com/CTSRD-CHERI/llvm-project/issues/588 + return sizeclass_t::from_raw( + static_cast(remote_and_sizeclass) & + (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); + } + } // namespace snmalloc