#pragma once #include "../mem/allocconfig.h" #include "../mem/metadata.h" #include "../pal/pal.h" #include "commitrange.h" #include "commonconfig.h" #include "empty_range.h" #include "globalrange.h" #include "largebuddyrange.h" #include "pagemap.h" #include "pagemapregisterrange.h" #include "palrange.h" #include "range_helpers.h" #include "smallbuddyrange.h" #include "statsrange.h" #include "subrange.h" #if defined(SNMALLOC_CHECK_CLIENT) && !defined(OPEN_ENCLAVE) /** * Protect meta data blocks by allocating separate from chunks for * user allocations. This involves leaving gaps in address space. * This is less efficient, so should only be applied for the checked * build. * * On Open Enclave the address space is limited, so we disable this * feature. */ # define SNMALLOC_META_PROTECTED #endif namespace snmalloc { /** * This class implements the standard backend for handling allocations. * It abstracts page table management and address space management. */ template class BackendAllocator : public CommonConfig { public: using Pal = PAL; using SlabMetadata = FrontendSlabMetadata; class Pagemap { friend class BackendAllocator; public: /** * Export the type stored in the pagemap. * The following class could be replaced by: * * ``` * using Entry = FrontendMetaEntry; * ``` * * The full form here provides an example of how to extend the pagemap * entries. It also guarantees that the front end never directly * constructs meta entries, it only ever reads them or modifies them in * place. */ class Entry : public FrontendMetaEntry { /** * The private initialising constructor is usable only by this back end. */ friend class BackendAllocator; /** * The private default constructor is usable only by the pagemap. */ friend class FlatPagemap; /** * The only constructor that creates newly initialised meta entries. * This is callable only by the back end. The front end may copy, * query, and update these entries, but it may not create them * directly. This contract allows the back end to store any arbitrary * metadata in meta entries when they are first constructed. */ SNMALLOC_FAST_PATH Entry(SlabMetadata* meta, uintptr_t ras) : FrontendMetaEntry(meta, ras) {} /** * Default constructor. This must be callable from the pagemap. */ SNMALLOC_FAST_PATH Entry() = default; /** * Copy assignment is used only by the pagemap. */ Entry& operator=(const Entry& other) { FrontendMetaEntry::operator=(other); return *this; } }; private: SNMALLOC_REQUIRE_CONSTINIT static inline FlatPagemap concretePagemap; /** * Set the metadata associated with a chunk. */ SNMALLOC_FAST_PATH static void set_metaentry(address_t p, size_t size, const Entry& t) { for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) { concretePagemap.set(a, t); } } public: /** * Get the metadata associated with a chunk. * * Set template parameter to true if it not an error * to access a location that is not backed by a chunk. */ template SNMALLOC_FAST_PATH static const auto& get_metaentry(address_t p) { return concretePagemap.template get(p); } /** * Get the metadata associated with a chunk. * * Set template parameter to true if it not an error * to access a location that is not backed by a chunk. */ template SNMALLOC_FAST_PATH static auto& get_metaentry_mut(address_t p) { return concretePagemap.template get_mut(p); } static void register_range(address_t p, size_t sz) { concretePagemap.register_range(p, sz); } /** * Return the bounds of the memory this back-end manages as a pair of * addresses (start then end). This is available iff this is a * fixed-range Backend. */ template static SNMALLOC_FAST_PATH std::enable_if_t> get_bounds() { static_assert( fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); return concretePagemap.get_bounds(); } static bool is_initialised() { return concretePagemap.is_initialised(); } }; #if defined(_WIN32) || defined(__CHERI_PURE_CAPABILITY__) static constexpr bool CONSOLIDATE_PAL_ALLOCS = false; #else static constexpr bool CONSOLIDATE_PAL_ALLOCS = true; #endif #if defined(OPEN_ENCLAVE) // Single global buddy allocator is used on open enclave due to // the limited address space. using StatsR = StatsRange>>; using GlobalR = GlobalRange; using ObjectRange = GlobalR; using GlobalMetaRange = ObjectRange; #else // Set up source of memory using P = PalRange; using Base = std:: conditional_t>; // Global range of memory using StatsR = StatsRange>; using GlobalR = GlobalRange; # ifdef SNMALLOC_META_PROTECTED // Source for object allocations using ObjectRange = LargeBuddyRange, 21, 21, Pagemap>; // Set up protected range for metadata using SubR = CommitRange, DefaultPal>; using MetaRange = SmallBuddyRange>; using GlobalMetaRange = GlobalRange; # else // Source for object allocations and metadata // No separation between the two using ObjectRange = SmallBuddyRange< LargeBuddyRange, 21, 21, Pagemap>>; using GlobalMetaRange = GlobalRange; # endif #endif struct LocalState { typename ObjectRange::State object_range; #ifdef SNMALLOC_META_PROTECTED typename MetaRange::State meta_range; typename MetaRange::State& get_meta_range() { return meta_range; } #else typename ObjectRange::State& get_meta_range() { return object_range; } #endif }; public: template static std::enable_if_t init() { static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); Pagemap::concretePagemap.init(); } template static std::enable_if_t init(void* base, size_t length) { static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); auto [heap_base, heap_length] = Pagemap::concretePagemap.init(base, length); Pagemap::register_range(address_cast(heap_base), heap_length); // Push memory into the global range. range_to_pow_2_blocks( capptr::Chunk(heap_base), heap_length, [&](capptr::Chunk p, size_t sz, bool) { typename GlobalR::State g; g->dealloc_range(p, sz); }); } /** * Provide a block of meta-data with size and align. * * Backend allocator may use guard pages and separate area of * address space to protect this from corruption. * * The template argument is the type of the metadata being allocated. This * allows the backend to allocate different types of metadata in different * places or with different policies. The default implementation, here, * does not avail itself of this degree of freedom. */ template static capptr::Chunk alloc_meta_data(LocalState* local_state, size_t size) { capptr::Chunk p; if (local_state != nullptr) { p = local_state->get_meta_range()->alloc_range_with_leftover(size); } else { static_assert( GlobalMetaRange::ConcurrencySafe, "Global meta data range needs to be concurrency safe."); typename GlobalMetaRange::State global_state; p = global_state->alloc_range(bits::next_pow2(size)); } if (p == nullptr) errno = ENOMEM; return p; } /** * Returns a chunk of memory with alignment and size of `size`, and a * block containing metadata about the slab. * * It additionally set the meta-data for this chunk of memory to * be * (remote, sizeclass, slab_metadata) * where slab_metadata, is the second element of the pair return. */ static std::pair, SlabMetadata*> alloc_chunk(LocalState& local_state, size_t size, uintptr_t ras) { SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); auto meta_cap = local_state.get_meta_range()->alloc_range(sizeof(SlabMetadata)); auto meta = meta_cap.template as_reinterpret().unsafe_ptr(); if (meta == nullptr) { errno = ENOMEM; return {nullptr, nullptr}; } auto p = local_state.object_range->alloc_range(size); #ifdef SNMALLOC_TRACING message<1024>("Alloc chunk: {} ({})", p.unsafe_ptr(), size); #endif if (p == nullptr) { local_state.get_meta_range()->dealloc_range( meta_cap, sizeof(SlabMetadata)); errno = ENOMEM; #ifdef SNMALLOC_TRACING message<1024>("Out of memory"); #endif return {p, nullptr}; } typename Pagemap::Entry t(meta, ras); Pagemap::set_metaentry(address_cast(p), size, t); p = Aal::capptr_bound(p, size); return {p, meta}; } static void dealloc_chunk( LocalState& local_state, SlabMetadata& slab_metadata, capptr::Alloc alloc, size_t size) { /* * The backend takes possession of these chunks now, by disassociating * any existing remote allocator and metadata structure. If * interrogated, the sizeclass reported by the FrontendMetaEntry is 0, * which has size 0. */ typename Pagemap::Entry t(nullptr, 0); t.claim_for_backend(); SNMALLOC_ASSERT_MSG( Pagemap::get_metaentry(address_cast(alloc)).get_slab_metadata() == &slab_metadata, "Slab metadata {} passed for address {} does not match the meta entry " "{} that is used for that address", &slab_metadata, address_cast(alloc), Pagemap::get_metaentry(address_cast(alloc)).get_slab_metadata()); Pagemap::set_metaentry(address_cast(alloc), size, t); local_state.get_meta_range()->dealloc_range( capptr::Chunk(&slab_metadata), sizeof(SlabMetadata)); // On non-CHERI platforms, we don't need to re-derive to get a pointer to // the chunk. On CHERI platforms this will need to be stored in the // SlabMetadata or similar. capptr::Chunk chunk{alloc.unsafe_ptr()}; local_state.object_range->dealloc_range(chunk, size); } static size_t get_current_usage() { typename StatsR::State stats_state; return stats_state->get_current_usage(); } static size_t get_peak_usage() { typename StatsR::State stats_state; return stats_state->get_peak_usage(); } }; } // namespace snmalloc