From e8374479f41b030065e477623f42b8568da232dc Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Thu, 5 Aug 2021 15:08:12 +0100 Subject: [PATCH] Snmalloc2 API cleanups for sandbox use. (#359) This is the set of changes required for snmalloc2 to be usable by the process sandboxing code and incorporates some API changes that reduce the amount of code required to embed snmalloc. Highlights: - Merge the config and back-end classes. - Everything in config is now global (all methods are static) - The GlobalState class is gone (all global state is managed by global methods on the config class) - LocalState is now a member of the config class, all methods are instance methods. - Not every configuration needs to use the lazy initialisation hooks. They now need to be provided only if they are used. If the configuration does not provide an `ensure_init` method, it is not called. If it does not provide an `is_initialised` method then the global initialisation state is not checked. - There is now an `snmalloc::Options` class that default initialises itself to the default behaviour. Every configuration must provide a `constexpr` instance of this class. Each flag can be separately overridden and new flags can be added without breaking any existing API consumers. The config classes are moved into the backend directory. --- src/backend/address_space_core.h | 2 + src/backend/backend.h | 111 ++++---- src/backend/commonconfig.h | 118 +++++++++ src/{mem => backend}/fixedglobalconfig.h | 42 +-- src/{mem => backend}/globalconfig.h | 41 ++- src/backend/pagemap.h | 36 ++- src/mem/commonconfig.h | 41 --- src/mem/corealloc.h | 181 ++++++++++--- src/mem/globalalloc.h | 72 +++-- src/mem/localalloc.h | 247 +++++++++++++----- src/mem/localcache.h | 14 +- src/mem/metaslab.h | 21 ++ src/mem/pool.h | 26 +- src/mem/remotecache.h | 12 +- src/mem/slaballocator.h | 29 +- src/mem/threadalloc.h | 2 +- src/override/malloc-extensions.cc | 8 +- src/override/malloc.cc | 2 +- src/snmalloc.h | 10 +- src/snmalloc_core.h | 5 +- src/test/func/fixed_region/fixed_region.cc | 5 +- src/test/func/malloc/malloc.cc | 2 +- src/test/func/memory/memory.cc | 6 +- src/test/func/statistics/stats.cc | 10 +- .../thread_alloc_external.cc | 2 +- src/test/func/two_alloc_types/alloc1.cc | 5 +- src/test/perf/contention/contention.cc | 2 +- .../perf/external_pointer/externalpointer.cc | 2 +- src/test/perf/singlethread/singlethread.cc | 2 +- 29 files changed, 676 insertions(+), 380 deletions(-) create mode 100644 src/backend/commonconfig.h rename src/{mem => backend}/fixedglobalconfig.h (52%) rename src/{mem => backend}/globalconfig.h (75%) delete mode 100644 src/mem/commonconfig.h diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 549c317..d24b89f 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -1,6 +1,8 @@ #pragma once #include "../ds/address.h" #include "../ds/flaglock.h" +#include "../mem/allocconfig.h" +#include "../mem/metaslab.h" #include "../pal/pal.h" #include diff --git a/src/backend/backend.h b/src/backend/backend.h index f0311ae..b9b62b9 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -3,6 +3,7 @@ #include "../mem/metaslab.h" #include "../pal/pal.h" #include "address_space.h" +#include "commonconfig.h" #include "pagemap.h" namespace snmalloc @@ -15,7 +16,7 @@ namespace snmalloc SNMALLOC_CONCEPT(ConceptPAL) PAL, bool fixed_range, typename PageMapEntry = MetaEntry> - class BackendAllocator + class BackendAllocator : public CommonConfig { // Size of local address space requests. Currently aimed at 2MiB large // pages but should make this configurable (i.e. for OE, so we don't need as @@ -54,55 +55,31 @@ namespace snmalloc #endif }; - /** - * Global state for the backend allocator - * - * This contains the various global datastructures required to store - * meta-data for each chunk of memory, and to provide well aligned chunks - * of memory. - * - * This type is required by snmalloc to exist as part of the Backend. - */ - class GlobalState + SNMALLOC_REQUIRE_CONSTINIT + static inline AddressSpaceManager address_space; + + SNMALLOC_REQUIRE_CONSTINIT + static inline FlatPagemap + pagemap; + + public: + template + static std::enable_if_t init() { - friend BackendAllocator; + static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - protected: - AddressSpaceManager address_space; + pagemap.init(); + } - FlatPagemap pagemap; + template + static std::enable_if_t init(void* base, size_t length) + { + static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - public: - template - std::enable_if_t init() - { - static_assert( - fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - - pagemap.init(); - - if constexpr (fixed_range) - { - abort(); - } - } - - template - std::enable_if_t init(void* base, size_t length) - { - static_assert( - fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - - auto [heap_base, heap_length] = pagemap.init(base, length); - address_space.add_range( - CapPtr(heap_base), heap_length, pagemap); - - if constexpr (!fixed_range) - { - abort(); - } - } - }; + auto [heap_base, heap_length] = pagemap.init(base, length); + address_space.add_range( + CapPtr(heap_base), heap_length, pagemap); + } private: #ifdef SNMALLOC_CHECK_CLIENT @@ -138,8 +115,7 @@ namespace snmalloc * space managers. */ template - static CapPtr - reserve(GlobalState& h, LocalState* local_state, size_t size) + static CapPtr reserve(LocalState* local_state, size_t size) { #ifdef SNMALLOC_CHECK_CLIENT constexpr auto MAX_CACHED_SIZE = @@ -148,7 +124,7 @@ namespace snmalloc constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK; #endif - auto& global = h.address_space; + auto& global = address_space; CapPtr p; if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) @@ -160,14 +136,14 @@ namespace snmalloc auto& local = local_state->local_address_space; #endif - p = local.template reserve_with_left_over(size, h.pagemap); + p = local.template reserve_with_left_over(size, pagemap); if (p != nullptr) { return p; } auto refill_size = LOCAL_CACHE_BLOCK; - auto refill = global.template reserve(refill_size, h.pagemap); + auto refill = global.template reserve(refill_size, pagemap); if (refill == nullptr) return nullptr; @@ -179,10 +155,10 @@ namespace snmalloc } #endif PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range(refill, refill_size, h.pagemap); + local.template add_range(refill, refill_size, pagemap); // This should succeed - return local.template reserve_with_left_over(size, h.pagemap); + return local.template reserve_with_left_over(size, pagemap); } #ifdef SNMALLOC_CHECK_CLIENT @@ -193,7 +169,7 @@ namespace snmalloc size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); size_t size_request = rsize * 64; - p = global.template reserve(size_request, h.pagemap); + p = global.template reserve(size_request, pagemap); if (p == nullptr) return nullptr; @@ -209,7 +185,7 @@ namespace snmalloc SNMALLOC_ASSERT(!is_meta); #endif - p = global.template reserve_with_left_over(size, h.pagemap); + p = global.template reserve_with_left_over(size, pagemap); return p; } @@ -219,11 +195,16 @@ namespace snmalloc * * Backend allocator may use guard pages and separate area of * address space to protect this from corruption. + * + * The template argument is the type of the metadata being allocated. This + * allows the backend to allocate different types of metadata in different + * places or with different policies. */ + template static CapPtr - alloc_meta_data(GlobalState& h, LocalState* local_state, size_t size) + alloc_meta_data(LocalState* local_state, size_t size) { - return reserve(h, local_state, size); + return reserve(local_state, size); } /** @@ -236,7 +217,6 @@ namespace snmalloc * where metaslab, is the second element of the pair return. */ static std::pair, Metaslab*> alloc_chunk( - GlobalState& h, LocalState* local_state, size_t size, RemoteAllocator* remote, @@ -246,12 +226,12 @@ namespace snmalloc SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); auto meta = reinterpret_cast( - reserve(h, local_state, sizeof(Metaslab)).unsafe_ptr()); + reserve(local_state, sizeof(Metaslab)).unsafe_ptr()); if (meta == nullptr) return {nullptr, nullptr}; - CapPtr p = reserve(h, local_state, size); + CapPtr p = reserve(local_state, size); #ifdef SNMALLOC_TRACING std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" @@ -274,7 +254,7 @@ namespace snmalloc a < address_cast(pointer_offset(p, size)); a += MIN_CHUNK_SIZE) { - h.pagemap.set(a, t); + pagemap.set(a, t); } return {p, meta}; } @@ -286,20 +266,19 @@ namespace snmalloc * to access a location that is not backed by a chunk. */ template - static const MetaEntry& get_meta_data(GlobalState& h, address_t p) + static const MetaEntry& get_meta_data(address_t p) { - return h.pagemap.template get(p); + return pagemap.template get(p); } /** * Set the metadata associated with a chunk. */ - static void - set_meta_data(GlobalState& h, address_t p, size_t size, MetaEntry t) + static void set_meta_data(address_t p, size_t size, MetaEntry t) { for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) { - h.pagemap.set(a, t); + pagemap.set(a, t); } } }; diff --git a/src/backend/commonconfig.h b/src/backend/commonconfig.h new file mode 100644 index 0000000..689a139 --- /dev/null +++ b/src/backend/commonconfig.h @@ -0,0 +1,118 @@ +#pragma once + +#include "../ds/defines.h" +#include "../mem/remotecache.h" + +namespace snmalloc +{ + // Forward reference to thread local cleanup. + void register_clean_up(); + + /** + * Options for a specific snmalloc configuration. Every globals object must + * have one `constexpr` instance of this class called `Options`. This should + * be constructed to explicitly override any of the defaults. A + * configuration that does not need to override anything would simply declare + * this as a field of the global object: + * + * ```c++ + * constexpr static snmalloc::Flags Options{}; + * ``` + * + * A global configuration that wished to use out-of-line message queues but + * accept the defaults for everything else would instead do this: + * + * ```c++ + * constexpr static snmalloc::Flags Options{.IsQueueInline = false}; + * ``` + * + * To maintain backwards source compatibility in future versions, any new + * option added here should have its default set to be whatever snmalloc was + * doing before the new option was added. + */ + struct Flags + { + /** + * Should allocators have inline message queues? If this is true then + * the `CoreAllocator` is responsible for allocating the + * `RemoteAllocator` that contains its message queue. If this is false + * then the `RemoteAllocator` must be separately allocated and provided + * to the `CoreAllocator` before it is used. + * + * Setting this to `false` currently requires also setting + * `LocalAllocSupportsLazyInit` to false so that the `CoreAllocator` can + * be provided to the `LocalAllocator` fully initialised but in the + * future it may be possible to allocate the `RemoteAllocator` via + * `alloc_meta_data` or a similar API in the back end. + */ + bool IsQueueInline = true; + + /** + * Does the `CoreAllocator` own a `Backend::LocalState` object? If this is + * true then the `CoreAllocator` is responsible for allocating and + * deallocating a local state object, otherwise the surrounding code is + * responsible for creating it. + * + * Use cases that set this to false will probably also need to set + * `LocalAllocSupportsLazyInit` to false so that they can provide the local + * state explicitly during allocator creation. + */ + bool CoreAllocOwnsLocalState = true; + + /** + * Are `CoreAllocator` allocated by the pool allocator? If not then the + * code embedding this snmalloc configuration is responsible for allocating + * `CoreAllocator` instances. + * + * Users setting this flag must also set `LocalAllocSupportsLazyInit` to + * false currently because there is no alternative mechanism for allocating + * core allocators. This may change in future versions. + */ + bool CoreAllocIsPoolAllocated = true; + + /** + * Do `LocalAllocator` instances in this configuration support lazy + * initialisation? If so, then the first exit from a fast path will + * trigger allocation of a `CoreAllocator` and associated state. If not + * then the code embedding this configuration of snmalloc is responsible + * for allocating core allocators. + */ + bool LocalAllocSupportsLazyInit = true; + }; + + /** + * Class containing definitions that are likely to be used by all except for + * the most unusual back-end implementations. This can be subclassed as a + * convenience for back-end implementers, but is not required. + */ + class CommonConfig + { + public: + /** + * Special remote that should never be used as a real remote. + * This is used to initialise allocators that should always hit the + * remote path for deallocation. Hence moving a branch off the critical + * path. + */ + SNMALLOC_REQUIRE_CONSTINIT + inline static RemoteAllocator unused_remote; + + /** + * Special remote that is used in meta-data for large allocations. + * + * nullptr is considered a large allocations for this purpose to move + * of the critical path. + * + * Bottom bits of the remote pointer are used for a sizeclass, we need + * size bits to represent the non-large sizeclasses, we can then get + * the large sizeclass by having the fake large_remote considerably + * more aligned. + */ + SNMALLOC_REQUIRE_CONSTINIT + inline static constexpr RemoteAllocator* fake_large_remote{nullptr}; + + static_assert( + &unused_remote != fake_large_remote, + "Compilation should ensure these are different"); + }; +} // namespace snmalloc diff --git a/src/mem/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h similarity index 52% rename from src/mem/fixedglobalconfig.h rename to src/backend/fixedglobalconfig.h index 01b268f..2e4473e 100644 --- a/src/mem/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -12,50 +12,27 @@ namespace snmalloc * A single fixed address range allocator configuration */ template - class FixedGlobals : public CommonConfig + class FixedGlobals final : public BackendAllocator { - public: - using Backend = BackendAllocator; - private: - inline static typename Backend::GlobalState backend_state; - + using Backend = BackendAllocator; inline static ChunkAllocatorState slab_allocator_state; inline static PoolState> alloc_pool; public: - static typename Backend::GlobalState& get_backend_state() - { - return backend_state; - } - - ChunkAllocatorState& get_slab_allocator_state() + static ChunkAllocatorState& + get_slab_allocator_state(typename Backend::LocalState*) { return slab_allocator_state; } - PoolState>& pool() + static PoolState>& pool() { return alloc_pool; } - static constexpr bool IsQueueInline = true; - - // Performs initialisation for this configuration - // of allocators. Will be called at most once - // before any other datastructures are accessed. - void ensure_init() noexcept - { -#ifdef SNMALLOC_TRACING - std::cout << "Run init_impl" << std::endl; -#endif - } - - static bool is_initialised() - { - return true; - } + static constexpr Flags Options{}; // This needs to be a forward reference as the // thread local state will need to know about this. @@ -68,12 +45,7 @@ namespace snmalloc static void init(void* base, size_t length) { - get_backend_state().init(base, length); - } - - constexpr static FixedGlobals get_handle() - { - return {}; + Backend::init(base, length); } }; } diff --git a/src/mem/globalconfig.h b/src/backend/globalconfig.h similarity index 75% rename from src/mem/globalconfig.h rename to src/backend/globalconfig.h index cc5dc76..f084703 100644 --- a/src/mem/globalconfig.h +++ b/src/backend/globalconfig.h @@ -23,15 +23,15 @@ namespace snmalloc } #endif - class Globals : public CommonConfig + /** + * The default configuration for a global snmalloc. This allocates memory + * from the operating system and expects to manage memory anywhere in the + * address space. + */ + class Globals final : public BackendAllocator { - public: - using Backend = BackendAllocator; - private: - SNMALLOC_REQUIRE_CONSTINIT - inline static Backend::GlobalState backend_state; - + using Backend = BackendAllocator; SNMALLOC_REQUIRE_CONSTINIT inline static ChunkAllocatorState slab_allocator_state; @@ -45,27 +45,23 @@ namespace snmalloc inline static std::atomic_flag initialisation_lock{}; public: - Backend::GlobalState& get_backend_state() - { - return backend_state; - } - - ChunkAllocatorState& get_slab_allocator_state() + static ChunkAllocatorState& + get_slab_allocator_state(Backend::LocalState* = nullptr) { return slab_allocator_state; } - PoolState>& pool() + static PoolState>& pool() { return alloc_pool; } - static constexpr bool IsQueueInline = true; + static constexpr Flags Options{}; // Performs initialisation for this configuration // of allocators. Needs to be idempotent, // and concurrency safe. - void ensure_init() + static void ensure_init() { FlagLock lock{initialisation_lock}; #ifdef SNMALLOC_TRACING @@ -81,7 +77,7 @@ namespace snmalloc key_global = FreeListKey(entropy.get_free_list_key()); // Need to initialise pagemap. - backend_state.init(); + Backend::init(); #ifdef USE_SNMALLOC_STATS atexit(snmalloc::print_stats); @@ -90,7 +86,7 @@ namespace snmalloc initialised = true; } - bool is_initialised() + static bool is_initialised() { return initialised; } @@ -99,16 +95,9 @@ namespace snmalloc // thread local state will need to know about this. // This may allocate, so should only be called once // a thread local allocator is available. - void register_clean_up() + static void register_clean_up() { snmalloc::register_clean_up(); } - - // This is an empty structure as all the state is global - // for this allocator configuration. - static constexpr Globals get_handle() - { - return {}; - } }; } // namespace snmalloc diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 4f4ab6e..d640ae1 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -66,6 +66,36 @@ namespace snmalloc constexpr FlatPagemap() = default; + /** + * For pagemaps that cover an entire fixed address space, return the size + * that they must be. This allows the caller to allocate the correct + * amount of memory to be passed to `init`. This is not available for + * fixed-range pagemaps, whose size depends on dynamic configuration. + */ + template + static constexpr std::enable_if_t required_size() + { + static_assert( + has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); + constexpr size_t COVERED_BITS = bits::ADDRESS_BITS - GRANULARITY_BITS; + constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); + return ENTRIES * sizeof(T); + } + + /** + * Initialise with pre-allocated memory. + * + * This is currently disabled for bounded pagemaps but may be reenabled if + * `required_size` is enabled for the has-bounds case. + */ + template + std::enable_if_t init(T* address) + { + static_assert( + has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); + body = address; + } + /** * Initialise the pagemap with bounds. * @@ -117,11 +147,7 @@ namespace snmalloc { static_assert( has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); - static constexpr size_t COVERED_BITS = - bits::ADDRESS_BITS - GRANULARITY_BITS; - static constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); - - static constexpr size_t REQUIRED_SIZE = ENTRIES * sizeof(T); + static constexpr size_t REQUIRED_SIZE = required_size(); #ifdef SNMALLOC_CHECK_CLIENT // Allocate a power of two extra to allow the placement of the diff --git a/src/mem/commonconfig.h b/src/mem/commonconfig.h deleted file mode 100644 index 9c24371..0000000 --- a/src/mem/commonconfig.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include "../ds/defines.h" -#include "remotecache.h" - -namespace snmalloc -{ - // Forward reference to thread local cleanup. - void register_clean_up(); - - class CommonConfig - { - public: - /** - * Special remote that should never be used as a real remote. - * This is used to initialise allocators that should always hit the - * remote path for deallocation. Hence moving a branch off the critical - * path. - */ - SNMALLOC_REQUIRE_CONSTINIT - inline static RemoteAllocator unused_remote; - - /** - * Special remote that is used in meta-data for large allocations. - * - * nullptr is considered a large allocations for this purpose to move - * of the critical path. - * - * Bottom bits of the remote pointer are used for a sizeclass, we need - * size bits to represent the non-large sizeclasses, we can then get - * the large sizeclass by having the fake large_remote considerably - * more aligned. - */ - SNMALLOC_REQUIRE_CONSTINIT - inline static constexpr RemoteAllocator* fake_large_remote{nullptr}; - - static_assert( - &unused_remote != fake_large_remote, - "Compilation should ensure these are different"); - }; -} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 96490a7..0097b12 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -11,10 +11,45 @@ namespace snmalloc { + /** + * Empty class used as the superclass for `CoreAllocator` when it does not + * opt into pool allocation. This class exists because `std::conditional` + * (or other equivalent features in C++) can choose between options for + * superclasses but they cannot choose whether a class has a superclass. + * Setting the superclass to an empty class is equivalent to no superclass. + */ + class NotPoolAllocated + {}; + + /** + * The core, stateful, part of a memory allocator. Each `LocalAllocator` + * owns one `CoreAllocator` once it is initialised. + * + * The template parameter provides all of the global configuration for this + * instantiation of snmalloc. This includes three options that apply to this + * class: + * + * - `CoreAllocIsPoolAllocated` defines whether this `CoreAlloc` + * configuration should support pool allocation. This defaults to true but + * a configuration that allocates allocators eagerly may opt out. + * - `CoreAllocOwnsLocalState` defines whether the `CoreAllocator` owns the + * associated `LocalState` object. If this is true (the default) then + * `CoreAllocator` embeds the LocalState object. If this is set to false + * then a `LocalState` object must be provided to the constructor. This + * allows external code to provide explicit configuration of the address + * range managed by this object. + * - `IsQueueInline` (defaults to true) defines whether the message queue + * (`RemoteAllocator`) for this class is inline or provided externally. If + * provided externally, then it must be set explicitly with + * `init_message_queue`. + */ template - class CoreAllocator : public Pooled> + class CoreAllocator : public std::conditional_t< + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + Pooled>, + NotPoolAllocated> { - template + template friend class LocalAllocator; /** @@ -33,16 +68,27 @@ namespace snmalloc * allocator */ std::conditional_t< - SharedStateHandle::IsQueueInline, + SharedStateHandle::Options.IsQueueInline, RemoteAllocator, RemoteAllocator*> remote_alloc; /** - * A local area of address space managed by this allocator. - * Used to reduce calls on the global address space. + * The type used local state. This is defined by the back end. */ - typename SharedStateHandle::Backend::LocalState backend_state; + using LocalState = typename SharedStateHandle::LocalState; + + /** + * A local area of address space managed by this allocator. + * Used to reduce calls on the global address space. This is inline if the + * core allocator owns the local state or indirect if it is owned + * externally. + */ + std::conditional_t< + SharedStateHandle::Options.CoreAllocOwnsLocalState, + LocalState, + LocalState*> + backend_state; /** * This is the thread local structure associated to this @@ -50,12 +96,6 @@ namespace snmalloc */ LocalCache* attached_cache; - /** - * This contains the way to access all the global state and - * configuration for the system setup. - */ - SharedStateHandle handle; - /** * The message queue needs to be accessible from other threads * @@ -64,7 +104,7 @@ namespace snmalloc */ auto* public_state() { - if constexpr (SharedStateHandle::IsQueueInline) + if constexpr (SharedStateHandle::Options.IsQueueInline) { return &remote_alloc; } @@ -260,8 +300,10 @@ namespace snmalloc // TODO delay the clear to the next user of the slab, or teardown so // don't touch the cache lines at this point in check_client. auto chunk_record = clear_slab(meta, sizeclass); - ChunkAllocator::dealloc( - handle, chunk_record, sizeclass_to_slab_sizeclass(sizeclass)); + ChunkAllocator::dealloc( + get_backend_local_state(), + chunk_record, + sizeclass_to_slab_sizeclass(sizeclass)); } else { @@ -335,8 +377,8 @@ namespace snmalloc for (size_t i = 0; i < REMOTE_BATCH; i++) { auto p = message_queue().peek(); - auto& entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), snmalloc::address_cast(p)); + auto& entry = + SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); auto r = message_queue().dequeue(key_global); @@ -388,9 +430,11 @@ namespace snmalloc } } - public: - CoreAllocator(LocalCache* cache, SharedStateHandle handle) - : attached_cache(cache), handle(handle) + /** + * Initialiser, shared code between the constructors for different + * configurations. + */ + void init() { #ifdef SNMALLOC_TRACING std::cout << "Making an allocator." << std::endl; @@ -398,13 +442,16 @@ namespace snmalloc // Entropy must be first, so that all data-structures can use the key // it generates. // This must occur before any freelists are constructed. - entropy.init(); + entropy.init(); // Ignoring stats for now. // stats().start(); - init_message_queue(); - message_queue().invariant(); + if constexpr (SharedStateHandle::Options.IsQueueInline) + { + init_message_queue(); + message_queue().invariant(); + } #ifndef NDEBUG for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) @@ -423,6 +470,44 @@ namespace snmalloc #endif } + public: + /** + * Constructor for the case that the core allocator owns the local state. + * SFINAE disabled if the allocator does not own the local state. + */ + template< + typename Config = SharedStateHandle, + typename = std::enable_if_t> + CoreAllocator(LocalCache* cache) : attached_cache(cache) + { + init(); + } + + /** + * Constructor for the case that the core allocator does not owns the local + * state. SFINAE disabled if the allocator does own the local state. + */ + template< + typename Config = SharedStateHandle, + typename = std::enable_if_t> + CoreAllocator(LocalCache* cache, LocalState* backend = nullptr) + : backend_state(backend), attached_cache(cache) + { + init(); + } + + /** + * If the message queue is not inline, provide it. This will then + * configure the message queue for use. + */ + template + std::enable_if_t init_message_queue(RemoteAllocator* q) + { + remote_alloc = q; + init_message_queue(); + message_queue().invariant(); + } + /** * Post deallocations onto other threads. * @@ -432,9 +517,9 @@ namespace snmalloc SNMALLOC_FAST_PATH bool post() { // stats().remote_post(); // TODO queue not in line! - bool sent_something = - attached_cache->remote_dealloc_cache.post( - handle, public_state()->trunc_id(), key_global); + bool sent_something = attached_cache->remote_dealloc_cache + .post( + public_state()->trunc_id(), key_global); return sent_something; } @@ -454,8 +539,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(void* p) { - auto entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), snmalloc::address_cast(p)); + auto entry = SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); if (likely(dealloc_local_object_fast(entry, p, entropy))) return; @@ -506,6 +590,24 @@ namespace snmalloc return small_alloc_slow(sizeclass, fast_free_list, rsize); } + /** + * Accessor for the local state. This hides whether the local state is + * stored inline or provided externally from the rest of the code. + */ + SNMALLOC_FAST_PATH + LocalState& get_backend_local_state() + { + if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + { + return backend_state; + } + else + { + SNMALLOC_ASSERT(backend_state); + return *backend_state; + } + } + template SNMALLOC_SLOW_PATH void* small_alloc_slow( sizeclass_t sizeclass, FreeListIter& fast_free_list, size_t rsize) @@ -519,13 +621,13 @@ namespace snmalloc std::cout << "slab size " << slab_size << std::endl; #endif - auto [slab, meta] = snmalloc::ChunkAllocator::alloc_chunk( - handle, - backend_state, - sizeclass, - slab_sizeclass, - slab_size, - public_state()); + auto [slab, meta] = + snmalloc::ChunkAllocator::alloc_chunk( + get_backend_local_state(), + sizeclass, + slab_sizeclass, + slab_size, + public_state()); if (slab == nullptr) { @@ -563,8 +665,8 @@ namespace snmalloc { bool need_post = true; // Always going to post, so ignore. auto n = p->atomic_read_next(key_global); - auto& entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), snmalloc::address_cast(p)); + auto& entry = + SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); p = n; } @@ -577,8 +679,9 @@ namespace snmalloc handle_message_queue([]() {}); } - auto posted = attached_cache->flush( - [&](auto p) { dealloc_local_object(p); }, handle); + auto posted = + attached_cache->flush( + [&](auto p) { dealloc_local_object(p); }); // We may now have unused slabs, return to the global allocator. for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index 0026598..cc4ecd5 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -6,9 +6,13 @@ namespace snmalloc { template - inline static void aggregate_stats(SharedStateHandle handle, Stats& stats) + inline static void aggregate_stats(Stats& stats) { - auto* alloc = Pool>::iterate(handle); + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global statistics are available only for pool-allocated configurations"); + auto* alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { @@ -16,45 +20,51 @@ namespace snmalloc if (a != nullptr) stats.add(*a); stats.add(alloc->stats()); - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } #ifdef USE_SNMALLOC_STATS template - inline static void print_all_stats( - SharedStateHandle handle, std::ostream& o, uint64_t dumpid = 0) + inline static void print_all_stats(std::ostream& o, uint64_t dumpid = 0) { - auto alloc = Pool>::iterate(handle); + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global statistics are available only for pool-allocated configurations"); + auto alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { auto stats = alloc->stats(); if (stats != nullptr) - stats->template print(o, dumpid, alloc->id()); - alloc = Pool>::iterate(handle, alloc); + stats->template print(o, dumpid, alloc->id()); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } #else template - inline static void - print_all_stats(SharedStateHandle handle, void*& o, uint64_t dumpid = 0) + inline static void print_all_stats(void*& o, uint64_t dumpid = 0) { UNUSED(o); UNUSED(dumpid); - UNUSED(handle); } #endif template - inline static void cleanup_unused(SharedStateHandle handle) + inline static void cleanup_unused() { #ifndef SNMALLOC_PASS_THROUGH + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global cleanup is available only for pool-allocated configurations"); // Call this periodically to free and coalesce memory allocated by // allocators that are not currently in use by any thread. // One atomic operation to extract the stack, another to restore it. // Handling the message queue for each stack is non-atomic. - auto* first = Pool>::extract(handle); + auto* first = Pool>::extract(); auto* alloc = first; decltype(alloc) last; @@ -64,10 +74,10 @@ namespace snmalloc { alloc->flush(); last = alloc; - alloc = Pool>::extract(handle, alloc); + alloc = Pool>::extract(alloc); } - Pool>::restore(handle, first, last); + Pool>::restore(first, last); } #endif } @@ -78,13 +88,16 @@ namespace snmalloc raise an error all the allocators are not empty. */ template - inline static void - debug_check_empty(SharedStateHandle handle, bool* result = nullptr) + inline static void debug_check_empty(bool* result = nullptr) { #ifndef SNMALLOC_PASS_THROUGH + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global status is available only for pool-allocated configurations"); // This is a debugging function. It checks that all memory from all // allocators has been freed. - auto* alloc = Pool>::iterate(handle); + auto* alloc = Pool>::template iterate< + SharedStateHandle>(); # ifdef SNMALLOC_TRACING std::cout << "debug check empty: first " << alloc << std::endl; @@ -98,7 +111,8 @@ namespace snmalloc std::cout << "debug_check_empty: Check all allocators!" << std::endl; # endif done = true; - alloc = Pool>::iterate(handle); + alloc = Pool>::template iterate< + SharedStateHandle>(); okay = true; while (alloc != nullptr) @@ -120,7 +134,8 @@ namespace snmalloc # ifdef SNMALLOC_TRACING std::cout << "debug check empty: okay = " << okay << std::endl; # endif - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } @@ -133,11 +148,13 @@ namespace snmalloc // Redo check so abort is on allocator with allocation left. if (!okay) { - alloc = Pool>::iterate(handle); + alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { alloc->debug_is_empty(nullptr); - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } #else @@ -146,9 +163,13 @@ namespace snmalloc } template - inline static void debug_in_use(SharedStateHandle handle, size_t count) + inline static void debug_in_use(size_t count) { - auto alloc = Pool>::iterate(handle); + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global status is available only for pool-allocated configurations"); + auto alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { if (alloc->debug_is_in_use()) @@ -159,7 +180,8 @@ namespace snmalloc } count--; } - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); if (count != 0) { diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index c42685e..ad58071 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -38,24 +38,34 @@ namespace snmalloc OnePastEnd }; - // This class contains the fastest path code for the allocator. + /** + * A local allocator contains the fast-path allocation routines and + * encapsulates all of the behaviour of an allocator that is local to some + * context, typically a thread. This delegates to a `CoreAllocator` for all + * slow-path operations, including anything that requires claiming new chunks + * of address space. + * + * The template parameter defines the configuration of this allocator and is + * passed through to the associated `CoreAllocator`. The `Options` structure + * of this defines one property that directly affects the behaviour of the + * local allocator: `LocalAllocSupportsLazyInit`, which defaults to true, + * defines whether the local allocator supports lazy initialisation. If this + * is true then the local allocator will construct a core allocator the first + * time it needs to perform a slow-path operation. If this is false then the + * core allocator must be provided externally by invoking the `init` method + * on this class *before* any allocation-related methods are called. + */ template class LocalAllocator { using CoreAlloc = CoreAllocator; private: - /** - * Contains a way to access all the shared state for this allocator. - * This may have no dynamic state, and be purely static. - */ - SharedStateHandle handle; - // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. // Also contains remote deallocation cache. - LocalCache local_cache; + LocalCache local_cache{&SharedStateHandle::unused_remote}; // Underlying allocator for most non-fast path operations. CoreAlloc* core_alloc{nullptr}; @@ -90,40 +100,59 @@ namespace snmalloc /** * This initialises the fast allocator by acquiring a core allocator, and * setting up its local copy of data structures. + * + * If the allocator does not support lazy initialisation then this assumes + * that initialisation has already taken place and invokes the action + * immediately. */ template SNMALLOC_SLOW_PATH decltype(auto) lazy_init(Action action, Args... args) { SNMALLOC_ASSERT(core_alloc == nullptr); - - // Initialise the thread local allocator - init(); - - // register_clean_up must be called after init. register clean up may be - // implemented with allocation, so need to ensure we have a valid - // allocator at this point. - if (!post_teardown) - // Must be called at least once per thread. - // A pthread implementation only calls the thread destruction handle - // if the key has been set. - handle.register_clean_up(); - - // Perform underlying operation - auto r = action(core_alloc, args...); - - // After performing underlying operation, in the case of teardown already - // having begun, we must flush any state we just acquired. - if (post_teardown) + if constexpr (!SharedStateHandle::Options.LocalAllocSupportsLazyInit) { -#ifdef SNMALLOC_TRACING - std::cout << "post_teardown flush()" << std::endl; -#endif - // We didn't have an allocator because the thread is being torndown. - // We need to return any local state, so we don't leak it. - flush(); + SNMALLOC_CHECK( + false && + "lazy_init called on an allocator that doesn't support lazy " + "initialisation"); + // Unreachable, but needed to keep the type checker happy in deducing + // the return type of this function. + return static_cast(nullptr); } + else + { + // Initialise the thread local allocator + if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + { + init(); + } - return r; + // register_clean_up must be called after init. register clean up may + // be implemented with allocation, so need to ensure we have a valid + // allocator at this point. + if (!post_teardown) + // Must be called at least once per thread. + // A pthread implementation only calls the thread destruction handle + // if the key has been set. + SharedStateHandle::register_clean_up(); + + // Perform underlying operation + auto r = action(core_alloc, args...); + + // After performing underlying operation, in the case of teardown + // already having begun, we must flush any state we just acquired. + if (post_teardown) + { +#ifdef SNMALLOC_TRACING + std::cout << "post_teardown flush()" << std::endl; +#endif + // We didn't have an allocator because the thread is being torndown. + // We need to return any local state, so we don't leak it. + flush(); + } + + return r; + } } /** @@ -144,13 +173,12 @@ namespace snmalloc return check_init([&](CoreAlloc* core_alloc) { // Grab slab of correct size // Set remote as large allocator remote. - auto [chunk, meta] = ChunkAllocator::alloc_chunk( - handle, - core_alloc->backend_state, + auto [chunk, meta] = ChunkAllocator::alloc_chunk( + core_alloc->get_backend_local_state(), bits::next_pow2_bits(size), // TODO large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), - handle.fake_large_remote); + SharedStateHandle::fake_large_remote); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING @@ -164,7 +192,7 @@ namespace snmalloc if (zero_mem == YesZero) { - SharedStateHandle::Backend::Pal::template zero( + SharedStateHandle::Pal::template zero( chunk.unsafe_ptr(), size); } @@ -225,8 +253,7 @@ namespace snmalloc std::cout << "Remote dealloc post" << p << " size " << alloc_size(p) << std::endl; #endif - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(p)); + MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), CapPtr(p), key_global); post_remote_cache(); @@ -252,30 +279,84 @@ namespace snmalloc return local_cache.remote_allocator; } + /** + * SFINAE helper. Matched only if `T` implements `is_initialised`. Calls + * it if it exists. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) + -> decltype(T::is_initialised()) + { + return T::is_initialised(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `is_initialised`. + * Unconditionally returns true if invoked. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, long) + { + return true; + } + + /** + * Call `SharedStateHandle::is_initialised()` if it is implemented, + * unconditionally returns true otherwise. + */ + SNMALLOC_FAST_PATH + bool is_initialised() + { + return call_is_initialised(nullptr, 0); + } + + /** + * SFINAE helper. Matched only if `T` implements `ensure_init`. Calls it + * if it exists. + */ + template + SNMALLOC_FAST_PATH auto call_ensure_init(T*, int) + -> decltype(T::ensure_init()) + { + T::ensure_init(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `ensure_init`. + * Does nothing if called. + */ + template + SNMALLOC_FAST_PATH auto call_ensure_init(T*, long) + {} + + /** + * Call `SharedStateHandle::ensure_init()` if it is implemented, do nothing + * otherwise. + */ + SNMALLOC_FAST_PATH + void ensure_init() + { + call_ensure_init(nullptr, 0); + } + public: - constexpr LocalAllocator() - : handle(SharedStateHandle::get_handle()), - local_cache(&handle.unused_remote) - {} + constexpr LocalAllocator() = default; - LocalAllocator(SharedStateHandle handle) - : handle(handle), local_cache(&handle.unused_remote) - {} - - // This is effectively the constructor for the LocalAllocator, but due to - // not wanting initialisation checks on the fast path, it is initialised - // lazily. - void init() + /** + * Initialise the allocator. For allocators that support local + * initialisation, this is called with a core allocator that this class + * allocates (from a pool allocator) the first time it encounters a slow + * path. If this class is configured without lazy initialisation support + * then this must be called externally + */ + void init(CoreAlloc* c) { // Initialise the global allocator structures - handle.ensure_init(); + ensure_init(); // Should only be called if the allocator has not been initialised. SNMALLOC_ASSERT(core_alloc == nullptr); - // Grab an allocator for this thread. - auto c = Pool::acquire(handle, &(this->local_cache), handle); - // Attach to it. c->attach(&local_cache); core_alloc = c; @@ -286,6 +367,18 @@ namespace snmalloc // local_cache.stats.sta rt(); } + // This is effectively the constructor for the LocalAllocator, but due to + // not wanting initialisation checks on the fast path, it is initialised + // lazily. + void init() + { + // Initialise the global allocator structures + ensure_init(); + // Grab an allocator for this thread. + init(Pool::template acquire( + &(this->local_cache))); + } + // Return all state in the fast allocator and release the underlying // core allocator. This is used during teardown to empty the thread // local state. @@ -303,7 +396,10 @@ namespace snmalloc // Detach underlying allocator core_alloc->attached_cache = nullptr; // Return underlying allocator to the system. - Pool::release(handle, core_alloc); + if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + { + Pool::template release(core_alloc); + } // Set up thread local allocator to look like // it is new to hit slow paths. @@ -311,7 +407,7 @@ namespace snmalloc #ifdef SNMALLOC_TRACING std::cout << "flush(): core_alloc=" << core_alloc << std::endl; #endif - local_cache.remote_allocator = &handle.unused_remote; + local_cache.remote_allocator = &SharedStateHandle::unused_remote; local_cache.remote_dealloc_cache.capacity = 0; } } @@ -365,8 +461,8 @@ namespace snmalloc // before init, that maps null to a remote_deallocator that will never be // in thread local state. - const MetaEntry& entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(p)); + const MetaEntry& entry = + SharedStateHandle::get_meta_data(address_cast(p)); if (likely(local_cache.remote_allocator == entry.get_remote())) { if (likely(CoreAlloc::dealloc_local_object_fast( @@ -376,7 +472,7 @@ namespace snmalloc return; } - if (likely(entry.get_remote() != handle.fake_large_remote)) + if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) { // Check if we have space for the remote deallocation if (local_cache.remote_dealloc_cache.reserve_space(entry)) @@ -416,7 +512,8 @@ namespace snmalloc ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); slab_record->chunk = CapPtr(p); - ChunkAllocator::dealloc(handle, slab_record, slab_sizeclass); + ChunkAllocator::dealloc( + core_alloc->get_backend_local_state(), slab_record, slab_sizeclass); return; } @@ -460,10 +557,9 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a large // object, so we can pull the check for null off the fast path. - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(p_raw)); + MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(p_raw)); - if (likely(entry.get_remote() != handle.fake_large_remote)) + if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) return sizeclass_to_size(entry.get_sizeclass()); // Sizeclass zero is for large is actually zero @@ -484,13 +580,12 @@ namespace snmalloc void* external_pointer(void* p_raw) { // TODO bring back the CHERI bits. Wes to review if required. - if (likely(handle.is_initialised())) + if (likely(is_initialised())) { MetaEntry entry = - SharedStateHandle::Backend::template get_meta_data( - handle.get_backend_state(), address_cast(p_raw)); + SharedStateHandle::template get_meta_data(address_cast(p_raw)); auto sizeclass = entry.get_sizeclass(); - if (likely(entry.get_remote() != handle.fake_large_remote)) + if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) { auto rsize = sizeclass_to_size(sizeclass); auto offset = @@ -533,5 +628,15 @@ namespace snmalloc // We don't know the Start, so return MIN_PTR return nullptr; } + + /** + * Accessor, returns the local cache. If embedding code is allocating the + * core allocator for use by this local allocator then it needs to access + * this field. + */ + LocalCache& get_local_cache() + { + return local_cache; + } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/mem/localcache.h b/src/mem/localcache.h index ae871c9..7e7687f 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -32,7 +32,7 @@ namespace snmalloc auto r = finish_alloc_no_zero(p, sizeclass); if constexpr (zero_mem == YesZero) - SharedStateHandle::Backend::Pal::zero(r, sizeclass_to_size(sizeclass)); + SharedStateHandle::Pal::zero(r, sizeclass_to_size(sizeclass)); // TODO: Should this be zeroing the FreeObject state, in the non-zeroing // case? @@ -72,9 +72,9 @@ namespace snmalloc template< size_t allocator_size, - typename DeallocFun, - typename SharedStateHandle> - bool flush(DeallocFun dealloc, SharedStateHandle handle) + typename SharedStateHandle, + typename DeallocFun> + bool flush(DeallocFun dealloc) { auto& key = entropy.get_free_list_key(); @@ -91,8 +91,8 @@ namespace snmalloc } } - return remote_dealloc_cache.post( - handle, remote_allocator->trunc_id(), key_global); + return remote_dealloc_cache.post( + remote_allocator->trunc_id(), key_global); } template @@ -113,4 +113,4 @@ namespace snmalloc } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 2c88528..ff2da87 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -177,6 +177,16 @@ namespace snmalloc public: constexpr MetaEntry() = default; + /** + * Constructor, provides the remote and sizeclass embedded in a single + * pointer-sized word. This format is not guaranteed to be stable and so + * the second argument of this must always be the return value from + * `get_remote_and_sizeclass`. + */ + MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) + : meta(meta), remote_and_sizeclass(remote_and_sizeclass) + {} + MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass) : meta(meta) { @@ -190,6 +200,17 @@ namespace snmalloc return meta; } + /** + * Return the remote and sizeclass in an implementation-defined encoding. + * This is not guaranteed to be stable across snmalloc releases and so the + * only safe use for this is to pass it to the two-argument constructor of + * this class. + */ + uintptr_t get_remote_and_sizeclass() + { + return remote_and_sizeclass; + } + [[nodiscard]] RemoteAllocator* get_remote() const { return reinterpret_cast( diff --git a/src/mem/pool.h b/src/mem/pool.h index bedbce5..d3c25b4 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -37,9 +37,9 @@ namespace snmalloc { public: template - static T* acquire(SharedStateHandle h, Args&&... args) + static T* acquire(Args&&... args) { - PoolState& pool = h.pool(); + PoolState& pool = SharedStateHandle::pool(); T* p = pool.stack.pop(); if (p != nullptr) @@ -48,12 +48,12 @@ namespace snmalloc return p; } - p = ChunkAllocator::alloc_meta_data( - h, nullptr, std::forward(args)...); + p = ChunkAllocator::alloc_meta_data( + nullptr, std::forward(args)...); if (p == nullptr) { - SharedStateHandle::Backend::Pal::error( + SharedStateHandle::Pal::error( "Failed to initialise thread local allocator."); } @@ -71,21 +71,21 @@ namespace snmalloc * Do not return objects from `extract`. */ template - static void release(SharedStateHandle h, T* p) + static void release(T* p) { // The object's destructor is not run. If the object is "reallocated", it // is returned without the constructor being run, so the object is reused // without re-initialisation. p->reset_in_use(); - h.pool().stack.push(p); + SharedStateHandle::pool().stack.push(p); } template - static T* extract(SharedStateHandle h, T* p = nullptr) + static T* extract(T* p = nullptr) { // Returns a linked list of all objects in the stack, emptying the stack. if (p == nullptr) - return h.pool().stack.pop_all(); + return SharedStateHandle::pool().stack.pop_all(); return p->next; } @@ -96,18 +96,18 @@ namespace snmalloc * Do not return objects from `acquire`. */ template - static void restore(SharedStateHandle h, T* first, T* last) + static void restore(T* first, T* last) { // Pushes a linked list of objects onto the stack. Use to put a linked // list returned by extract back onto the stack. - h.pool().stack.push(first, last); + SharedStateHandle::pool().stack.push(first, last); } template - static T* iterate(SharedStateHandle h, T* p = nullptr) + static T* iterate(T* p = nullptr) { if (p == nullptr) - return h.pool().list; + return SharedStateHandle::pool().list; return p->list_next; } diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 25680b0..a062b9a 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -76,10 +76,7 @@ namespace snmalloc } template - bool post( - SharedStateHandle handle, - RemoteAllocator::alloc_id_t id, - const FreeListKey& key) + bool post(RemoteAllocator::alloc_id_t id, const FreeListKey& key) { SNMALLOC_ASSERT(initialised); size_t post_round = 0; @@ -97,8 +94,8 @@ namespace snmalloc if (!list[i].empty()) { auto [first, last] = list[i].extract_segment(key); - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(first)); + MetaEntry entry = + SharedStateHandle::get_meta_data(address_cast(first)); entry.get_remote()->enqueue(first, last, key); sent_something = true; } @@ -120,8 +117,7 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key); - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(r)); + MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(r)); auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); list[slot].add(r, key); diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 30b3daf..13dc161 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -75,14 +75,14 @@ namespace snmalloc public: template static std::pair, Metaslab*> alloc_chunk( - SharedStateHandle h, - typename SharedStateHandle::Backend::LocalState& backend_state, + typename SharedStateHandle::LocalState& local_state, sizeclass_t sizeclass, sizeclass_t slab_sizeclass, // TODO sizeclass_t size_t slab_size, RemoteAllocator* remote) { - ChunkAllocatorState& state = h.get_slab_allocator_state(); + ChunkAllocatorState& state = + SharedStateHandle::get_slab_allocator_state(&local_state); // Pop a slab auto chunk_record = state.chunk_stack[slab_sizeclass].pop(); @@ -98,15 +98,14 @@ namespace snmalloc << std::endl; #endif MetaEntry entry{meta, remote, sizeclass}; - SharedStateHandle::Backend::set_meta_data( - h.get_backend_state(), address_cast(slab), slab_size, entry); + SharedStateHandle::set_meta_data(address_cast(slab), slab_size, entry); return {slab, meta}; } // Allocate a fresh slab as there are no available ones. // First create meta-data - auto [slab, meta] = SharedStateHandle::Backend::alloc_chunk( - h.get_backend_state(), &backend_state, slab_size, remote, sizeclass); + auto [slab, meta] = SharedStateHandle::alloc_chunk( + &local_state, slab_size, remote, sizeclass); #ifdef SNMALLOC_TRACING std::cout << "Create slab:" << slab.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " << slab_size << std::endl; @@ -122,10 +121,12 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH static void - dealloc(SharedStateHandle h, ChunkRecord* p, size_t slab_sizeclass) + SNMALLOC_SLOW_PATH static void dealloc( + typename SharedStateHandle::LocalState& local_state, + ChunkRecord* p, + size_t slab_sizeclass) { - auto& state = h.get_slab_allocator_state(); + auto& state = SharedStateHandle::get_slab_allocator_state(&local_state); #ifdef SNMALLOC_TRACING std::cout << "Return slab:" << p->chunk.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " @@ -144,15 +145,13 @@ namespace snmalloc */ template static U* alloc_meta_data( - SharedStateHandle h, - typename SharedStateHandle::Backend::LocalState* local_state, - Args&&... args) + typename SharedStateHandle::LocalState* local_state, Args&&... args) { // Cache line align size_t size = bits::align_up(sizeof(U), 64); - CapPtr p = SharedStateHandle::Backend::alloc_meta_data( - h.get_backend_state(), local_state, size); + CapPtr p = + SharedStateHandle::template alloc_meta_data(local_state, size); if (p == nullptr) return nullptr; diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 101e092..dc9cee6 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -156,6 +156,6 @@ namespace snmalloc */ void _malloc_thread_cleanup() { - ThreadAlloc::get().teardown(); + snmalloc::ThreadAlloc::get().teardown(); } #endif diff --git a/src/override/malloc-extensions.cc b/src/override/malloc-extensions.cc index 9245cc9..d672e05 100644 --- a/src/override/malloc-extensions.cc +++ b/src/override/malloc-extensions.cc @@ -6,10 +6,8 @@ using namespace snmalloc; void get_malloc_info_v1(malloc_info_v1* stats) { - auto unused_chunks = - Globals::get_handle().get_slab_allocator_state().unused_memory(); - auto peak = - Globals::get_handle().get_slab_allocator_state().peak_memory_usage(); + auto unused_chunks = Globals::get_slab_allocator_state().unused_memory(); + auto peak = Globals::get_slab_allocator_state().peak_memory_usage(); stats->current_memory_usage = peak - unused_chunks; stats->peak_memory_usage = peak; -} \ No newline at end of file +} diff --git a/src/override/malloc.cc b/src/override/malloc.cc index acd6402..0a9ad36 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -2,7 +2,7 @@ #include "../snmalloc_core.h" #ifndef SNMALLOC_PROVIDE_OWN_CONFIG -# include "../mem/globalconfig.h" +# include "../backend/globalconfig.h" // The default configuration for snmalloc is used if alternative not defined namespace snmalloc { diff --git a/src/snmalloc.h b/src/snmalloc.h index 6abd703..e1d90d4 100644 --- a/src/snmalloc.h +++ b/src/snmalloc.h @@ -3,8 +3,13 @@ // Core implementation of snmalloc independent of the configuration mode #include "snmalloc_core.h" +// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own +// definition of `snmalloc::Alloc` and include `snmalloc_front.h` before +// including any files that include `snmalloc.h` and consume the global +// allocation APIs. +#ifndef SNMALLOC_PROVIDE_OWN_CONFIG // Default implementation of global state -#include "mem/globalconfig.h" +# include "backend/globalconfig.h" // The default configuration for snmalloc namespace snmalloc @@ -13,4 +18,5 @@ namespace snmalloc } // User facing API surface, needs to know what `Alloc` is. -#include "snmalloc_front.h" \ No newline at end of file +# include "snmalloc_front.h" +#endif diff --git a/src/snmalloc_core.h b/src/snmalloc_core.h index 0c6a2d3..d8b14aa 100644 --- a/src/snmalloc_core.h +++ b/src/snmalloc_core.h @@ -1,3 +1,6 @@ #pragma once -#include "mem/globalalloc.h" \ No newline at end of file +#include "backend/address_space.h" +#include "backend/commonconfig.h" +#include "backend/pagemap.h" +#include "mem/globalalloc.h" diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 1f0ca0b..db75ccf 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -1,4 +1,4 @@ -#include "mem/fixedglobalconfig.h" +#include "backend/fixedglobalconfig.h" #include "test/setup.h" #include @@ -29,9 +29,8 @@ int main() std::cout << "Allocated region " << oe_base << " - " << pointer_offset(oe_base, size) << std::endl; - CustomGlobals fixed_handle; CustomGlobals::init(oe_base, size); - FixedAlloc a(fixed_handle); + FixedAlloc a; size_t object_size = 128; size_t count = 0; diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index e87e0ee..c8fc74a 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -232,6 +232,6 @@ int main(int argc, char** argv) abort(); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); return 0; } diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 9c6cd35..f4afd8f 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -183,7 +183,7 @@ void test_calloc() alloc.dealloc(p, size); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } void test_double_alloc() @@ -228,7 +228,7 @@ void test_double_alloc() } } } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } void test_external_pointer() @@ -264,7 +264,7 @@ void test_external_pointer() alloc.dealloc(p1, size); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); }; void check_offset(void* base, void* interior) diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index 36b4788..156612c 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -8,7 +8,7 @@ int main() auto r = a.alloc(16); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != false) { abort(); @@ -16,7 +16,7 @@ int main() a.dealloc(r); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != true) { abort(); @@ -24,7 +24,7 @@ int main() r = a.alloc(16); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != false) { abort(); @@ -32,10 +32,10 @@ int main() a.dealloc(r); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != true) { abort(); } #endif -} \ No newline at end of file +} diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index a5b1fe8..effb62f 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -4,7 +4,7 @@ // Specify using own #define SNMALLOC_EXTERNAL_THREAD_ALLOC -#include "mem/globalconfig.h" +#include "backend/globalconfig.h" namespace snmalloc { diff --git a/src/test/func/two_alloc_types/alloc1.cc b/src/test/func/two_alloc_types/alloc1.cc index 4e5f9d0..feb47bd 100644 --- a/src/test/func/two_alloc_types/alloc1.cc +++ b/src/test/func/two_alloc_types/alloc1.cc @@ -3,7 +3,7 @@ // Redefine the namespace, so we can have two versions. #define snmalloc snmalloc_enclave -#include +#include #include // Specify type of allocator @@ -19,6 +19,5 @@ namespace snmalloc extern "C" void oe_allocator_init(void* base, void* end) { - snmalloc::CustomGlobals fixed_handle; - fixed_handle.init(base, address_cast(end) - address_cast(base)); + snmalloc::CustomGlobals::init(base, address_cast(end) - address_cast(base)); } diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 5be5641..74f9cd2 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -142,7 +142,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) } #ifndef NDEBUG - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); #endif }; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index ac66a13..90aa1ba 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -47,7 +47,7 @@ namespace test alloc.dealloc(objects[i]); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } void test_external_pointer(xoroshiro::p128r64& r) diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index 2bf4980..cd15a12 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -60,7 +60,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) } } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } int main(int, char**)