diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 549c317..d24b89f 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -1,6 +1,8 @@ #pragma once #include "../ds/address.h" #include "../ds/flaglock.h" +#include "../mem/allocconfig.h" +#include "../mem/metaslab.h" #include "../pal/pal.h" #include diff --git a/src/backend/backend.h b/src/backend/backend.h index f0311ae..b9b62b9 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -3,6 +3,7 @@ #include "../mem/metaslab.h" #include "../pal/pal.h" #include "address_space.h" +#include "commonconfig.h" #include "pagemap.h" namespace snmalloc @@ -15,7 +16,7 @@ namespace snmalloc SNMALLOC_CONCEPT(ConceptPAL) PAL, bool fixed_range, typename PageMapEntry = MetaEntry> - class BackendAllocator + class BackendAllocator : public CommonConfig { // Size of local address space requests. Currently aimed at 2MiB large // pages but should make this configurable (i.e. for OE, so we don't need as @@ -54,55 +55,31 @@ namespace snmalloc #endif }; - /** - * Global state for the backend allocator - * - * This contains the various global datastructures required to store - * meta-data for each chunk of memory, and to provide well aligned chunks - * of memory. - * - * This type is required by snmalloc to exist as part of the Backend. - */ - class GlobalState + SNMALLOC_REQUIRE_CONSTINIT + static inline AddressSpaceManager address_space; + + SNMALLOC_REQUIRE_CONSTINIT + static inline FlatPagemap + pagemap; + + public: + template + static std::enable_if_t init() { - friend BackendAllocator; + static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - protected: - AddressSpaceManager address_space; + pagemap.init(); + } - FlatPagemap pagemap; + template + static std::enable_if_t init(void* base, size_t length) + { + static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - public: - template - std::enable_if_t init() - { - static_assert( - fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - - pagemap.init(); - - if constexpr (fixed_range) - { - abort(); - } - } - - template - std::enable_if_t init(void* base, size_t length) - { - static_assert( - fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - - auto [heap_base, heap_length] = pagemap.init(base, length); - address_space.add_range( - CapPtr(heap_base), heap_length, pagemap); - - if constexpr (!fixed_range) - { - abort(); - } - } - }; + auto [heap_base, heap_length] = pagemap.init(base, length); + address_space.add_range( + CapPtr(heap_base), heap_length, pagemap); + } private: #ifdef SNMALLOC_CHECK_CLIENT @@ -138,8 +115,7 @@ namespace snmalloc * space managers. */ template - static CapPtr - reserve(GlobalState& h, LocalState* local_state, size_t size) + static CapPtr reserve(LocalState* local_state, size_t size) { #ifdef SNMALLOC_CHECK_CLIENT constexpr auto MAX_CACHED_SIZE = @@ -148,7 +124,7 @@ namespace snmalloc constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK; #endif - auto& global = h.address_space; + auto& global = address_space; CapPtr p; if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) @@ -160,14 +136,14 @@ namespace snmalloc auto& local = local_state->local_address_space; #endif - p = local.template reserve_with_left_over(size, h.pagemap); + p = local.template reserve_with_left_over(size, pagemap); if (p != nullptr) { return p; } auto refill_size = LOCAL_CACHE_BLOCK; - auto refill = global.template reserve(refill_size, h.pagemap); + auto refill = global.template reserve(refill_size, pagemap); if (refill == nullptr) return nullptr; @@ -179,10 +155,10 @@ namespace snmalloc } #endif PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range(refill, refill_size, h.pagemap); + local.template add_range(refill, refill_size, pagemap); // This should succeed - return local.template reserve_with_left_over(size, h.pagemap); + return local.template reserve_with_left_over(size, pagemap); } #ifdef SNMALLOC_CHECK_CLIENT @@ -193,7 +169,7 @@ namespace snmalloc size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); size_t size_request = rsize * 64; - p = global.template reserve(size_request, h.pagemap); + p = global.template reserve(size_request, pagemap); if (p == nullptr) return nullptr; @@ -209,7 +185,7 @@ namespace snmalloc SNMALLOC_ASSERT(!is_meta); #endif - p = global.template reserve_with_left_over(size, h.pagemap); + p = global.template reserve_with_left_over(size, pagemap); return p; } @@ -219,11 +195,16 @@ namespace snmalloc * * Backend allocator may use guard pages and separate area of * address space to protect this from corruption. + * + * The template argument is the type of the metadata being allocated. This + * allows the backend to allocate different types of metadata in different + * places or with different policies. */ + template static CapPtr - alloc_meta_data(GlobalState& h, LocalState* local_state, size_t size) + alloc_meta_data(LocalState* local_state, size_t size) { - return reserve(h, local_state, size); + return reserve(local_state, size); } /** @@ -236,7 +217,6 @@ namespace snmalloc * where metaslab, is the second element of the pair return. */ static std::pair, Metaslab*> alloc_chunk( - GlobalState& h, LocalState* local_state, size_t size, RemoteAllocator* remote, @@ -246,12 +226,12 @@ namespace snmalloc SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); auto meta = reinterpret_cast( - reserve(h, local_state, sizeof(Metaslab)).unsafe_ptr()); + reserve(local_state, sizeof(Metaslab)).unsafe_ptr()); if (meta == nullptr) return {nullptr, nullptr}; - CapPtr p = reserve(h, local_state, size); + CapPtr p = reserve(local_state, size); #ifdef SNMALLOC_TRACING std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" @@ -274,7 +254,7 @@ namespace snmalloc a < address_cast(pointer_offset(p, size)); a += MIN_CHUNK_SIZE) { - h.pagemap.set(a, t); + pagemap.set(a, t); } return {p, meta}; } @@ -286,20 +266,19 @@ namespace snmalloc * to access a location that is not backed by a chunk. */ template - static const MetaEntry& get_meta_data(GlobalState& h, address_t p) + static const MetaEntry& get_meta_data(address_t p) { - return h.pagemap.template get(p); + return pagemap.template get(p); } /** * Set the metadata associated with a chunk. */ - static void - set_meta_data(GlobalState& h, address_t p, size_t size, MetaEntry t) + static void set_meta_data(address_t p, size_t size, MetaEntry t) { for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) { - h.pagemap.set(a, t); + pagemap.set(a, t); } } }; diff --git a/src/backend/commonconfig.h b/src/backend/commonconfig.h new file mode 100644 index 0000000..689a139 --- /dev/null +++ b/src/backend/commonconfig.h @@ -0,0 +1,118 @@ +#pragma once + +#include "../ds/defines.h" +#include "../mem/remotecache.h" + +namespace snmalloc +{ + // Forward reference to thread local cleanup. + void register_clean_up(); + + /** + * Options for a specific snmalloc configuration. Every globals object must + * have one `constexpr` instance of this class called `Options`. This should + * be constructed to explicitly override any of the defaults. A + * configuration that does not need to override anything would simply declare + * this as a field of the global object: + * + * ```c++ + * constexpr static snmalloc::Flags Options{}; + * ``` + * + * A global configuration that wished to use out-of-line message queues but + * accept the defaults for everything else would instead do this: + * + * ```c++ + * constexpr static snmalloc::Flags Options{.IsQueueInline = false}; + * ``` + * + * To maintain backwards source compatibility in future versions, any new + * option added here should have its default set to be whatever snmalloc was + * doing before the new option was added. + */ + struct Flags + { + /** + * Should allocators have inline message queues? If this is true then + * the `CoreAllocator` is responsible for allocating the + * `RemoteAllocator` that contains its message queue. If this is false + * then the `RemoteAllocator` must be separately allocated and provided + * to the `CoreAllocator` before it is used. + * + * Setting this to `false` currently requires also setting + * `LocalAllocSupportsLazyInit` to false so that the `CoreAllocator` can + * be provided to the `LocalAllocator` fully initialised but in the + * future it may be possible to allocate the `RemoteAllocator` via + * `alloc_meta_data` or a similar API in the back end. + */ + bool IsQueueInline = true; + + /** + * Does the `CoreAllocator` own a `Backend::LocalState` object? If this is + * true then the `CoreAllocator` is responsible for allocating and + * deallocating a local state object, otherwise the surrounding code is + * responsible for creating it. + * + * Use cases that set this to false will probably also need to set + * `LocalAllocSupportsLazyInit` to false so that they can provide the local + * state explicitly during allocator creation. + */ + bool CoreAllocOwnsLocalState = true; + + /** + * Are `CoreAllocator` allocated by the pool allocator? If not then the + * code embedding this snmalloc configuration is responsible for allocating + * `CoreAllocator` instances. + * + * Users setting this flag must also set `LocalAllocSupportsLazyInit` to + * false currently because there is no alternative mechanism for allocating + * core allocators. This may change in future versions. + */ + bool CoreAllocIsPoolAllocated = true; + + /** + * Do `LocalAllocator` instances in this configuration support lazy + * initialisation? If so, then the first exit from a fast path will + * trigger allocation of a `CoreAllocator` and associated state. If not + * then the code embedding this configuration of snmalloc is responsible + * for allocating core allocators. + */ + bool LocalAllocSupportsLazyInit = true; + }; + + /** + * Class containing definitions that are likely to be used by all except for + * the most unusual back-end implementations. This can be subclassed as a + * convenience for back-end implementers, but is not required. + */ + class CommonConfig + { + public: + /** + * Special remote that should never be used as a real remote. + * This is used to initialise allocators that should always hit the + * remote path for deallocation. Hence moving a branch off the critical + * path. + */ + SNMALLOC_REQUIRE_CONSTINIT + inline static RemoteAllocator unused_remote; + + /** + * Special remote that is used in meta-data for large allocations. + * + * nullptr is considered a large allocations for this purpose to move + * of the critical path. + * + * Bottom bits of the remote pointer are used for a sizeclass, we need + * size bits to represent the non-large sizeclasses, we can then get + * the large sizeclass by having the fake large_remote considerably + * more aligned. + */ + SNMALLOC_REQUIRE_CONSTINIT + inline static constexpr RemoteAllocator* fake_large_remote{nullptr}; + + static_assert( + &unused_remote != fake_large_remote, + "Compilation should ensure these are different"); + }; +} // namespace snmalloc diff --git a/src/mem/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h similarity index 52% rename from src/mem/fixedglobalconfig.h rename to src/backend/fixedglobalconfig.h index 01b268f..2e4473e 100644 --- a/src/mem/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -12,50 +12,27 @@ namespace snmalloc * A single fixed address range allocator configuration */ template - class FixedGlobals : public CommonConfig + class FixedGlobals final : public BackendAllocator { - public: - using Backend = BackendAllocator; - private: - inline static typename Backend::GlobalState backend_state; - + using Backend = BackendAllocator; inline static ChunkAllocatorState slab_allocator_state; inline static PoolState> alloc_pool; public: - static typename Backend::GlobalState& get_backend_state() - { - return backend_state; - } - - ChunkAllocatorState& get_slab_allocator_state() + static ChunkAllocatorState& + get_slab_allocator_state(typename Backend::LocalState*) { return slab_allocator_state; } - PoolState>& pool() + static PoolState>& pool() { return alloc_pool; } - static constexpr bool IsQueueInline = true; - - // Performs initialisation for this configuration - // of allocators. Will be called at most once - // before any other datastructures are accessed. - void ensure_init() noexcept - { -#ifdef SNMALLOC_TRACING - std::cout << "Run init_impl" << std::endl; -#endif - } - - static bool is_initialised() - { - return true; - } + static constexpr Flags Options{}; // This needs to be a forward reference as the // thread local state will need to know about this. @@ -68,12 +45,7 @@ namespace snmalloc static void init(void* base, size_t length) { - get_backend_state().init(base, length); - } - - constexpr static FixedGlobals get_handle() - { - return {}; + Backend::init(base, length); } }; } diff --git a/src/mem/globalconfig.h b/src/backend/globalconfig.h similarity index 75% rename from src/mem/globalconfig.h rename to src/backend/globalconfig.h index cc5dc76..f084703 100644 --- a/src/mem/globalconfig.h +++ b/src/backend/globalconfig.h @@ -23,15 +23,15 @@ namespace snmalloc } #endif - class Globals : public CommonConfig + /** + * The default configuration for a global snmalloc. This allocates memory + * from the operating system and expects to manage memory anywhere in the + * address space. + */ + class Globals final : public BackendAllocator { - public: - using Backend = BackendAllocator; - private: - SNMALLOC_REQUIRE_CONSTINIT - inline static Backend::GlobalState backend_state; - + using Backend = BackendAllocator; SNMALLOC_REQUIRE_CONSTINIT inline static ChunkAllocatorState slab_allocator_state; @@ -45,27 +45,23 @@ namespace snmalloc inline static std::atomic_flag initialisation_lock{}; public: - Backend::GlobalState& get_backend_state() - { - return backend_state; - } - - ChunkAllocatorState& get_slab_allocator_state() + static ChunkAllocatorState& + get_slab_allocator_state(Backend::LocalState* = nullptr) { return slab_allocator_state; } - PoolState>& pool() + static PoolState>& pool() { return alloc_pool; } - static constexpr bool IsQueueInline = true; + static constexpr Flags Options{}; // Performs initialisation for this configuration // of allocators. Needs to be idempotent, // and concurrency safe. - void ensure_init() + static void ensure_init() { FlagLock lock{initialisation_lock}; #ifdef SNMALLOC_TRACING @@ -81,7 +77,7 @@ namespace snmalloc key_global = FreeListKey(entropy.get_free_list_key()); // Need to initialise pagemap. - backend_state.init(); + Backend::init(); #ifdef USE_SNMALLOC_STATS atexit(snmalloc::print_stats); @@ -90,7 +86,7 @@ namespace snmalloc initialised = true; } - bool is_initialised() + static bool is_initialised() { return initialised; } @@ -99,16 +95,9 @@ namespace snmalloc // thread local state will need to know about this. // This may allocate, so should only be called once // a thread local allocator is available. - void register_clean_up() + static void register_clean_up() { snmalloc::register_clean_up(); } - - // This is an empty structure as all the state is global - // for this allocator configuration. - static constexpr Globals get_handle() - { - return {}; - } }; } // namespace snmalloc diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 4f4ab6e..d640ae1 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -66,6 +66,36 @@ namespace snmalloc constexpr FlatPagemap() = default; + /** + * For pagemaps that cover an entire fixed address space, return the size + * that they must be. This allows the caller to allocate the correct + * amount of memory to be passed to `init`. This is not available for + * fixed-range pagemaps, whose size depends on dynamic configuration. + */ + template + static constexpr std::enable_if_t required_size() + { + static_assert( + has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); + constexpr size_t COVERED_BITS = bits::ADDRESS_BITS - GRANULARITY_BITS; + constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); + return ENTRIES * sizeof(T); + } + + /** + * Initialise with pre-allocated memory. + * + * This is currently disabled for bounded pagemaps but may be reenabled if + * `required_size` is enabled for the has-bounds case. + */ + template + std::enable_if_t init(T* address) + { + static_assert( + has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); + body = address; + } + /** * Initialise the pagemap with bounds. * @@ -117,11 +147,7 @@ namespace snmalloc { static_assert( has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); - static constexpr size_t COVERED_BITS = - bits::ADDRESS_BITS - GRANULARITY_BITS; - static constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); - - static constexpr size_t REQUIRED_SIZE = ENTRIES * sizeof(T); + static constexpr size_t REQUIRED_SIZE = required_size(); #ifdef SNMALLOC_CHECK_CLIENT // Allocate a power of two extra to allow the placement of the diff --git a/src/mem/commonconfig.h b/src/mem/commonconfig.h deleted file mode 100644 index 9c24371..0000000 --- a/src/mem/commonconfig.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include "../ds/defines.h" -#include "remotecache.h" - -namespace snmalloc -{ - // Forward reference to thread local cleanup. - void register_clean_up(); - - class CommonConfig - { - public: - /** - * Special remote that should never be used as a real remote. - * This is used to initialise allocators that should always hit the - * remote path for deallocation. Hence moving a branch off the critical - * path. - */ - SNMALLOC_REQUIRE_CONSTINIT - inline static RemoteAllocator unused_remote; - - /** - * Special remote that is used in meta-data for large allocations. - * - * nullptr is considered a large allocations for this purpose to move - * of the critical path. - * - * Bottom bits of the remote pointer are used for a sizeclass, we need - * size bits to represent the non-large sizeclasses, we can then get - * the large sizeclass by having the fake large_remote considerably - * more aligned. - */ - SNMALLOC_REQUIRE_CONSTINIT - inline static constexpr RemoteAllocator* fake_large_remote{nullptr}; - - static_assert( - &unused_remote != fake_large_remote, - "Compilation should ensure these are different"); - }; -} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 96490a7..0097b12 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -11,10 +11,45 @@ namespace snmalloc { + /** + * Empty class used as the superclass for `CoreAllocator` when it does not + * opt into pool allocation. This class exists because `std::conditional` + * (or other equivalent features in C++) can choose between options for + * superclasses but they cannot choose whether a class has a superclass. + * Setting the superclass to an empty class is equivalent to no superclass. + */ + class NotPoolAllocated + {}; + + /** + * The core, stateful, part of a memory allocator. Each `LocalAllocator` + * owns one `CoreAllocator` once it is initialised. + * + * The template parameter provides all of the global configuration for this + * instantiation of snmalloc. This includes three options that apply to this + * class: + * + * - `CoreAllocIsPoolAllocated` defines whether this `CoreAlloc` + * configuration should support pool allocation. This defaults to true but + * a configuration that allocates allocators eagerly may opt out. + * - `CoreAllocOwnsLocalState` defines whether the `CoreAllocator` owns the + * associated `LocalState` object. If this is true (the default) then + * `CoreAllocator` embeds the LocalState object. If this is set to false + * then a `LocalState` object must be provided to the constructor. This + * allows external code to provide explicit configuration of the address + * range managed by this object. + * - `IsQueueInline` (defaults to true) defines whether the message queue + * (`RemoteAllocator`) for this class is inline or provided externally. If + * provided externally, then it must be set explicitly with + * `init_message_queue`. + */ template - class CoreAllocator : public Pooled> + class CoreAllocator : public std::conditional_t< + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + Pooled>, + NotPoolAllocated> { - template + template friend class LocalAllocator; /** @@ -33,16 +68,27 @@ namespace snmalloc * allocator */ std::conditional_t< - SharedStateHandle::IsQueueInline, + SharedStateHandle::Options.IsQueueInline, RemoteAllocator, RemoteAllocator*> remote_alloc; /** - * A local area of address space managed by this allocator. - * Used to reduce calls on the global address space. + * The type used local state. This is defined by the back end. */ - typename SharedStateHandle::Backend::LocalState backend_state; + using LocalState = typename SharedStateHandle::LocalState; + + /** + * A local area of address space managed by this allocator. + * Used to reduce calls on the global address space. This is inline if the + * core allocator owns the local state or indirect if it is owned + * externally. + */ + std::conditional_t< + SharedStateHandle::Options.CoreAllocOwnsLocalState, + LocalState, + LocalState*> + backend_state; /** * This is the thread local structure associated to this @@ -50,12 +96,6 @@ namespace snmalloc */ LocalCache* attached_cache; - /** - * This contains the way to access all the global state and - * configuration for the system setup. - */ - SharedStateHandle handle; - /** * The message queue needs to be accessible from other threads * @@ -64,7 +104,7 @@ namespace snmalloc */ auto* public_state() { - if constexpr (SharedStateHandle::IsQueueInline) + if constexpr (SharedStateHandle::Options.IsQueueInline) { return &remote_alloc; } @@ -260,8 +300,10 @@ namespace snmalloc // TODO delay the clear to the next user of the slab, or teardown so // don't touch the cache lines at this point in check_client. auto chunk_record = clear_slab(meta, sizeclass); - ChunkAllocator::dealloc( - handle, chunk_record, sizeclass_to_slab_sizeclass(sizeclass)); + ChunkAllocator::dealloc( + get_backend_local_state(), + chunk_record, + sizeclass_to_slab_sizeclass(sizeclass)); } else { @@ -335,8 +377,8 @@ namespace snmalloc for (size_t i = 0; i < REMOTE_BATCH; i++) { auto p = message_queue().peek(); - auto& entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), snmalloc::address_cast(p)); + auto& entry = + SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); auto r = message_queue().dequeue(key_global); @@ -388,9 +430,11 @@ namespace snmalloc } } - public: - CoreAllocator(LocalCache* cache, SharedStateHandle handle) - : attached_cache(cache), handle(handle) + /** + * Initialiser, shared code between the constructors for different + * configurations. + */ + void init() { #ifdef SNMALLOC_TRACING std::cout << "Making an allocator." << std::endl; @@ -398,13 +442,16 @@ namespace snmalloc // Entropy must be first, so that all data-structures can use the key // it generates. // This must occur before any freelists are constructed. - entropy.init(); + entropy.init(); // Ignoring stats for now. // stats().start(); - init_message_queue(); - message_queue().invariant(); + if constexpr (SharedStateHandle::Options.IsQueueInline) + { + init_message_queue(); + message_queue().invariant(); + } #ifndef NDEBUG for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) @@ -423,6 +470,44 @@ namespace snmalloc #endif } + public: + /** + * Constructor for the case that the core allocator owns the local state. + * SFINAE disabled if the allocator does not own the local state. + */ + template< + typename Config = SharedStateHandle, + typename = std::enable_if_t> + CoreAllocator(LocalCache* cache) : attached_cache(cache) + { + init(); + } + + /** + * Constructor for the case that the core allocator does not owns the local + * state. SFINAE disabled if the allocator does own the local state. + */ + template< + typename Config = SharedStateHandle, + typename = std::enable_if_t> + CoreAllocator(LocalCache* cache, LocalState* backend = nullptr) + : backend_state(backend), attached_cache(cache) + { + init(); + } + + /** + * If the message queue is not inline, provide it. This will then + * configure the message queue for use. + */ + template + std::enable_if_t init_message_queue(RemoteAllocator* q) + { + remote_alloc = q; + init_message_queue(); + message_queue().invariant(); + } + /** * Post deallocations onto other threads. * @@ -432,9 +517,9 @@ namespace snmalloc SNMALLOC_FAST_PATH bool post() { // stats().remote_post(); // TODO queue not in line! - bool sent_something = - attached_cache->remote_dealloc_cache.post( - handle, public_state()->trunc_id(), key_global); + bool sent_something = attached_cache->remote_dealloc_cache + .post( + public_state()->trunc_id(), key_global); return sent_something; } @@ -454,8 +539,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(void* p) { - auto entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), snmalloc::address_cast(p)); + auto entry = SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); if (likely(dealloc_local_object_fast(entry, p, entropy))) return; @@ -506,6 +590,24 @@ namespace snmalloc return small_alloc_slow(sizeclass, fast_free_list, rsize); } + /** + * Accessor for the local state. This hides whether the local state is + * stored inline or provided externally from the rest of the code. + */ + SNMALLOC_FAST_PATH + LocalState& get_backend_local_state() + { + if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + { + return backend_state; + } + else + { + SNMALLOC_ASSERT(backend_state); + return *backend_state; + } + } + template SNMALLOC_SLOW_PATH void* small_alloc_slow( sizeclass_t sizeclass, FreeListIter& fast_free_list, size_t rsize) @@ -519,13 +621,13 @@ namespace snmalloc std::cout << "slab size " << slab_size << std::endl; #endif - auto [slab, meta] = snmalloc::ChunkAllocator::alloc_chunk( - handle, - backend_state, - sizeclass, - slab_sizeclass, - slab_size, - public_state()); + auto [slab, meta] = + snmalloc::ChunkAllocator::alloc_chunk( + get_backend_local_state(), + sizeclass, + slab_sizeclass, + slab_size, + public_state()); if (slab == nullptr) { @@ -563,8 +665,8 @@ namespace snmalloc { bool need_post = true; // Always going to post, so ignore. auto n = p->atomic_read_next(key_global); - auto& entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), snmalloc::address_cast(p)); + auto& entry = + SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); p = n; } @@ -577,8 +679,9 @@ namespace snmalloc handle_message_queue([]() {}); } - auto posted = attached_cache->flush( - [&](auto p) { dealloc_local_object(p); }, handle); + auto posted = + attached_cache->flush( + [&](auto p) { dealloc_local_object(p); }); // We may now have unused slabs, return to the global allocator. for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index 0026598..cc4ecd5 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -6,9 +6,13 @@ namespace snmalloc { template - inline static void aggregate_stats(SharedStateHandle handle, Stats& stats) + inline static void aggregate_stats(Stats& stats) { - auto* alloc = Pool>::iterate(handle); + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global statistics are available only for pool-allocated configurations"); + auto* alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { @@ -16,45 +20,51 @@ namespace snmalloc if (a != nullptr) stats.add(*a); stats.add(alloc->stats()); - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } #ifdef USE_SNMALLOC_STATS template - inline static void print_all_stats( - SharedStateHandle handle, std::ostream& o, uint64_t dumpid = 0) + inline static void print_all_stats(std::ostream& o, uint64_t dumpid = 0) { - auto alloc = Pool>::iterate(handle); + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global statistics are available only for pool-allocated configurations"); + auto alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { auto stats = alloc->stats(); if (stats != nullptr) - stats->template print(o, dumpid, alloc->id()); - alloc = Pool>::iterate(handle, alloc); + stats->template print(o, dumpid, alloc->id()); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } #else template - inline static void - print_all_stats(SharedStateHandle handle, void*& o, uint64_t dumpid = 0) + inline static void print_all_stats(void*& o, uint64_t dumpid = 0) { UNUSED(o); UNUSED(dumpid); - UNUSED(handle); } #endif template - inline static void cleanup_unused(SharedStateHandle handle) + inline static void cleanup_unused() { #ifndef SNMALLOC_PASS_THROUGH + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global cleanup is available only for pool-allocated configurations"); // Call this periodically to free and coalesce memory allocated by // allocators that are not currently in use by any thread. // One atomic operation to extract the stack, another to restore it. // Handling the message queue for each stack is non-atomic. - auto* first = Pool>::extract(handle); + auto* first = Pool>::extract(); auto* alloc = first; decltype(alloc) last; @@ -64,10 +74,10 @@ namespace snmalloc { alloc->flush(); last = alloc; - alloc = Pool>::extract(handle, alloc); + alloc = Pool>::extract(alloc); } - Pool>::restore(handle, first, last); + Pool>::restore(first, last); } #endif } @@ -78,13 +88,16 @@ namespace snmalloc raise an error all the allocators are not empty. */ template - inline static void - debug_check_empty(SharedStateHandle handle, bool* result = nullptr) + inline static void debug_check_empty(bool* result = nullptr) { #ifndef SNMALLOC_PASS_THROUGH + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global status is available only for pool-allocated configurations"); // This is a debugging function. It checks that all memory from all // allocators has been freed. - auto* alloc = Pool>::iterate(handle); + auto* alloc = Pool>::template iterate< + SharedStateHandle>(); # ifdef SNMALLOC_TRACING std::cout << "debug check empty: first " << alloc << std::endl; @@ -98,7 +111,8 @@ namespace snmalloc std::cout << "debug_check_empty: Check all allocators!" << std::endl; # endif done = true; - alloc = Pool>::iterate(handle); + alloc = Pool>::template iterate< + SharedStateHandle>(); okay = true; while (alloc != nullptr) @@ -120,7 +134,8 @@ namespace snmalloc # ifdef SNMALLOC_TRACING std::cout << "debug check empty: okay = " << okay << std::endl; # endif - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } @@ -133,11 +148,13 @@ namespace snmalloc // Redo check so abort is on allocator with allocation left. if (!okay) { - alloc = Pool>::iterate(handle); + alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { alloc->debug_is_empty(nullptr); - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } #else @@ -146,9 +163,13 @@ namespace snmalloc } template - inline static void debug_in_use(SharedStateHandle handle, size_t count) + inline static void debug_in_use(size_t count) { - auto alloc = Pool>::iterate(handle); + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global status is available only for pool-allocated configurations"); + auto alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { if (alloc->debug_is_in_use()) @@ -159,7 +180,8 @@ namespace snmalloc } count--; } - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); if (count != 0) { diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index c42685e..ad58071 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -38,24 +38,34 @@ namespace snmalloc OnePastEnd }; - // This class contains the fastest path code for the allocator. + /** + * A local allocator contains the fast-path allocation routines and + * encapsulates all of the behaviour of an allocator that is local to some + * context, typically a thread. This delegates to a `CoreAllocator` for all + * slow-path operations, including anything that requires claiming new chunks + * of address space. + * + * The template parameter defines the configuration of this allocator and is + * passed through to the associated `CoreAllocator`. The `Options` structure + * of this defines one property that directly affects the behaviour of the + * local allocator: `LocalAllocSupportsLazyInit`, which defaults to true, + * defines whether the local allocator supports lazy initialisation. If this + * is true then the local allocator will construct a core allocator the first + * time it needs to perform a slow-path operation. If this is false then the + * core allocator must be provided externally by invoking the `init` method + * on this class *before* any allocation-related methods are called. + */ template class LocalAllocator { using CoreAlloc = CoreAllocator; private: - /** - * Contains a way to access all the shared state for this allocator. - * This may have no dynamic state, and be purely static. - */ - SharedStateHandle handle; - // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. // Also contains remote deallocation cache. - LocalCache local_cache; + LocalCache local_cache{&SharedStateHandle::unused_remote}; // Underlying allocator for most non-fast path operations. CoreAlloc* core_alloc{nullptr}; @@ -90,40 +100,59 @@ namespace snmalloc /** * This initialises the fast allocator by acquiring a core allocator, and * setting up its local copy of data structures. + * + * If the allocator does not support lazy initialisation then this assumes + * that initialisation has already taken place and invokes the action + * immediately. */ template SNMALLOC_SLOW_PATH decltype(auto) lazy_init(Action action, Args... args) { SNMALLOC_ASSERT(core_alloc == nullptr); - - // Initialise the thread local allocator - init(); - - // register_clean_up must be called after init. register clean up may be - // implemented with allocation, so need to ensure we have a valid - // allocator at this point. - if (!post_teardown) - // Must be called at least once per thread. - // A pthread implementation only calls the thread destruction handle - // if the key has been set. - handle.register_clean_up(); - - // Perform underlying operation - auto r = action(core_alloc, args...); - - // After performing underlying operation, in the case of teardown already - // having begun, we must flush any state we just acquired. - if (post_teardown) + if constexpr (!SharedStateHandle::Options.LocalAllocSupportsLazyInit) { -#ifdef SNMALLOC_TRACING - std::cout << "post_teardown flush()" << std::endl; -#endif - // We didn't have an allocator because the thread is being torndown. - // We need to return any local state, so we don't leak it. - flush(); + SNMALLOC_CHECK( + false && + "lazy_init called on an allocator that doesn't support lazy " + "initialisation"); + // Unreachable, but needed to keep the type checker happy in deducing + // the return type of this function. + return static_cast(nullptr); } + else + { + // Initialise the thread local allocator + if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + { + init(); + } - return r; + // register_clean_up must be called after init. register clean up may + // be implemented with allocation, so need to ensure we have a valid + // allocator at this point. + if (!post_teardown) + // Must be called at least once per thread. + // A pthread implementation only calls the thread destruction handle + // if the key has been set. + SharedStateHandle::register_clean_up(); + + // Perform underlying operation + auto r = action(core_alloc, args...); + + // After performing underlying operation, in the case of teardown + // already having begun, we must flush any state we just acquired. + if (post_teardown) + { +#ifdef SNMALLOC_TRACING + std::cout << "post_teardown flush()" << std::endl; +#endif + // We didn't have an allocator because the thread is being torndown. + // We need to return any local state, so we don't leak it. + flush(); + } + + return r; + } } /** @@ -144,13 +173,12 @@ namespace snmalloc return check_init([&](CoreAlloc* core_alloc) { // Grab slab of correct size // Set remote as large allocator remote. - auto [chunk, meta] = ChunkAllocator::alloc_chunk( - handle, - core_alloc->backend_state, + auto [chunk, meta] = ChunkAllocator::alloc_chunk( + core_alloc->get_backend_local_state(), bits::next_pow2_bits(size), // TODO large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), - handle.fake_large_remote); + SharedStateHandle::fake_large_remote); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING @@ -164,7 +192,7 @@ namespace snmalloc if (zero_mem == YesZero) { - SharedStateHandle::Backend::Pal::template zero( + SharedStateHandle::Pal::template zero( chunk.unsafe_ptr(), size); } @@ -225,8 +253,7 @@ namespace snmalloc std::cout << "Remote dealloc post" << p << " size " << alloc_size(p) << std::endl; #endif - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(p)); + MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), CapPtr(p), key_global); post_remote_cache(); @@ -252,30 +279,84 @@ namespace snmalloc return local_cache.remote_allocator; } + /** + * SFINAE helper. Matched only if `T` implements `is_initialised`. Calls + * it if it exists. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) + -> decltype(T::is_initialised()) + { + return T::is_initialised(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `is_initialised`. + * Unconditionally returns true if invoked. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, long) + { + return true; + } + + /** + * Call `SharedStateHandle::is_initialised()` if it is implemented, + * unconditionally returns true otherwise. + */ + SNMALLOC_FAST_PATH + bool is_initialised() + { + return call_is_initialised(nullptr, 0); + } + + /** + * SFINAE helper. Matched only if `T` implements `ensure_init`. Calls it + * if it exists. + */ + template + SNMALLOC_FAST_PATH auto call_ensure_init(T*, int) + -> decltype(T::ensure_init()) + { + T::ensure_init(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `ensure_init`. + * Does nothing if called. + */ + template + SNMALLOC_FAST_PATH auto call_ensure_init(T*, long) + {} + + /** + * Call `SharedStateHandle::ensure_init()` if it is implemented, do nothing + * otherwise. + */ + SNMALLOC_FAST_PATH + void ensure_init() + { + call_ensure_init(nullptr, 0); + } + public: - constexpr LocalAllocator() - : handle(SharedStateHandle::get_handle()), - local_cache(&handle.unused_remote) - {} + constexpr LocalAllocator() = default; - LocalAllocator(SharedStateHandle handle) - : handle(handle), local_cache(&handle.unused_remote) - {} - - // This is effectively the constructor for the LocalAllocator, but due to - // not wanting initialisation checks on the fast path, it is initialised - // lazily. - void init() + /** + * Initialise the allocator. For allocators that support local + * initialisation, this is called with a core allocator that this class + * allocates (from a pool allocator) the first time it encounters a slow + * path. If this class is configured without lazy initialisation support + * then this must be called externally + */ + void init(CoreAlloc* c) { // Initialise the global allocator structures - handle.ensure_init(); + ensure_init(); // Should only be called if the allocator has not been initialised. SNMALLOC_ASSERT(core_alloc == nullptr); - // Grab an allocator for this thread. - auto c = Pool::acquire(handle, &(this->local_cache), handle); - // Attach to it. c->attach(&local_cache); core_alloc = c; @@ -286,6 +367,18 @@ namespace snmalloc // local_cache.stats.sta rt(); } + // This is effectively the constructor for the LocalAllocator, but due to + // not wanting initialisation checks on the fast path, it is initialised + // lazily. + void init() + { + // Initialise the global allocator structures + ensure_init(); + // Grab an allocator for this thread. + init(Pool::template acquire( + &(this->local_cache))); + } + // Return all state in the fast allocator and release the underlying // core allocator. This is used during teardown to empty the thread // local state. @@ -303,7 +396,10 @@ namespace snmalloc // Detach underlying allocator core_alloc->attached_cache = nullptr; // Return underlying allocator to the system. - Pool::release(handle, core_alloc); + if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + { + Pool::template release(core_alloc); + } // Set up thread local allocator to look like // it is new to hit slow paths. @@ -311,7 +407,7 @@ namespace snmalloc #ifdef SNMALLOC_TRACING std::cout << "flush(): core_alloc=" << core_alloc << std::endl; #endif - local_cache.remote_allocator = &handle.unused_remote; + local_cache.remote_allocator = &SharedStateHandle::unused_remote; local_cache.remote_dealloc_cache.capacity = 0; } } @@ -365,8 +461,8 @@ namespace snmalloc // before init, that maps null to a remote_deallocator that will never be // in thread local state. - const MetaEntry& entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(p)); + const MetaEntry& entry = + SharedStateHandle::get_meta_data(address_cast(p)); if (likely(local_cache.remote_allocator == entry.get_remote())) { if (likely(CoreAlloc::dealloc_local_object_fast( @@ -376,7 +472,7 @@ namespace snmalloc return; } - if (likely(entry.get_remote() != handle.fake_large_remote)) + if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) { // Check if we have space for the remote deallocation if (local_cache.remote_dealloc_cache.reserve_space(entry)) @@ -416,7 +512,8 @@ namespace snmalloc ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); slab_record->chunk = CapPtr(p); - ChunkAllocator::dealloc(handle, slab_record, slab_sizeclass); + ChunkAllocator::dealloc( + core_alloc->get_backend_local_state(), slab_record, slab_sizeclass); return; } @@ -460,10 +557,9 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a large // object, so we can pull the check for null off the fast path. - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(p_raw)); + MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(p_raw)); - if (likely(entry.get_remote() != handle.fake_large_remote)) + if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) return sizeclass_to_size(entry.get_sizeclass()); // Sizeclass zero is for large is actually zero @@ -484,13 +580,12 @@ namespace snmalloc void* external_pointer(void* p_raw) { // TODO bring back the CHERI bits. Wes to review if required. - if (likely(handle.is_initialised())) + if (likely(is_initialised())) { MetaEntry entry = - SharedStateHandle::Backend::template get_meta_data( - handle.get_backend_state(), address_cast(p_raw)); + SharedStateHandle::template get_meta_data(address_cast(p_raw)); auto sizeclass = entry.get_sizeclass(); - if (likely(entry.get_remote() != handle.fake_large_remote)) + if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) { auto rsize = sizeclass_to_size(sizeclass); auto offset = @@ -533,5 +628,15 @@ namespace snmalloc // We don't know the Start, so return MIN_PTR return nullptr; } + + /** + * Accessor, returns the local cache. If embedding code is allocating the + * core allocator for use by this local allocator then it needs to access + * this field. + */ + LocalCache& get_local_cache() + { + return local_cache; + } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/mem/localcache.h b/src/mem/localcache.h index ae871c9..7e7687f 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -32,7 +32,7 @@ namespace snmalloc auto r = finish_alloc_no_zero(p, sizeclass); if constexpr (zero_mem == YesZero) - SharedStateHandle::Backend::Pal::zero(r, sizeclass_to_size(sizeclass)); + SharedStateHandle::Pal::zero(r, sizeclass_to_size(sizeclass)); // TODO: Should this be zeroing the FreeObject state, in the non-zeroing // case? @@ -72,9 +72,9 @@ namespace snmalloc template< size_t allocator_size, - typename DeallocFun, - typename SharedStateHandle> - bool flush(DeallocFun dealloc, SharedStateHandle handle) + typename SharedStateHandle, + typename DeallocFun> + bool flush(DeallocFun dealloc) { auto& key = entropy.get_free_list_key(); @@ -91,8 +91,8 @@ namespace snmalloc } } - return remote_dealloc_cache.post( - handle, remote_allocator->trunc_id(), key_global); + return remote_dealloc_cache.post( + remote_allocator->trunc_id(), key_global); } template @@ -113,4 +113,4 @@ namespace snmalloc } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 2c88528..ff2da87 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -177,6 +177,16 @@ namespace snmalloc public: constexpr MetaEntry() = default; + /** + * Constructor, provides the remote and sizeclass embedded in a single + * pointer-sized word. This format is not guaranteed to be stable and so + * the second argument of this must always be the return value from + * `get_remote_and_sizeclass`. + */ + MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) + : meta(meta), remote_and_sizeclass(remote_and_sizeclass) + {} + MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass) : meta(meta) { @@ -190,6 +200,17 @@ namespace snmalloc return meta; } + /** + * Return the remote and sizeclass in an implementation-defined encoding. + * This is not guaranteed to be stable across snmalloc releases and so the + * only safe use for this is to pass it to the two-argument constructor of + * this class. + */ + uintptr_t get_remote_and_sizeclass() + { + return remote_and_sizeclass; + } + [[nodiscard]] RemoteAllocator* get_remote() const { return reinterpret_cast( diff --git a/src/mem/pool.h b/src/mem/pool.h index bedbce5..d3c25b4 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -37,9 +37,9 @@ namespace snmalloc { public: template - static T* acquire(SharedStateHandle h, Args&&... args) + static T* acquire(Args&&... args) { - PoolState& pool = h.pool(); + PoolState& pool = SharedStateHandle::pool(); T* p = pool.stack.pop(); if (p != nullptr) @@ -48,12 +48,12 @@ namespace snmalloc return p; } - p = ChunkAllocator::alloc_meta_data( - h, nullptr, std::forward(args)...); + p = ChunkAllocator::alloc_meta_data( + nullptr, std::forward(args)...); if (p == nullptr) { - SharedStateHandle::Backend::Pal::error( + SharedStateHandle::Pal::error( "Failed to initialise thread local allocator."); } @@ -71,21 +71,21 @@ namespace snmalloc * Do not return objects from `extract`. */ template - static void release(SharedStateHandle h, T* p) + static void release(T* p) { // The object's destructor is not run. If the object is "reallocated", it // is returned without the constructor being run, so the object is reused // without re-initialisation. p->reset_in_use(); - h.pool().stack.push(p); + SharedStateHandle::pool().stack.push(p); } template - static T* extract(SharedStateHandle h, T* p = nullptr) + static T* extract(T* p = nullptr) { // Returns a linked list of all objects in the stack, emptying the stack. if (p == nullptr) - return h.pool().stack.pop_all(); + return SharedStateHandle::pool().stack.pop_all(); return p->next; } @@ -96,18 +96,18 @@ namespace snmalloc * Do not return objects from `acquire`. */ template - static void restore(SharedStateHandle h, T* first, T* last) + static void restore(T* first, T* last) { // Pushes a linked list of objects onto the stack. Use to put a linked // list returned by extract back onto the stack. - h.pool().stack.push(first, last); + SharedStateHandle::pool().stack.push(first, last); } template - static T* iterate(SharedStateHandle h, T* p = nullptr) + static T* iterate(T* p = nullptr) { if (p == nullptr) - return h.pool().list; + return SharedStateHandle::pool().list; return p->list_next; } diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 25680b0..a062b9a 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -76,10 +76,7 @@ namespace snmalloc } template - bool post( - SharedStateHandle handle, - RemoteAllocator::alloc_id_t id, - const FreeListKey& key) + bool post(RemoteAllocator::alloc_id_t id, const FreeListKey& key) { SNMALLOC_ASSERT(initialised); size_t post_round = 0; @@ -97,8 +94,8 @@ namespace snmalloc if (!list[i].empty()) { auto [first, last] = list[i].extract_segment(key); - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(first)); + MetaEntry entry = + SharedStateHandle::get_meta_data(address_cast(first)); entry.get_remote()->enqueue(first, last, key); sent_something = true; } @@ -120,8 +117,7 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key); - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(r)); + MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(r)); auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); list[slot].add(r, key); diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 30b3daf..13dc161 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -75,14 +75,14 @@ namespace snmalloc public: template static std::pair, Metaslab*> alloc_chunk( - SharedStateHandle h, - typename SharedStateHandle::Backend::LocalState& backend_state, + typename SharedStateHandle::LocalState& local_state, sizeclass_t sizeclass, sizeclass_t slab_sizeclass, // TODO sizeclass_t size_t slab_size, RemoteAllocator* remote) { - ChunkAllocatorState& state = h.get_slab_allocator_state(); + ChunkAllocatorState& state = + SharedStateHandle::get_slab_allocator_state(&local_state); // Pop a slab auto chunk_record = state.chunk_stack[slab_sizeclass].pop(); @@ -98,15 +98,14 @@ namespace snmalloc << std::endl; #endif MetaEntry entry{meta, remote, sizeclass}; - SharedStateHandle::Backend::set_meta_data( - h.get_backend_state(), address_cast(slab), slab_size, entry); + SharedStateHandle::set_meta_data(address_cast(slab), slab_size, entry); return {slab, meta}; } // Allocate a fresh slab as there are no available ones. // First create meta-data - auto [slab, meta] = SharedStateHandle::Backend::alloc_chunk( - h.get_backend_state(), &backend_state, slab_size, remote, sizeclass); + auto [slab, meta] = SharedStateHandle::alloc_chunk( + &local_state, slab_size, remote, sizeclass); #ifdef SNMALLOC_TRACING std::cout << "Create slab:" << slab.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " << slab_size << std::endl; @@ -122,10 +121,12 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH static void - dealloc(SharedStateHandle h, ChunkRecord* p, size_t slab_sizeclass) + SNMALLOC_SLOW_PATH static void dealloc( + typename SharedStateHandle::LocalState& local_state, + ChunkRecord* p, + size_t slab_sizeclass) { - auto& state = h.get_slab_allocator_state(); + auto& state = SharedStateHandle::get_slab_allocator_state(&local_state); #ifdef SNMALLOC_TRACING std::cout << "Return slab:" << p->chunk.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " @@ -144,15 +145,13 @@ namespace snmalloc */ template static U* alloc_meta_data( - SharedStateHandle h, - typename SharedStateHandle::Backend::LocalState* local_state, - Args&&... args) + typename SharedStateHandle::LocalState* local_state, Args&&... args) { // Cache line align size_t size = bits::align_up(sizeof(U), 64); - CapPtr p = SharedStateHandle::Backend::alloc_meta_data( - h.get_backend_state(), local_state, size); + CapPtr p = + SharedStateHandle::template alloc_meta_data(local_state, size); if (p == nullptr) return nullptr; diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 101e092..dc9cee6 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -156,6 +156,6 @@ namespace snmalloc */ void _malloc_thread_cleanup() { - ThreadAlloc::get().teardown(); + snmalloc::ThreadAlloc::get().teardown(); } #endif diff --git a/src/override/malloc-extensions.cc b/src/override/malloc-extensions.cc index 9245cc9..d672e05 100644 --- a/src/override/malloc-extensions.cc +++ b/src/override/malloc-extensions.cc @@ -6,10 +6,8 @@ using namespace snmalloc; void get_malloc_info_v1(malloc_info_v1* stats) { - auto unused_chunks = - Globals::get_handle().get_slab_allocator_state().unused_memory(); - auto peak = - Globals::get_handle().get_slab_allocator_state().peak_memory_usage(); + auto unused_chunks = Globals::get_slab_allocator_state().unused_memory(); + auto peak = Globals::get_slab_allocator_state().peak_memory_usage(); stats->current_memory_usage = peak - unused_chunks; stats->peak_memory_usage = peak; -} \ No newline at end of file +} diff --git a/src/override/malloc.cc b/src/override/malloc.cc index acd6402..0a9ad36 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -2,7 +2,7 @@ #include "../snmalloc_core.h" #ifndef SNMALLOC_PROVIDE_OWN_CONFIG -# include "../mem/globalconfig.h" +# include "../backend/globalconfig.h" // The default configuration for snmalloc is used if alternative not defined namespace snmalloc { diff --git a/src/snmalloc.h b/src/snmalloc.h index 6abd703..e1d90d4 100644 --- a/src/snmalloc.h +++ b/src/snmalloc.h @@ -3,8 +3,13 @@ // Core implementation of snmalloc independent of the configuration mode #include "snmalloc_core.h" +// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own +// definition of `snmalloc::Alloc` and include `snmalloc_front.h` before +// including any files that include `snmalloc.h` and consume the global +// allocation APIs. +#ifndef SNMALLOC_PROVIDE_OWN_CONFIG // Default implementation of global state -#include "mem/globalconfig.h" +# include "backend/globalconfig.h" // The default configuration for snmalloc namespace snmalloc @@ -13,4 +18,5 @@ namespace snmalloc } // User facing API surface, needs to know what `Alloc` is. -#include "snmalloc_front.h" \ No newline at end of file +# include "snmalloc_front.h" +#endif diff --git a/src/snmalloc_core.h b/src/snmalloc_core.h index 0c6a2d3..d8b14aa 100644 --- a/src/snmalloc_core.h +++ b/src/snmalloc_core.h @@ -1,3 +1,6 @@ #pragma once -#include "mem/globalalloc.h" \ No newline at end of file +#include "backend/address_space.h" +#include "backend/commonconfig.h" +#include "backend/pagemap.h" +#include "mem/globalalloc.h" diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 1f0ca0b..db75ccf 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -1,4 +1,4 @@ -#include "mem/fixedglobalconfig.h" +#include "backend/fixedglobalconfig.h" #include "test/setup.h" #include @@ -29,9 +29,8 @@ int main() std::cout << "Allocated region " << oe_base << " - " << pointer_offset(oe_base, size) << std::endl; - CustomGlobals fixed_handle; CustomGlobals::init(oe_base, size); - FixedAlloc a(fixed_handle); + FixedAlloc a; size_t object_size = 128; size_t count = 0; diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index e87e0ee..c8fc74a 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -232,6 +232,6 @@ int main(int argc, char** argv) abort(); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); return 0; } diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 9c6cd35..f4afd8f 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -183,7 +183,7 @@ void test_calloc() alloc.dealloc(p, size); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } void test_double_alloc() @@ -228,7 +228,7 @@ void test_double_alloc() } } } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } void test_external_pointer() @@ -264,7 +264,7 @@ void test_external_pointer() alloc.dealloc(p1, size); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); }; void check_offset(void* base, void* interior) diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index 36b4788..156612c 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -8,7 +8,7 @@ int main() auto r = a.alloc(16); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != false) { abort(); @@ -16,7 +16,7 @@ int main() a.dealloc(r); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != true) { abort(); @@ -24,7 +24,7 @@ int main() r = a.alloc(16); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != false) { abort(); @@ -32,10 +32,10 @@ int main() a.dealloc(r); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != true) { abort(); } #endif -} \ No newline at end of file +} diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index a5b1fe8..effb62f 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -4,7 +4,7 @@ // Specify using own #define SNMALLOC_EXTERNAL_THREAD_ALLOC -#include "mem/globalconfig.h" +#include "backend/globalconfig.h" namespace snmalloc { diff --git a/src/test/func/two_alloc_types/alloc1.cc b/src/test/func/two_alloc_types/alloc1.cc index 4e5f9d0..feb47bd 100644 --- a/src/test/func/two_alloc_types/alloc1.cc +++ b/src/test/func/two_alloc_types/alloc1.cc @@ -3,7 +3,7 @@ // Redefine the namespace, so we can have two versions. #define snmalloc snmalloc_enclave -#include +#include #include // Specify type of allocator @@ -19,6 +19,5 @@ namespace snmalloc extern "C" void oe_allocator_init(void* base, void* end) { - snmalloc::CustomGlobals fixed_handle; - fixed_handle.init(base, address_cast(end) - address_cast(base)); + snmalloc::CustomGlobals::init(base, address_cast(end) - address_cast(base)); } diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 5be5641..74f9cd2 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -142,7 +142,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) } #ifndef NDEBUG - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); #endif }; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index ac66a13..90aa1ba 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -47,7 +47,7 @@ namespace test alloc.dealloc(objects[i]); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } void test_external_pointer(xoroshiro::p128r64& r) diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index 2bf4980..cd15a12 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -60,7 +60,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) } } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } int main(int, char**)