Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Snmalloc2 API cleanups for sandbox use. (#359)

Browse Source 
This is the set of changes required for snmalloc2 to be usable by the
process sandboxing code and incorporates some API changes that reduce
the amount of code required to embed snmalloc.  Highlights:

 - Merge the config and back-end classes.
 - Everything in config is now global (all methods are static)
 - The GlobalState class is gone (all global state is managed by global
   methods on the config class)
 - LocalState is now a member of the config class, all methods are
   instance methods.
 - Not every configuration needs to use the lazy initialisation hooks.
   They now need to be provided only if they are used.  If the
   configuration does not provide an `ensure_init` method, it is not
   called.  If it does not provide an `is_initialised` method then the
   global initialisation state is not checked.
 - There is now an `snmalloc::Options` class that default initialises
   itself to the default behaviour.  Every configuration must provide a
   `constexpr` instance of this class.  Each flag can be separately
   overridden and new flags can be added without breaking any existing
   API consumers.

The config classes are moved into the backend directory.
This commit is contained in:
David Chisnall
2021-08-05 15:08:12 +01:00
committed by GitHub
parent 2ef1eea3ba
commit e8374479f4
29 changed files with 676 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/backend/address_space_core.h
View File

@@ -1,6 +1,8 @@
#pragma once
#include "../ds/address.h"
#include "../ds/flaglock.h"
#include "../mem/allocconfig.h"
#include "../mem/metaslab.h"
#include "../pal/pal.h"
#include <array>

111
src/backend/backend.h
View File

@@ -3,6 +3,7 @@
#include "../mem/metaslab.h"
#include "../pal/pal.h"
#include "address_space.h"
#include "commonconfig.h"
#include "pagemap.h"
namespace snmalloc
@@ -15,7 +16,7 @@ namespace snmalloc
SNMALLOC_CONCEPT(ConceptPAL) PAL,
bool fixed_range,
typename PageMapEntry = MetaEntry>
class BackendAllocator
class BackendAllocator : public CommonConfig
{
// Size of local address space requests. Currently aimed at 2MiB large
// pages but should make this configurable (i.e. for OE, so we don't need as
@@ -54,55 +55,31 @@ namespace snmalloc
#endif
};
/**
* Global state for the backend allocator
*
* This contains the various global datastructures required to store
* meta-data for each chunk of memory, and to provide well aligned chunks
* of memory.
*
* This type is required by snmalloc to exist as part of the Backend.
*/
class GlobalState
SNMALLOC_REQUIRE_CONSTINIT
static inline AddressSpaceManager<PAL> address_space;
SNMALLOC_REQUIRE_CONSTINIT
static inline FlatPagemap<MIN_CHUNK_BITS, PageMapEntry, PAL, fixed_range>
pagemap;
public:
template<bool fixed_range_ = fixed_range>
static std::enable_if_t<!fixed_range_> init()
{
friend BackendAllocator;
static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!");
protected:
AddressSpaceManager<PAL> address_space;
pagemap.init();
}
FlatPagemap<MIN_CHUNK_BITS, PageMapEntry, PAL, fixed_range> pagemap;
template<bool fixed_range_ = fixed_range>
static std::enable_if_t<fixed_range_> init(void* base, size_t length)
{
static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!");
public:
template<bool fixed_range_ = fixed_range>
std::enable_if_t<!fixed_range_> init()
{
static_assert(
fixed_range_ == fixed_range, "Don't set SFINAE parameter!");
pagemap.init();
if constexpr (fixed_range)
{
abort();
}
}
template<bool fixed_range_ = fixed_range>
std::enable_if_t<fixed_range_> init(void* base, size_t length)
{
static_assert(
fixed_range_ == fixed_range, "Don't set SFINAE parameter!");
auto [heap_base, heap_length] = pagemap.init(base, length);
address_space.add_range(
CapPtr<void, CBChunk>(heap_base), heap_length, pagemap);
if constexpr (!fixed_range)
{
abort();
}
}
};
auto [heap_base, heap_length] = pagemap.init(base, length);
address_space.add_range(
CapPtr<void, CBChunk>(heap_base), heap_length, pagemap);
}
private:
#ifdef SNMALLOC_CHECK_CLIENT
@@ -138,8 +115,7 @@ namespace snmalloc
* space managers.
*/
template<bool is_meta>
static CapPtr<void, CBChunk>
reserve(GlobalState& h, LocalState* local_state, size_t size)
static CapPtr<void, CBChunk> reserve(LocalState* local_state, size_t size)
{
#ifdef SNMALLOC_CHECK_CLIENT
constexpr auto MAX_CACHED_SIZE =
@@ -148,7 +124,7 @@ namespace snmalloc
constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK;
#endif
auto& global = h.address_space;
auto& global = address_space;
CapPtr<void, CBChunk> p;
if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE))
@@ -160,14 +136,14 @@ namespace snmalloc
auto& local = local_state->local_address_space;
#endif
p = local.template reserve_with_left_over<PAL>(size, h.pagemap);
p = local.template reserve_with_left_over<PAL>(size, pagemap);
if (p != nullptr)
{
return p;
}
auto refill_size = LOCAL_CACHE_BLOCK;
auto refill = global.template reserve<false>(refill_size, h.pagemap);
auto refill = global.template reserve<false>(refill_size, pagemap);
if (refill == nullptr)
return nullptr;
@@ -179,10 +155,10 @@ namespace snmalloc
}
#endif
PAL::template notify_using<NoZero>(refill.unsafe_ptr(), refill_size);
local.template add_range<PAL>(refill, refill_size, h.pagemap);
local.template add_range<PAL>(refill, refill_size, pagemap);
// This should succeed
return local.template reserve_with_left_over<PAL>(size, h.pagemap);
return local.template reserve_with_left_over<PAL>(size, pagemap);
}
#ifdef SNMALLOC_CHECK_CLIENT
@@ -193,7 +169,7 @@ namespace snmalloc
size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size));
size_t size_request = rsize * 64;
p = global.template reserve<false>(size_request, h.pagemap);
p = global.template reserve<false>(size_request, pagemap);
if (p == nullptr)
return nullptr;
@@ -209,7 +185,7 @@ namespace snmalloc
SNMALLOC_ASSERT(!is_meta);
#endif
p = global.template reserve_with_left_over<true>(size, h.pagemap);
p = global.template reserve_with_left_over<true>(size, pagemap);
return p;
}
@@ -219,11 +195,16 @@ namespace snmalloc
*
* Backend allocator may use guard pages and separate area of
* address space to protect this from corruption.
*
* The template argument is the type of the metadata being allocated. This
* allows the backend to allocate different types of metadata in different
* places or with different policies.
*/
template<typename T>
static CapPtr<void, CBChunk>
alloc_meta_data(GlobalState& h, LocalState* local_state, size_t size)
alloc_meta_data(LocalState* local_state, size_t size)
{
return reserve<true>(h, local_state, size);
return reserve<true>(local_state, size);
}
/**
@@ -236,7 +217,6 @@ namespace snmalloc
* where metaslab, is the second element of the pair return.
*/
static std::pair<CapPtr<void, CBChunk>, Metaslab*> alloc_chunk(
GlobalState& h,
LocalState* local_state,
size_t size,
RemoteAllocator* remote,
@@ -246,12 +226,12 @@ namespace snmalloc
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
auto meta = reinterpret_cast<Metaslab*>(
reserve<true>(h, local_state, sizeof(Metaslab)).unsafe_ptr());
reserve<true>(local_state, sizeof(Metaslab)).unsafe_ptr());
if (meta == nullptr)
return {nullptr, nullptr};
CapPtr<void, CBChunk> p = reserve<false>(h, local_state, size);
CapPtr<void, CBChunk> p = reserve<false>(local_state, size);
#ifdef SNMALLOC_TRACING
std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")"
@@ -274,7 +254,7 @@ namespace snmalloc
a < address_cast(pointer_offset(p, size));
a += MIN_CHUNK_SIZE)
{
h.pagemap.set(a, t);
pagemap.set(a, t);
}
return {p, meta};
}
@@ -286,20 +266,19 @@ namespace snmalloc
* to access a location that is not backed by a chunk.
*/
template<bool potentially_out_of_range = false>
static const MetaEntry& get_meta_data(GlobalState& h, address_t p)
static const MetaEntry& get_meta_data(address_t p)
{
return h.pagemap.template get<potentially_out_of_range>(p);
return pagemap.template get<potentially_out_of_range>(p);
}
/**
* Set the metadata associated with a chunk.
*/
static void
set_meta_data(GlobalState& h, address_t p, size_t size, MetaEntry t)
static void set_meta_data(address_t p, size_t size, MetaEntry t)
{
for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE)
{
h.pagemap.set(a, t);
pagemap.set(a, t);
}
}
};

118
src/backend/commonconfig.h Normal file
View File

@@ -0,0 +1,118 @@
#pragma once
#include "../ds/defines.h"
#include "../mem/remotecache.h"
namespace snmalloc
{
// Forward reference to thread local cleanup.
void register_clean_up();
/**
* Options for a specific snmalloc configuration. Every globals object must
* have one `constexpr` instance of this class called `Options`. This should
* be constructed to explicitly override any of the defaults. A
* configuration that does not need to override anything would simply declare
* this as a field of the global object:
*
* ```c++
* constexpr static snmalloc::Flags Options{};
* ```
*
* A global configuration that wished to use out-of-line message queues but
* accept the defaults for everything else would instead do this:
*
* ```c++
* constexpr static snmalloc::Flags Options{.IsQueueInline = false};
* ```
*
* To maintain backwards source compatibility in future versions, any new
* option added here should have its default set to be whatever snmalloc was
* doing before the new option was added.
*/
struct Flags
{
/**
* Should allocators have inline message queues? If this is true then
* the `CoreAllocator` is responsible for allocating the
* `RemoteAllocator` that contains its message queue. If this is false
* then the `RemoteAllocator` must be separately allocated and provided
* to the `CoreAllocator` before it is used.
*
* Setting this to `false` currently requires also setting
* `LocalAllocSupportsLazyInit` to false so that the `CoreAllocator` can
* be provided to the `LocalAllocator` fully initialised but in the
* future it may be possible to allocate the `RemoteAllocator` via
* `alloc_meta_data` or a similar API in the back end.
*/
bool IsQueueInline = true;
/**
* Does the `CoreAllocator` own a `Backend::LocalState` object? If this is
* true then the `CoreAllocator` is responsible for allocating and
* deallocating a local state object, otherwise the surrounding code is
* responsible for creating it.
*
* Use cases that set this to false will probably also need to set
* `LocalAllocSupportsLazyInit` to false so that they can provide the local
* state explicitly during allocator creation.
*/
bool CoreAllocOwnsLocalState = true;
/**
* Are `CoreAllocator` allocated by the pool allocator? If not then the
* code embedding this snmalloc configuration is responsible for allocating
* `CoreAllocator` instances.
*
* Users setting this flag must also set `LocalAllocSupportsLazyInit` to
* false currently because there is no alternative mechanism for allocating
* core allocators. This may change in future versions.
*/
bool CoreAllocIsPoolAllocated = true;
/**
* Do `LocalAllocator` instances in this configuration support lazy
* initialisation? If so, then the first exit from a fast path will
* trigger allocation of a `CoreAllocator` and associated state. If not
* then the code embedding this configuration of snmalloc is responsible
* for allocating core allocators.
*/
bool LocalAllocSupportsLazyInit = true;
};
/**
* Class containing definitions that are likely to be used by all except for
* the most unusual back-end implementations. This can be subclassed as a
* convenience for back-end implementers, but is not required.
*/
class CommonConfig
{
public:
/**
* Special remote that should never be used as a real remote.
* This is used to initialise allocators that should always hit the
* remote path for deallocation. Hence moving a branch off the critical
* path.
*/
SNMALLOC_REQUIRE_CONSTINIT
inline static RemoteAllocator unused_remote;
/**
* Special remote that is used in meta-data for large allocations.
*
* nullptr is considered a large allocations for this purpose to move
* of the critical path.
*
* Bottom bits of the remote pointer are used for a sizeclass, we need
* size bits to represent the non-large sizeclasses, we can then get
* the large sizeclass by having the fake large_remote considerably
* more aligned.
*/
SNMALLOC_REQUIRE_CONSTINIT
inline static constexpr RemoteAllocator* fake_large_remote{nullptr};
static_assert(
&unused_remote != fake_large_remote,
"Compilation should ensure these are different");
};
} // namespace snmalloc

42
src/mem/fixedglobalconfig.h → src/backend/fixedglobalconfig.h
View File

@@ -12,50 +12,27 @@ namespace snmalloc
* A single fixed address range allocator configuration
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
class FixedGlobals : public CommonConfig
class FixedGlobals final : public BackendAllocator<PAL, true>
{
public:
using Backend = BackendAllocator<PAL, true>;
private:
inline static typename Backend::GlobalState backend_state;
using Backend = BackendAllocator<PAL, true>;
inline static ChunkAllocatorState slab_allocator_state;
inline static PoolState<CoreAllocator<FixedGlobals>> alloc_pool;
public:
static typename Backend::GlobalState& get_backend_state()
{
return backend_state;
}
ChunkAllocatorState& get_slab_allocator_state()
static ChunkAllocatorState&
get_slab_allocator_state(typename Backend::LocalState*)
{
return slab_allocator_state;
}
PoolState<CoreAllocator<FixedGlobals>>& pool()
static PoolState<CoreAllocator<FixedGlobals>>& pool()
{
return alloc_pool;
}
static constexpr bool IsQueueInline = true;
// Performs initialisation for this configuration
// of allocators. Will be called at most once
// before any other datastructures are accessed.
void ensure_init() noexcept
{
#ifdef SNMALLOC_TRACING
std::cout << "Run init_impl" << std::endl;
#endif
}
static bool is_initialised()
{
return true;
}
static constexpr Flags Options{};
// This needs to be a forward reference as the
// thread local state will need to know about this.
@@ -68,12 +45,7 @@ namespace snmalloc
static void init(void* base, size_t length)
{
get_backend_state().init(base, length);
}
constexpr static FixedGlobals get_handle()
{
return {};
Backend::init(base, length);
}
};
}

41
src/mem/globalconfig.h → src/backend/globalconfig.h
View File

@@ -23,15 +23,15 @@ namespace snmalloc
}
#endif
class Globals : public CommonConfig
/**
* The default configuration for a global snmalloc. This allocates memory
* from the operating system and expects to manage memory anywhere in the
* address space.
*/
class Globals final : public BackendAllocator<Pal, false>
{
public:
using Backend = BackendAllocator<Pal, false>;
private:
SNMALLOC_REQUIRE_CONSTINIT
inline static Backend::GlobalState backend_state;
using Backend = BackendAllocator<Pal, false>;
SNMALLOC_REQUIRE_CONSTINIT
inline static ChunkAllocatorState slab_allocator_state;
@@ -45,27 +45,23 @@ namespace snmalloc
inline static std::atomic_flag initialisation_lock{};
public:
Backend::GlobalState& get_backend_state()
{
return backend_state;
}
ChunkAllocatorState& get_slab_allocator_state()
static ChunkAllocatorState&
get_slab_allocator_state(Backend::LocalState* = nullptr)
{
return slab_allocator_state;
}
PoolState<CoreAllocator<Globals>>& pool()
static PoolState<CoreAllocator<Globals>>& pool()
{
return alloc_pool;
}
static constexpr bool IsQueueInline = true;
static constexpr Flags Options{};
// Performs initialisation for this configuration
// of allocators. Needs to be idempotent,
// and concurrency safe.
void ensure_init()
static void ensure_init()
{
FlagLock lock{initialisation_lock};
#ifdef SNMALLOC_TRACING
@@ -81,7 +77,7 @@ namespace snmalloc
key_global = FreeListKey(entropy.get_free_list_key());
// Need to initialise pagemap.
backend_state.init();
Backend::init();
#ifdef USE_SNMALLOC_STATS
atexit(snmalloc::print_stats);
@@ -90,7 +86,7 @@ namespace snmalloc
initialised = true;
}
bool is_initialised()
static bool is_initialised()
{
return initialised;
}
@@ -99,16 +95,9 @@ namespace snmalloc
// thread local state will need to know about this.
// This may allocate, so should only be called once
// a thread local allocator is available.
void register_clean_up()
static void register_clean_up()
{
snmalloc::register_clean_up();
}
// This is an empty structure as all the state is global
// for this allocator configuration.
static constexpr Globals get_handle()
{
return {};
}
};
} // namespace snmalloc

36
src/backend/pagemap.h
View File

@@ -66,6 +66,36 @@ namespace snmalloc
constexpr FlatPagemap() = default;
/**
* For pagemaps that cover an entire fixed address space, return the size
* that they must be. This allows the caller to allocate the correct
* amount of memory to be passed to `init`. This is not available for
* fixed-range pagemaps, whose size depends on dynamic configuration.
*/
template<bool has_bounds_ = has_bounds>
static constexpr std::enable_if_t<!has_bounds_, size_t> required_size()
{
static_assert(
has_bounds_ == has_bounds, "Don't set SFINAE template parameter!");
constexpr size_t COVERED_BITS = bits::ADDRESS_BITS - GRANULARITY_BITS;
constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS);
return ENTRIES * sizeof(T);
}
/**
* Initialise with pre-allocated memory.
*
* This is currently disabled for bounded pagemaps but may be reenabled if
* `required_size` is enabled for the has-bounds case.
*/
template<bool has_bounds_ = has_bounds>
std::enable_if_t<!has_bounds_> init(T* address)
{
static_assert(
has_bounds_ == has_bounds, "Don't set SFINAE template parameter!");
body = address;
}
/**
* Initialise the pagemap with bounds.
*
@@ -117,11 +147,7 @@ namespace snmalloc
{
static_assert(
has_bounds_ == has_bounds, "Don't set SFINAE template parameter!");
static constexpr size_t COVERED_BITS =
bits::ADDRESS_BITS - GRANULARITY_BITS;
static constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS);
static constexpr size_t REQUIRED_SIZE = ENTRIES * sizeof(T);
static constexpr size_t REQUIRED_SIZE = required_size();
#ifdef SNMALLOC_CHECK_CLIENT
// Allocate a power of two extra to allow the placement of the

41
src/mem/commonconfig.h
View File

@@ -1,41 +0,0 @@
#pragma once
#include "../ds/defines.h"
#include "remotecache.h"
namespace snmalloc
{
// Forward reference to thread local cleanup.
void register_clean_up();
class CommonConfig
{
public:
/**
* Special remote that should never be used as a real remote.
* This is used to initialise allocators that should always hit the
* remote path for deallocation. Hence moving a branch off the critical
* path.
*/
SNMALLOC_REQUIRE_CONSTINIT
inline static RemoteAllocator unused_remote;
/**
* Special remote that is used in meta-data for large allocations.
*
* nullptr is considered a large allocations for this purpose to move
* of the critical path.
*
* Bottom bits of the remote pointer are used for a sizeclass, we need
* size bits to represent the non-large sizeclasses, we can then get
* the large sizeclass by having the fake large_remote considerably
* more aligned.
*/
SNMALLOC_REQUIRE_CONSTINIT
inline static constexpr RemoteAllocator* fake_large_remote{nullptr};
static_assert(
&unused_remote != fake_large_remote,
"Compilation should ensure these are different");
};
} // namespace snmalloc

181
src/mem/corealloc.h
View File

@@ -11,10 +11,45 @@
namespace snmalloc
{
/**
* Empty class used as the superclass for `CoreAllocator` when it does not
* opt into pool allocation. This class exists because `std::conditional`
* (or other equivalent features in C++) can choose between options for
* superclasses but they cannot choose whether a class has a superclass.
* Setting the superclass to an empty class is equivalent to no superclass.
*/
class NotPoolAllocated
{};
/**
* The core, stateful, part of a memory allocator. Each `LocalAllocator`
* owns one `CoreAllocator` once it is initialised.
*
* The template parameter provides all of the global configuration for this
* instantiation of snmalloc. This includes three options that apply to this
* class:
*
* - `CoreAllocIsPoolAllocated` defines whether this `CoreAlloc`
* configuration should support pool allocation. This defaults to true but
* a configuration that allocates allocators eagerly may opt out.
* - `CoreAllocOwnsLocalState` defines whether the `CoreAllocator` owns the
* associated `LocalState` object. If this is true (the default) then
* `CoreAllocator` embeds the LocalState object. If this is set to false
* then a `LocalState` object must be provided to the constructor. This
* allows external code to provide explicit configuration of the address
* range managed by this object.
* - `IsQueueInline` (defaults to true) defines whether the message queue
* (`RemoteAllocator`) for this class is inline or provided externally. If
* provided externally, then it must be set explicitly with
* `init_message_queue`.
*/
template<typename SharedStateHandle>
class CoreAllocator : public Pooled<CoreAllocator<SharedStateHandle>>
class CoreAllocator : public std::conditional_t<
SharedStateHandle::Options.CoreAllocIsPoolAllocated,
Pooled<CoreAllocator<SharedStateHandle>>,
NotPoolAllocated>
{
template<class SharedStateHandle2>
template<typename SharedStateHandle2>
friend class LocalAllocator;
/**
@@ -33,16 +68,27 @@ namespace snmalloc
* allocator
*/
std::conditional_t<
SharedStateHandle::IsQueueInline,
SharedStateHandle::Options.IsQueueInline,
RemoteAllocator,
RemoteAllocator*>
remote_alloc;
/**
* A local area of address space managed by this allocator.
* Used to reduce calls on the global address space.
* The type used local state. This is defined by the back end.
*/
typename SharedStateHandle::Backend::LocalState backend_state;
using LocalState = typename SharedStateHandle::LocalState;
/**
* A local area of address space managed by this allocator.
* Used to reduce calls on the global address space. This is inline if the
* core allocator owns the local state or indirect if it is owned
* externally.
*/
std::conditional_t<
SharedStateHandle::Options.CoreAllocOwnsLocalState,
LocalState,
LocalState*>
backend_state;
/**
* This is the thread local structure associated to this
@@ -50,12 +96,6 @@ namespace snmalloc
*/
LocalCache* attached_cache;
/**
* This contains the way to access all the global state and
* configuration for the system setup.
*/
SharedStateHandle handle;
/**
* The message queue needs to be accessible from other threads
*
@@ -64,7 +104,7 @@ namespace snmalloc
*/
auto* public_state()
{
if constexpr (SharedStateHandle::IsQueueInline)
if constexpr (SharedStateHandle::Options.IsQueueInline)
{
return &remote_alloc;
}
@@ -260,8 +300,10 @@ namespace snmalloc
// TODO delay the clear to the next user of the slab, or teardown so
// don't touch the cache lines at this point in check_client.
auto chunk_record = clear_slab(meta, sizeclass);
ChunkAllocator::dealloc(
handle, chunk_record, sizeclass_to_slab_sizeclass(sizeclass));
ChunkAllocator::dealloc<SharedStateHandle>(
get_backend_local_state(),
chunk_record,
sizeclass_to_slab_sizeclass(sizeclass));
}
else
{
@@ -335,8 +377,8 @@ namespace snmalloc
for (size_t i = 0; i < REMOTE_BATCH; i++)
{
auto p = message_queue().peek();
auto& entry = SharedStateHandle::Backend::get_meta_data(
handle.get_backend_state(), snmalloc::address_cast(p));
auto& entry =
SharedStateHandle::get_meta_data(snmalloc::address_cast(p));
auto r = message_queue().dequeue(key_global);
@@ -388,9 +430,11 @@ namespace snmalloc
}
}
public:
CoreAllocator(LocalCache* cache, SharedStateHandle handle)
: attached_cache(cache), handle(handle)
/**
* Initialiser, shared code between the constructors for different
* configurations.
*/
void init()
{
#ifdef SNMALLOC_TRACING
std::cout << "Making an allocator." << std::endl;
@@ -398,13 +442,16 @@ namespace snmalloc
// Entropy must be first, so that all data-structures can use the key
// it generates.
// This must occur before any freelists are constructed.
entropy.init<typename SharedStateHandle::Backend::Pal>();
entropy.init<typename SharedStateHandle::Pal>();
// Ignoring stats for now.
// stats().start();
init_message_queue();
message_queue().invariant();
if constexpr (SharedStateHandle::Options.IsQueueInline)
{
init_message_queue();
message_queue().invariant();
}
#ifndef NDEBUG
for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++)
@@ -423,6 +470,44 @@ namespace snmalloc
#endif
}
public:
/**
* Constructor for the case that the core allocator owns the local state.
* SFINAE disabled if the allocator does not own the local state.
*/
template<
typename Config = SharedStateHandle,
typename = std::enable_if_t<Config::Options.CoreAllocOwnsLocalState>>
CoreAllocator(LocalCache* cache) : attached_cache(cache)
{
init();
}
/**
* Constructor for the case that the core allocator does not owns the local
* state. SFINAE disabled if the allocator does own the local state.
*/
template<
typename Config = SharedStateHandle,
typename = std::enable_if_t<!Config::Options.CoreAllocOwnsLocalState>>
CoreAllocator(LocalCache* cache, LocalState* backend = nullptr)
: backend_state(backend), attached_cache(cache)
{
init();
}
/**
* If the message queue is not inline, provide it. This will then
* configure the message queue for use.
*/
template<bool InlineQueue = SharedStateHandle::Options.IsQueueInline>
std::enable_if_t<!InlineQueue> init_message_queue(RemoteAllocator* q)
{
remote_alloc = q;
init_message_queue();
message_queue().invariant();
}
/**
* Post deallocations onto other threads.
*
@@ -432,9 +517,9 @@ namespace snmalloc
SNMALLOC_FAST_PATH bool post()
{
// stats().remote_post(); // TODO queue not in line!
bool sent_something =
attached_cache->remote_dealloc_cache.post<sizeof(CoreAllocator)>(
handle, public_state()->trunc_id(), key_global);
bool sent_something = attached_cache->remote_dealloc_cache
.post<sizeof(CoreAllocator), SharedStateHandle>(
public_state()->trunc_id(), key_global);
return sent_something;
}
@@ -454,8 +539,7 @@ namespace snmalloc
SNMALLOC_FAST_PATH void dealloc_local_object(void* p)
{
auto entry = SharedStateHandle::Backend::get_meta_data(
handle.get_backend_state(), snmalloc::address_cast(p));
auto entry = SharedStateHandle::get_meta_data(snmalloc::address_cast(p));
if (likely(dealloc_local_object_fast(entry, p, entropy)))
return;
@@ -506,6 +590,24 @@ namespace snmalloc
return small_alloc_slow<zero_mem>(sizeclass, fast_free_list, rsize);
}
/**
* Accessor for the local state. This hides whether the local state is
* stored inline or provided externally from the rest of the code.
*/
SNMALLOC_FAST_PATH
LocalState& get_backend_local_state()
{
if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState)
{
return backend_state;
}
else
{
SNMALLOC_ASSERT(backend_state);
return *backend_state;
}
}
template<ZeroMem zero_mem>
SNMALLOC_SLOW_PATH void* small_alloc_slow(
sizeclass_t sizeclass, FreeListIter& fast_free_list, size_t rsize)
@@ -519,13 +621,13 @@ namespace snmalloc
std::cout << "slab size " << slab_size << std::endl;
#endif
auto [slab, meta] = snmalloc::ChunkAllocator::alloc_chunk(
handle,
backend_state,
sizeclass,
slab_sizeclass,
slab_size,
public_state());
auto [slab, meta] =
snmalloc::ChunkAllocator::alloc_chunk<SharedStateHandle>(
get_backend_local_state(),
sizeclass,
slab_sizeclass,
slab_size,
public_state());
if (slab == nullptr)
{
@@ -563,8 +665,8 @@ namespace snmalloc
{
bool need_post = true; // Always going to post, so ignore.
auto n = p->atomic_read_next(key_global);
auto& entry = SharedStateHandle::Backend::get_meta_data(
handle.get_backend_state(), snmalloc::address_cast(p));
auto& entry =
SharedStateHandle::get_meta_data(snmalloc::address_cast(p));
handle_dealloc_remote(entry, p, need_post);
p = n;
}
@@ -577,8 +679,9 @@ namespace snmalloc
handle_message_queue([]() {});
}
auto posted = attached_cache->flush<sizeof(CoreAllocator)>(
[&](auto p) { dealloc_local_object(p); }, handle);
auto posted =
attached_cache->flush<sizeof(CoreAllocator), SharedStateHandle>(
[&](auto p) { dealloc_local_object(p); });
// We may now have unused slabs, return to the global allocator.
for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++)

72
src/mem/globalalloc.h
View File

@@ -6,9 +6,13 @@
namespace snmalloc
{
template<class SharedStateHandle>
inline static void aggregate_stats(SharedStateHandle handle, Stats& stats)
inline static void aggregate_stats(Stats& stats)
{
auto* alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle);
static_assert(
SharedStateHandle::Options.CoreAllocIsPoolAllocated,
"Global statistics are available only for pool-allocated configurations");
auto* alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>();
while (alloc != nullptr)
{
@@ -16,45 +20,51 @@ namespace snmalloc
if (a != nullptr)
stats.add(*a);
stats.add(alloc->stats());
alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle, alloc);
alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>(alloc);
}
}
#ifdef USE_SNMALLOC_STATS
template<class SharedStateHandle>
inline static void print_all_stats(
SharedStateHandle handle, std::ostream& o, uint64_t dumpid = 0)
inline static void print_all_stats(std::ostream& o, uint64_t dumpid = 0)
{
auto alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle);
static_assert(
SharedStateHandle::Options.CoreAllocIsPoolAllocated,
"Global statistics are available only for pool-allocated configurations");
auto alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>();
while (alloc != nullptr)
{
auto stats = alloc->stats();
if (stats != nullptr)
stats->template print<Alloc>(o, dumpid, alloc->id());
alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle, alloc);
stats->template print<decltype(alloc)>(o, dumpid, alloc->id());
alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>(alloc);
}
}
#else
template<class SharedStateHandle>
inline static void
print_all_stats(SharedStateHandle handle, void*& o, uint64_t dumpid = 0)
inline static void print_all_stats(void*& o, uint64_t dumpid = 0)
{
UNUSED(o);
UNUSED(dumpid);
UNUSED(handle);
}
#endif
template<class SharedStateHandle>
inline static void cleanup_unused(SharedStateHandle handle)
inline static void cleanup_unused()
{
#ifndef SNMALLOC_PASS_THROUGH
static_assert(
SharedStateHandle::Options.CoreAllocIsPoolAllocated,
"Global cleanup is available only for pool-allocated configurations");
// Call this periodically to free and coalesce memory allocated by
// allocators that are not currently in use by any thread.
// One atomic operation to extract the stack, another to restore it.
// Handling the message queue for each stack is non-atomic.
auto* first = Pool<CoreAllocator<SharedStateHandle>>::extract(handle);
auto* first = Pool<CoreAllocator<SharedStateHandle>>::extract();
auto* alloc = first;
decltype(alloc) last;
@@ -64,10 +74,10 @@ namespace snmalloc
{
alloc->flush();
last = alloc;
alloc = Pool<CoreAllocator<SharedStateHandle>>::extract(handle, alloc);
alloc = Pool<CoreAllocator<SharedStateHandle>>::extract(alloc);
}
Pool<CoreAllocator<SharedStateHandle>>::restore(handle, first, last);
Pool<CoreAllocator<SharedStateHandle>>::restore(first, last);
}
#endif
}
@@ -78,13 +88,16 @@ namespace snmalloc
raise an error all the allocators are not empty.
*/
template<class SharedStateHandle>
inline static void
debug_check_empty(SharedStateHandle handle, bool* result = nullptr)
inline static void debug_check_empty(bool* result = nullptr)
{
#ifndef SNMALLOC_PASS_THROUGH
static_assert(
SharedStateHandle::Options.CoreAllocIsPoolAllocated,
"Global status is available only for pool-allocated configurations");
// This is a debugging function. It checks that all memory from all
// allocators has been freed.
auto* alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle);
auto* alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>();
# ifdef SNMALLOC_TRACING
std::cout << "debug check empty: first " << alloc << std::endl;
@@ -98,7 +111,8 @@ namespace snmalloc
std::cout << "debug_check_empty: Check all allocators!" << std::endl;
# endif
done = true;
alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle);
alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>();
okay = true;
while (alloc != nullptr)
@@ -120,7 +134,8 @@ namespace snmalloc
# ifdef SNMALLOC_TRACING
std::cout << "debug check empty: okay = " << okay << std::endl;
# endif
alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle, alloc);
alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>(alloc);
}
}
@@ -133,11 +148,13 @@ namespace snmalloc
// Redo check so abort is on allocator with allocation left.
if (!okay)
{
alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle);
alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>();
while (alloc != nullptr)
{
alloc->debug_is_empty(nullptr);
alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle, alloc);
alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>(alloc);
}
}
#else
@@ -146,9 +163,13 @@ namespace snmalloc
}
template<class SharedStateHandle>
inline static void debug_in_use(SharedStateHandle handle, size_t count)
inline static void debug_in_use(size_t count)
{
auto alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle);
static_assert(
SharedStateHandle::Options.CoreAllocIsPoolAllocated,
"Global status is available only for pool-allocated configurations");
auto alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>();
while (alloc != nullptr)
{
if (alloc->debug_is_in_use())
@@ -159,7 +180,8 @@ namespace snmalloc
}
count--;
}
alloc = Pool<CoreAllocator<SharedStateHandle>>::iterate(handle, alloc);
alloc = Pool<CoreAllocator<SharedStateHandle>>::template iterate<
SharedStateHandle>(alloc);
if (count != 0)
{

247
src/mem/localalloc.h
View File

@@ -38,24 +38,34 @@ namespace snmalloc
OnePastEnd
};
// This class contains the fastest path code for the allocator.
/**
* A local allocator contains the fast-path allocation routines and
* encapsulates all of the behaviour of an allocator that is local to some
* context, typically a thread. This delegates to a `CoreAllocator` for all
* slow-path operations, including anything that requires claiming new chunks
* of address space.
*
* The template parameter defines the configuration of this allocator and is
* passed through to the associated `CoreAllocator`. The `Options` structure
* of this defines one property that directly affects the behaviour of the
* local allocator: `LocalAllocSupportsLazyInit`, which defaults to true,
* defines whether the local allocator supports lazy initialisation. If this
* is true then the local allocator will construct a core allocator the first
* time it needs to perform a slow-path operation. If this is false then the
* core allocator must be provided externally by invoking the `init` method
* on this class *before* any allocation-related methods are called.
*/
template<class SharedStateHandle>
class LocalAllocator
{
using CoreAlloc = CoreAllocator<SharedStateHandle>;
private:
/**
* Contains a way to access all the shared state for this allocator.
* This may have no dynamic state, and be purely static.
*/
SharedStateHandle handle;
// Free list per small size class. These are used for
// allocation on the fast path. This part of the code is inspired by
// mimalloc.
// Also contains remote deallocation cache.
LocalCache local_cache;
LocalCache local_cache{&SharedStateHandle::unused_remote};
// Underlying allocator for most non-fast path operations.
CoreAlloc* core_alloc{nullptr};
@@ -90,40 +100,59 @@ namespace snmalloc
/**
* This initialises the fast allocator by acquiring a core allocator, and
* setting up its local copy of data structures.
*
* If the allocator does not support lazy initialisation then this assumes
* that initialisation has already taken place and invokes the action
* immediately.
*/
template<typename Action, typename... Args>
SNMALLOC_SLOW_PATH decltype(auto) lazy_init(Action action, Args... args)
{
SNMALLOC_ASSERT(core_alloc == nullptr);
// Initialise the thread local allocator
init();
// register_clean_up must be called after init. register clean up may be
// implemented with allocation, so need to ensure we have a valid
// allocator at this point.
if (!post_teardown)
// Must be called at least once per thread.
// A pthread implementation only calls the thread destruction handle
// if the key has been set.
handle.register_clean_up();
// Perform underlying operation
auto r = action(core_alloc, args...);
// After performing underlying operation, in the case of teardown already
// having begun, we must flush any state we just acquired.
if (post_teardown)
if constexpr (!SharedStateHandle::Options.LocalAllocSupportsLazyInit)
{
#ifdef SNMALLOC_TRACING
std::cout << "post_teardown flush()" << std::endl;
#endif
// We didn't have an allocator because the thread is being torndown.
// We need to return any local state, so we don't leak it.
flush();
SNMALLOC_CHECK(
false &&
"lazy_init called on an allocator that doesn't support lazy "
"initialisation");
// Unreachable, but needed to keep the type checker happy in deducing
// the return type of this function.
return static_cast<void*>(nullptr);
}
else
{
// Initialise the thread local allocator
if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState)
{
init();
}
return r;
// register_clean_up must be called after init. register clean up may
// be implemented with allocation, so need to ensure we have a valid
// allocator at this point.
if (!post_teardown)
// Must be called at least once per thread.
// A pthread implementation only calls the thread destruction handle
// if the key has been set.
SharedStateHandle::register_clean_up();
// Perform underlying operation
auto r = action(core_alloc, args...);
// After performing underlying operation, in the case of teardown
// already having begun, we must flush any state we just acquired.
if (post_teardown)
{
#ifdef SNMALLOC_TRACING
std::cout << "post_teardown flush()" << std::endl;
#endif
// We didn't have an allocator because the thread is being torndown.
// We need to return any local state, so we don't leak it.
flush();
}
return r;
}
}
/**
@@ -144,13 +173,12 @@ namespace snmalloc
return check_init([&](CoreAlloc* core_alloc) {
// Grab slab of correct size
// Set remote as large allocator remote.
auto [chunk, meta] = ChunkAllocator::alloc_chunk(
handle,
core_alloc->backend_state,
auto [chunk, meta] = ChunkAllocator::alloc_chunk<SharedStateHandle>(
core_alloc->get_backend_local_state(),
bits::next_pow2_bits(size), // TODO
large_size_to_chunk_sizeclass(size),
large_size_to_chunk_size(size),
handle.fake_large_remote);
SharedStateHandle::fake_large_remote);
// set up meta data so sizeclass is correct, and hence alloc size, and
// external pointer.
#ifdef SNMALLOC_TRACING
@@ -164,7 +192,7 @@ namespace snmalloc
if (zero_mem == YesZero)
{
SharedStateHandle::Backend::Pal::template zero<false>(
SharedStateHandle::Pal::template zero<false>(
chunk.unsafe_ptr(), size);
}
@@ -225,8 +253,7 @@ namespace snmalloc
std::cout << "Remote dealloc post" << p << " size " << alloc_size(p)
<< std::endl;
#endif
MetaEntry entry = SharedStateHandle::Backend::get_meta_data(
handle.get_backend_state(), address_cast(p));
MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(p));
local_cache.remote_dealloc_cache.template dealloc<sizeof(CoreAlloc)>(
entry.get_remote()->trunc_id(), CapPtr<void, CBAlloc>(p), key_global);
post_remote_cache();
@@ -252,30 +279,84 @@ namespace snmalloc
return local_cache.remote_allocator;
}
/**
* SFINAE helper. Matched only if `T` implements `is_initialised`. Calls
* it if it exists.
*/
template<typename T>
SNMALLOC_FAST_PATH auto call_is_initialised(T*, int)
-> decltype(T::is_initialised())
{
return T::is_initialised();
}
/**
* SFINAE helper. Matched only if `T` does not implement `is_initialised`.
* Unconditionally returns true if invoked.
*/
template<typename T>
SNMALLOC_FAST_PATH auto call_is_initialised(T*, long)
{
return true;
}
/**
* Call `SharedStateHandle::is_initialised()` if it is implemented,
* unconditionally returns true otherwise.
*/
SNMALLOC_FAST_PATH
bool is_initialised()
{
return call_is_initialised<SharedStateHandle>(nullptr, 0);
}
/**
* SFINAE helper. Matched only if `T` implements `ensure_init`. Calls it
* if it exists.
*/
template<typename T>
SNMALLOC_FAST_PATH auto call_ensure_init(T*, int)
-> decltype(T::ensure_init())
{
T::ensure_init();
}
/**
* SFINAE helper. Matched only if `T` does not implement `ensure_init`.
* Does nothing if called.
*/
template<typename T>
SNMALLOC_FAST_PATH auto call_ensure_init(T*, long)
{}
/**
* Call `SharedStateHandle::ensure_init()` if it is implemented, do nothing
* otherwise.
*/
SNMALLOC_FAST_PATH
void ensure_init()
{
call_ensure_init<SharedStateHandle>(nullptr, 0);
}
public:
constexpr LocalAllocator()
: handle(SharedStateHandle::get_handle()),
local_cache(&handle.unused_remote)
{}
constexpr LocalAllocator() = default;
LocalAllocator(SharedStateHandle handle)
: handle(handle), local_cache(&handle.unused_remote)
{}
// This is effectively the constructor for the LocalAllocator, but due to
// not wanting initialisation checks on the fast path, it is initialised
// lazily.
void init()
/**
* Initialise the allocator. For allocators that support local
* initialisation, this is called with a core allocator that this class
* allocates (from a pool allocator) the first time it encounters a slow
* path. If this class is configured without lazy initialisation support
* then this must be called externally
*/
void init(CoreAlloc* c)
{
// Initialise the global allocator structures
handle.ensure_init();
ensure_init();
// Should only be called if the allocator has not been initialised.
SNMALLOC_ASSERT(core_alloc == nullptr);
// Grab an allocator for this thread.
auto c = Pool<CoreAlloc>::acquire(handle, &(this->local_cache), handle);
// Attach to it.
c->attach(&local_cache);
core_alloc = c;
@@ -286,6 +367,18 @@ namespace snmalloc
// local_cache.stats.sta rt();
}
// This is effectively the constructor for the LocalAllocator, but due to
// not wanting initialisation checks on the fast path, it is initialised
// lazily.
void init()
{
// Initialise the global allocator structures
ensure_init();
// Grab an allocator for this thread.
init(Pool<CoreAlloc>::template acquire<SharedStateHandle>(
&(this->local_cache)));
}
// Return all state in the fast allocator and release the underlying
// core allocator. This is used during teardown to empty the thread
// local state.
@@ -303,7 +396,10 @@ namespace snmalloc
// Detach underlying allocator
core_alloc->attached_cache = nullptr;
// Return underlying allocator to the system.
Pool<CoreAlloc>::release(handle, core_alloc);
if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState)
{
Pool<CoreAlloc>::template release<SharedStateHandle>(core_alloc);
}
// Set up thread local allocator to look like
// it is new to hit slow paths.
@@ -311,7 +407,7 @@ namespace snmalloc
#ifdef SNMALLOC_TRACING
std::cout << "flush(): core_alloc=" << core_alloc << std::endl;
#endif
local_cache.remote_allocator = &handle.unused_remote;
local_cache.remote_allocator = &SharedStateHandle::unused_remote;
local_cache.remote_dealloc_cache.capacity = 0;
}
}
@@ -365,8 +461,8 @@ namespace snmalloc
// before init, that maps null to a remote_deallocator that will never be
// in thread local state.
const MetaEntry& entry = SharedStateHandle::Backend::get_meta_data(
handle.get_backend_state(), address_cast(p));
const MetaEntry& entry =
SharedStateHandle::get_meta_data(address_cast(p));
if (likely(local_cache.remote_allocator == entry.get_remote()))
{
if (likely(CoreAlloc::dealloc_local_object_fast(
@@ -376,7 +472,7 @@ namespace snmalloc
return;
}
if (likely(entry.get_remote() != handle.fake_large_remote))
if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote))
{
// Check if we have space for the remote deallocation
if (local_cache.remote_dealloc_cache.reserve_space(entry))
@@ -416,7 +512,8 @@ namespace snmalloc
ChunkRecord* slab_record =
reinterpret_cast<ChunkRecord*>(entry.get_metaslab());
slab_record->chunk = CapPtr<void, CBChunk>(p);
ChunkAllocator::dealloc(handle, slab_record, slab_sizeclass);
ChunkAllocator::dealloc<SharedStateHandle>(
core_alloc->get_backend_local_state(), slab_record, slab_sizeclass);
return;
}
@@ -460,10 +557,9 @@ namespace snmalloc
// To handle this case we require the uninitialised pagemap contain an
// entry for the first chunk of memory, that states it represents a large
// object, so we can pull the check for null off the fast path.
MetaEntry entry = SharedStateHandle::Backend::get_meta_data(
handle.get_backend_state(), address_cast(p_raw));
MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(p_raw));
if (likely(entry.get_remote() != handle.fake_large_remote))
if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote))
return sizeclass_to_size(entry.get_sizeclass());
// Sizeclass zero is for large is actually zero
@@ -484,13 +580,12 @@ namespace snmalloc
void* external_pointer(void* p_raw)
{
// TODO bring back the CHERI bits. Wes to review if required.
if (likely(handle.is_initialised()))
if (likely(is_initialised()))
{
MetaEntry entry =
SharedStateHandle::Backend::template get_meta_data<true>(
handle.get_backend_state(), address_cast(p_raw));
SharedStateHandle::template get_meta_data<true>(address_cast(p_raw));
auto sizeclass = entry.get_sizeclass();
if (likely(entry.get_remote() != handle.fake_large_remote))
if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote))
{
auto rsize = sizeclass_to_size(sizeclass);
auto offset =
@@ -533,5 +628,15 @@ namespace snmalloc
// We don't know the Start, so return MIN_PTR
return nullptr;
}
/**
* Accessor, returns the local cache. If embedding code is allocating the
* core allocator for use by this local allocator then it needs to access
* this field.
*/
LocalCache& get_local_cache()
{
return local_cache;
}
};
} // namespace snmalloc
} // namespace snmalloc

14
src/mem/localcache.h
View File

@@ -32,7 +32,7 @@ namespace snmalloc
auto r = finish_alloc_no_zero(p, sizeclass);
if constexpr (zero_mem == YesZero)
SharedStateHandle::Backend::Pal::zero(r, sizeclass_to_size(sizeclass));
SharedStateHandle::Pal::zero(r, sizeclass_to_size(sizeclass));
// TODO: Should this be zeroing the FreeObject state, in the non-zeroing
// case?
@@ -72,9 +72,9 @@ namespace snmalloc
template<
size_t allocator_size,
typename DeallocFun,
typename SharedStateHandle>
bool flush(DeallocFun dealloc, SharedStateHandle handle)
typename SharedStateHandle,
typename DeallocFun>
bool flush(DeallocFun dealloc)
{
auto& key = entropy.get_free_list_key();
@@ -91,8 +91,8 @@ namespace snmalloc
}
}
return remote_dealloc_cache.post<allocator_size>(
handle, remote_allocator->trunc_id(), key_global);
return remote_dealloc_cache.post<allocator_size, SharedStateHandle>(
remote_allocator->trunc_id(), key_global);
}
template<ZeroMem zero_mem, typename SharedStateHandle, typename Slowpath>
@@ -113,4 +113,4 @@ namespace snmalloc
}
};
} // namespace snmalloc
} // namespace snmalloc

21
src/mem/metaslab.h
View File

@@ -177,6 +177,16 @@ namespace snmalloc
public:
constexpr MetaEntry() = default;
/**
* Constructor, provides the remote and sizeclass embedded in a single
* pointer-sized word. This format is not guaranteed to be stable and so
* the second argument of this must always be the return value from
* `get_remote_and_sizeclass`.
*/
MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass)
: meta(meta), remote_and_sizeclass(remote_and_sizeclass)
{}
MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass)
: meta(meta)
{
@@ -190,6 +200,17 @@ namespace snmalloc
return meta;
}
/**
* Return the remote and sizeclass in an implementation-defined encoding.
* This is not guaranteed to be stable across snmalloc releases and so the
* only safe use for this is to pass it to the two-argument constructor of
* this class.
*/
uintptr_t get_remote_and_sizeclass()
{
return remote_and_sizeclass;
}
[[nodiscard]] RemoteAllocator* get_remote() const
{
return reinterpret_cast<RemoteAllocator*>(

26
src/mem/pool.h
View File

@@ -37,9 +37,9 @@ namespace snmalloc
{
public:
template<typename SharedStateHandle, typename... Args>
static T* acquire(SharedStateHandle h, Args&&... args)
static T* acquire(Args&&... args)
{
PoolState<T>& pool = h.pool();
PoolState<T>& pool = SharedStateHandle::pool();
T* p = pool.stack.pop();
if (p != nullptr)
@@ -48,12 +48,12 @@ namespace snmalloc
return p;
}
p = ChunkAllocator::alloc_meta_data<T>(
h, nullptr, std::forward<Args>(args)...);
p = ChunkAllocator::alloc_meta_data<T, SharedStateHandle>(
nullptr, std::forward<Args>(args)...);
if (p == nullptr)
{
SharedStateHandle::Backend::Pal::error(
SharedStateHandle::Pal::error(
"Failed to initialise thread local allocator.");
}
@@ -71,21 +71,21 @@ namespace snmalloc
* Do not return objects from `extract`.
*/
template<typename SharedStateHandle>
static void release(SharedStateHandle h, T* p)
static void release(T* p)
{
// The object's destructor is not run. If the object is "reallocated", it
// is returned without the constructor being run, so the object is reused
// without re-initialisation.
p->reset_in_use();
h.pool().stack.push(p);
SharedStateHandle::pool().stack.push(p);
}
template<typename SharedStateHandle>
static T* extract(SharedStateHandle h, T* p = nullptr)
static T* extract(T* p = nullptr)
{
// Returns a linked list of all objects in the stack, emptying the stack.
if (p == nullptr)
return h.pool().stack.pop_all();
return SharedStateHandle::pool().stack.pop_all();
return p->next;
}
@@ -96,18 +96,18 @@ namespace snmalloc
* Do not return objects from `acquire`.
*/
template<typename SharedStateHandle>
static void restore(SharedStateHandle h, T* first, T* last)
static void restore(T* first, T* last)
{
// Pushes a linked list of objects onto the stack. Use to put a linked
// list returned by extract back onto the stack.
h.pool().stack.push(first, last);
SharedStateHandle::pool().stack.push(first, last);
}
template<typename SharedStateHandle>
static T* iterate(SharedStateHandle h, T* p = nullptr)
static T* iterate(T* p = nullptr)
{
if (p == nullptr)
return h.pool().list;
return SharedStateHandle::pool().list;
return p->list_next;
}

12
src/mem/remotecache.h
View File

@@ -76,10 +76,7 @@ namespace snmalloc
}
template<size_t allocator_size, typename SharedStateHandle>
bool post(
SharedStateHandle handle,
RemoteAllocator::alloc_id_t id,
const FreeListKey& key)
bool post(RemoteAllocator::alloc_id_t id, const FreeListKey& key)
{
SNMALLOC_ASSERT(initialised);
size_t post_round = 0;
@@ -97,8 +94,8 @@ namespace snmalloc
if (!list[i].empty())
{
auto [first, last] = list[i].extract_segment(key);
MetaEntry entry = SharedStateHandle::Backend::get_meta_data(
handle.get_backend_state(), address_cast(first));
MetaEntry entry =
SharedStateHandle::get_meta_data(address_cast(first));
entry.get_remote()->enqueue(first, last, key);
sent_something = true;
}
@@ -120,8 +117,7 @@ namespace snmalloc
// Use the next N bits to spread out remote deallocs in our own
// slot.
auto r = resend.take(key);
MetaEntry entry = SharedStateHandle::Backend::get_meta_data(
handle.get_backend_state(), address_cast(r));
MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(r));
auto i = entry.get_remote()->trunc_id();
size_t slot = get_slot<allocator_size>(i, post_round);
list[slot].add(r, key);

29
src/mem/slaballocator.h
View File

@@ -75,14 +75,14 @@ namespace snmalloc
public:
template<typename SharedStateHandle>
static std::pair<CapPtr<void, CBChunk>, Metaslab*> alloc_chunk(
SharedStateHandle h,
typename SharedStateHandle::Backend::LocalState& backend_state,
typename SharedStateHandle::LocalState& local_state,
sizeclass_t sizeclass,
sizeclass_t slab_sizeclass, // TODO sizeclass_t
size_t slab_size,
RemoteAllocator* remote)
{
ChunkAllocatorState& state = h.get_slab_allocator_state();
ChunkAllocatorState& state =
SharedStateHandle::get_slab_allocator_state(&local_state);
// Pop a slab
auto chunk_record = state.chunk_stack[slab_sizeclass].pop();
@@ -98,15 +98,14 @@ namespace snmalloc
<< std::endl;
#endif
MetaEntry entry{meta, remote, sizeclass};
SharedStateHandle::Backend::set_meta_data(
h.get_backend_state(), address_cast(slab), slab_size, entry);
SharedStateHandle::set_meta_data(address_cast(slab), slab_size, entry);
return {slab, meta};
}
// Allocate a fresh slab as there are no available ones.
// First create meta-data
auto [slab, meta] = SharedStateHandle::Backend::alloc_chunk(
h.get_backend_state(), &backend_state, slab_size, remote, sizeclass);
auto [slab, meta] = SharedStateHandle::alloc_chunk(
&local_state, slab_size, remote, sizeclass);
#ifdef SNMALLOC_TRACING
std::cout << "Create slab:" << slab.unsafe_ptr() << " slab_sizeclass "
<< slab_sizeclass << " size " << slab_size << std::endl;
@@ -122,10 +121,12 @@ namespace snmalloc
}
template<typename SharedStateHandle>
SNMALLOC_SLOW_PATH static void
dealloc(SharedStateHandle h, ChunkRecord* p, size_t slab_sizeclass)
SNMALLOC_SLOW_PATH static void dealloc(
typename SharedStateHandle::LocalState& local_state,
ChunkRecord* p,
size_t slab_sizeclass)
{
auto& state = h.get_slab_allocator_state();
auto& state = SharedStateHandle::get_slab_allocator_state(&local_state);
#ifdef SNMALLOC_TRACING
std::cout << "Return slab:" << p->chunk.unsafe_ptr() << " slab_sizeclass "
<< slab_sizeclass << " size "
@@ -144,15 +145,13 @@ namespace snmalloc
*/
template<typename U, typename SharedStateHandle, typename... Args>
static U* alloc_meta_data(
SharedStateHandle h,
typename SharedStateHandle::Backend::LocalState* local_state,
Args&&... args)
typename SharedStateHandle::LocalState* local_state, Args&&... args)
{
// Cache line align
size_t size = bits::align_up(sizeof(U), 64);
CapPtr<void, CBChunk> p = SharedStateHandle::Backend::alloc_meta_data(
h.get_backend_state(), local_state, size);
CapPtr<void, CBChunk> p =
SharedStateHandle::template alloc_meta_data<U>(local_state, size);
if (p == nullptr)
return nullptr;

2
src/mem/threadalloc.h
View File

@@ -156,6 +156,6 @@ namespace snmalloc
*/
void _malloc_thread_cleanup()
{
ThreadAlloc::get().teardown();
snmalloc::ThreadAlloc::get().teardown();
}
#endif

8
src/override/malloc-extensions.cc
View File

@@ -6,10 +6,8 @@ using namespace snmalloc;
void get_malloc_info_v1(malloc_info_v1* stats)
{
auto unused_chunks =
Globals::get_handle().get_slab_allocator_state().unused_memory();
auto peak =
Globals::get_handle().get_slab_allocator_state().peak_memory_usage();
auto unused_chunks = Globals::get_slab_allocator_state().unused_memory();
auto peak = Globals::get_slab_allocator_state().peak_memory_usage();
stats->current_memory_usage = peak - unused_chunks;
stats->peak_memory_usage = peak;
}
}

2
src/override/malloc.cc
View File

@@ -2,7 +2,7 @@
#include "../snmalloc_core.h"
#ifndef SNMALLOC_PROVIDE_OWN_CONFIG
# include "../mem/globalconfig.h"
# include "../backend/globalconfig.h"
// The default configuration for snmalloc is used if alternative not defined
namespace snmalloc
{

10
src/snmalloc.h
View File

@@ -3,8 +3,13 @@
// Core implementation of snmalloc independent of the configuration mode
#include "snmalloc_core.h"
// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own
// definition of `snmalloc::Alloc` and include `snmalloc_front.h` before
// including any files that include `snmalloc.h` and consume the global
// allocation APIs.
#ifndef SNMALLOC_PROVIDE_OWN_CONFIG
// Default implementation of global state
#include "mem/globalconfig.h"
# include "backend/globalconfig.h"
// The default configuration for snmalloc
namespace snmalloc
@@ -13,4 +18,5 @@ namespace snmalloc
}
// User facing API surface, needs to know what `Alloc` is.
#include "snmalloc_front.h"
# include "snmalloc_front.h"
#endif

5
src/snmalloc_core.h
View File

@@ -1,3 +1,6 @@
#pragma once
#include "mem/globalalloc.h"
#include "backend/address_space.h"
#include "backend/commonconfig.h"
#include "backend/pagemap.h"
#include "mem/globalalloc.h"

5
src/test/func/fixed_region/fixed_region.cc
View File

@@ -1,4 +1,4 @@
#include "mem/fixedglobalconfig.h"
#include "backend/fixedglobalconfig.h"
#include "test/setup.h"
#include <iostream>
@@ -29,9 +29,8 @@ int main()
std::cout << "Allocated region " << oe_base << " - "
<< pointer_offset(oe_base, size) << std::endl;
CustomGlobals fixed_handle;
CustomGlobals::init(oe_base, size);
FixedAlloc a(fixed_handle);
FixedAlloc a;
size_t object_size = 128;
size_t count = 0;

2
src/test/func/malloc/malloc.cc
View File

@@ -232,6 +232,6 @@ int main(int argc, char** argv)
abort();
}
snmalloc::debug_check_empty(Globals::get_handle());
snmalloc::debug_check_empty<snmalloc::Globals>();
return 0;
}

6
src/test/func/memory/memory.cc
View File

@@ -183,7 +183,7 @@ void test_calloc()
alloc.dealloc(p, size);
}
snmalloc::debug_check_empty(Globals::get_handle());
snmalloc::debug_check_empty<Globals>();
}
void test_double_alloc()
@@ -228,7 +228,7 @@ void test_double_alloc()
}
}
}
snmalloc::debug_check_empty(Globals::get_handle());
snmalloc::debug_check_empty<Globals>();
}
void test_external_pointer()
@@ -264,7 +264,7 @@ void test_external_pointer()
alloc.dealloc(p1, size);
}
snmalloc::debug_check_empty(Globals::get_handle());
snmalloc::debug_check_empty<Globals>();
};
void check_offset(void* base, void* interior)

10
src/test/func/statistics/stats.cc
View File

@@ -8,7 +8,7 @@ int main()
auto r = a.alloc(16);
snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result);
snmalloc::debug_check_empty<snmalloc::Globals>(&result);
if (result != false)
{
abort();
@@ -16,7 +16,7 @@ int main()
a.dealloc(r);
snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result);
snmalloc::debug_check_empty<snmalloc::Globals>(&result);
if (result != true)
{
abort();
@@ -24,7 +24,7 @@ int main()
r = a.alloc(16);
snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result);
snmalloc::debug_check_empty<snmalloc::Globals>(&result);
if (result != false)
{
abort();
@@ -32,10 +32,10 @@ int main()
a.dealloc(r);
snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result);
snmalloc::debug_check_empty<snmalloc::Globals>(&result);
if (result != true)
{
abort();
}
#endif
}
}

2
src/test/func/thread_alloc_external/thread_alloc_external.cc
View File

@@ -4,7 +4,7 @@
// Specify using own
#define SNMALLOC_EXTERNAL_THREAD_ALLOC
#include "mem/globalconfig.h"
#include "backend/globalconfig.h"
namespace snmalloc
{

5
src/test/func/two_alloc_types/alloc1.cc
View File

@@ -3,7 +3,7 @@
// Redefine the namespace, so we can have two versions.
#define snmalloc snmalloc_enclave
#include <mem/fixedglobalconfig.h>
#include <backend/fixedglobalconfig.h>
#include <snmalloc_core.h>
// Specify type of allocator
@@ -19,6 +19,5 @@ namespace snmalloc
extern "C" void oe_allocator_init(void* base, void* end)
{
snmalloc::CustomGlobals fixed_handle;
fixed_handle.init(base, address_cast(end) - address_cast(base));
snmalloc::CustomGlobals::init(base, address_cast(end) - address_cast(base));
}

2
src/test/perf/contention/contention.cc
View File

@@ -142,7 +142,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size)
}
#ifndef NDEBUG
snmalloc::debug_check_empty(Globals::get_handle());
snmalloc::debug_check_empty<Globals>();
#endif
};

2
src/test/perf/external_pointer/externalpointer.cc
View File

@@ -47,7 +47,7 @@ namespace test
alloc.dealloc(objects[i]);
}
snmalloc::debug_check_empty(Globals::get_handle());
snmalloc::debug_check_empty<Globals>();
}
void test_external_pointer(xoroshiro::p128r64& r)

2
src/test/perf/singlethread/singlethread.cc
View File

@@ -60,7 +60,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write)
}
}
snmalloc::debug_check_empty(Globals::get_handle());
snmalloc::debug_check_empty<Globals>();
}
int main(int, char**)