Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move MetaEntry and MetaCommon to backend

Browse Source 
These are almost entirely backend concerns, so move their definitions over
there.  Use C++ friend classes to ensure that MetaCommon structures are opaque
to frontend code (at least, at compile time, and neglecting the rest of C++).
(These structures contain high-authority pointers and so should be as closely
guarded as we can make them.)

The bits that leak out are

- the encoding of RemoteAllocator* and sizeclass_t into the uinptr_t within a
  MetaEntry.  This, however, is almost entirely a frontend concern, so detach
  the method definitions from the class and leave those in mem/metaslab.h for
  the moment.

- the size of metadata structures pointed to by the MetaEntry meta field.
  Rather than use sizeof(Metaslab) (and assert that sizeof(ChunkRecord) is
  smaller), instead, define PAGEMAP_METADATA_STRUCT_SIZE once and assert that
  all records fit.  Additionally, add an assertion that Metaslab is exactly this
  size, not for semantic reasons, but because we expect it to be true.

The bits that leak in are

- the need to zero memory corresponding to a chunk.  Rather than having an
  escape hatch that reveals the MetaCommon.chunk, move the zeroing call into a
  small wrapper method within the MetaCommon class itself.

- the need to get the address of a chunk.  We want to assert that we've got the
  right chunk on occasion (well, at least once so far) and so add a class method
  to expose the address_t view of the chunk pointer without exposing the pointer
  itself.
This commit is contained in:
Nathaniel Wesley Filardo
2022-03-13 15:43:55 +00:00
committed by Nathaniel Wesley Filardo
parent 6ad7f65d19
commit 9d97a38806
6 changed files with 253 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/backend/backend.h
View File

@@ -1,6 +1,5 @@
#pragma once
#include "../mem/allocconfig.h"
#include "../mem/metaslab.h"
#include "../pal/pal.h"
#include "chunkallocator.h"
#include "commitrange.h"
@@ -8,6 +7,7 @@
#include "empty_range.h"
#include "globalrange.h"
#include "largebuddyrange.h"
#include "metatypes.h"
#include "pagemap.h"
#include "pagemapregisterrange.h"
#include "palrange.h"
@@ -259,7 +259,7 @@ namespace snmalloc
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
auto meta_cap =
local_state.get_meta_range()->alloc_range(sizeof(Metaslab));
local_state.get_meta_range()->alloc_range(PAGEMAP_METADATA_STRUCT_SIZE);
auto meta = meta_cap.template as_reinterpret<Metaslab>().unsafe_ptr();
@@ -277,7 +277,8 @@ namespace snmalloc
#endif
if (p == nullptr)
{
local_state.get_meta_range()->dealloc_range(meta_cap, sizeof(Metaslab));
local_state.get_meta_range()->dealloc_range(
meta_cap, PAGEMAP_METADATA_STRUCT_SIZE);
errno = ENOMEM;
#ifdef SNMALLOC_TRACING
std::cout << "Out of memory" << std::endl;
@@ -300,7 +301,7 @@ namespace snmalloc
auto chunk = chunk_record->meta_common.chunk;
local_state.get_meta_range()->dealloc_range(
capptr::Chunk<void>(chunk_record), sizeof(Metaslab));
capptr::Chunk<void>(chunk_record), PAGEMAP_METADATA_STRUCT_SIZE);
// TODO, should we set the sizeclass to something specific here?

16
src/backend/chunkallocator.h
View File

@@ -7,6 +7,7 @@
*/
#include "../backend/backend_concept.h"
#include "../backend/metatypes.h"
#include "../ds/mpmcstack.h"
#include "../ds/spmcstack.h"
#include "../mem/metaslab.h"
@@ -29,22 +30,15 @@ namespace snmalloc
MetaCommon meta_common;
std::atomic<ChunkRecord*> next;
};
static_assert(std::is_standard_layout_v<ChunkRecord>);
static_assert(
offsetof(ChunkRecord, meta_common) == 0,
"ChunkRecord and Metaslab must share a common prefix");
#if defined(USE_METADATA_CONCEPT)
static_assert(ConceptMetadataStruct<ChunkRecord>);
#endif
/**
* How many slab sizes that can be provided.
*/
constexpr size_t NUM_SLAB_SIZES = Pal::address_bits - MIN_CHUNK_BITS;
/**
* Used to ensure the per slab meta data is large enough for both use cases.
*/
static_assert(
sizeof(Metaslab) >= sizeof(ChunkRecord), "We conflate these two types.");
/**
* Number of free stacks per chunk size that each allocator will use.
* For performance ideally a power of 2. We will return to the central
@@ -256,7 +250,7 @@ namespace snmalloc
#endif
state.add_peak_memory_usage(slab_size);
state.add_peak_memory_usage(sizeof(Metaslab));
state.add_peak_memory_usage(PAGEMAP_METADATA_STRUCT_SIZE);
// TODO handle bounded versus lazy pagemaps in stats
state.add_peak_memory_usage(
(slab_size / MIN_CHUNK_SIZE) * sizeof(MetaEntry));

1
src/backend/largebuddyrange.h
View File

@@ -3,7 +3,6 @@
#include "../ds/address.h"
#include "../ds/bits.h"
#include "../mem/allocconfig.h"
#include "../mem/metaslab.h"
#include "../pal/pal.h"
#include "buddy.h"
#include "range_helpers.h"

196
src/backend/metatypes.h Normal file
View File

@@ -0,0 +1,196 @@
#pragma once
#include "../ds/concept.h"
#include "../mem/allocconfig.h" /* TODO: CACHELINE_SIZE */
#include "../pal/pal_concept.h"
namespace snmalloc
{
/**
* A guaranteed type-stable sub-structure of all metadata referenced by the
* Pagemap. Use-specific structures (Metaslab, ChunkRecord) are expected to
* have this at offset zero so that, even in the face of concurrent mutation
* and reuse of the memory backing that metadata, the types of these fields
* remain fixed.
*
* This class's data is fully private but is friends with the relevant backend
* types and, thus, is "opaque" to the frontend.
*/
class MetaCommon
{
friend class ChunkAllocator;
template<SNMALLOC_CONCEPT(ConceptPAL) PAL, bool, typename>
friend class BackendAllocator;
capptr::Chunk<void> chunk;
public:
/**
* Expose the address of, though not the authority to, our corresponding
* chunk.
*/
[[nodiscard]] SNMALLOC_FAST_PATH address_t chunk_address()
{
return address_cast(this->chunk);
}
/**
* Zero (possibly by unmapping) the memory backing this chunk. We must rely
* on the caller to tell us its size, which is a little unfortunate.
*/
template<SNMALLOC_CONCEPT(ConceptPAL) PAL>
SNMALLOC_FAST_PATH void zero_chunk(size_t chunk_size)
{
PAL::zero(this->chunk.unsafe_ptr(), chunk_size);
}
};
static const size_t PAGEMAP_METADATA_STRUCT_SIZE =
#ifdef __CHERI_PURE_CAPABILITY__
2 * CACHELINE_SIZE
#else
CACHELINE_SIZE
#endif
;
// clang-format off
/* This triggers an ICE (C1001) in MSVC, so disable it there */
#if defined(__cpp_concepts) && !defined(_MSC_VER)
template<typename Meta>
concept ConceptMetadataStruct =
// Metadata structures must be standard layout (for offsetof()),
std::is_standard_layout_v<Meta> &&
// must be of a sufficiently small size,
sizeof(Meta) <= PAGEMAP_METADATA_STRUCT_SIZE &&
// and must be pointer-interconvertable with MetaCommon.
(
ConceptSame<Meta, MetaCommon> ||
requires(Meta m) {
// Otherwise, meta must have MetaCommon field named meta_common ...
{ &m.meta_common } -> ConceptSame<MetaCommon*>;
// at offset zero.
(offsetof(Meta, meta_common) == 0);
}
);
static_assert(ConceptMetadataStruct<MetaCommon>);
# define USE_METADATA_CONCEPT
#endif
// clang-format on
struct RemoteAllocator;
class Metaslab;
class sizeclass_t;
/**
* Entry stored in the pagemap.
*/
class MetaEntry
{
template<typename Pagemap>
friend class BuddyChunkRep;
/**
* The pointer to the metaslab, the bottom bit is used to indicate if this
* is the first chunk in a PAL allocation, that cannot be combined with
* the preceeding chunk.
*/
uintptr_t meta{0};
/**
* Bit used to indicate this should not be considered part of the previous
* PAL allocation.
*
* Some platforms cannot treat different PalAllocs as a single allocation.
* This is true on CHERI as the combined permission might not be
* representable. It is also true on Windows as you cannot Commit across
* multiple continuous VirtualAllocs.
*/
static constexpr address_t BOUNDARY_BIT = 1;
/**
* A bit-packed pointer to the owning allocator (if any), and the sizeclass
* of this chunk. The sizeclass here is itself a union between two cases:
*
* * log_2(size), at least MIN_CHUNK_BITS, for large allocations.
*
* * a value in [0, NUM_SMALL_SIZECLASSES] for small allocations. These
* may be directly passed to the sizeclass (not slab_sizeclass) functions
* of sizeclasstable.h
*
*/
uintptr_t remote_and_sizeclass{0};
public:
constexpr MetaEntry() = default;
/**
* Constructor, provides the remote and sizeclass embedded in a single
* pointer-sized word. This format is not guaranteed to be stable and so
* the second argument of this must always be the return value from
* `get_remote_and_sizeclass`.
*/
SNMALLOC_FAST_PATH
MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass)
: meta(unsafe_to_uintptr<Metaslab>(meta)),
remote_and_sizeclass(remote_and_sizeclass)
{}
/* See mem/metaslab.h */
SNMALLOC_FAST_PATH
MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass);
/**
* Return the Metaslab metadata associated with this chunk, guarded by an
* assert that this chunk is being used as a slab (i.e., has an associated
* owning allocator).
*/
[[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const
{
SNMALLOC_ASSERT(get_remote() != nullptr);
return unsafe_from_uintptr<Metaslab>(meta & ~BOUNDARY_BIT);
}
/**
* Return the remote and sizeclass in an implementation-defined encoding.
* This is not guaranteed to be stable across snmalloc releases and so the
* only safe use for this is to pass it to the two-argument constructor of
* this class.
*/
[[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const
{
return remote_and_sizeclass;
}
/* See mem/metaslab.h */
[[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const;
[[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const;
MetaEntry(const MetaEntry&) = delete;
MetaEntry& operator=(const MetaEntry& other)
{
// Don't overwrite the boundary bit with the other's
meta = (other.meta & ~BOUNDARY_BIT) | address_cast(meta & BOUNDARY_BIT);
remote_and_sizeclass = other.remote_and_sizeclass;
return *this;
}
void set_boundary()
{
meta |= BOUNDARY_BIT;
}
[[nodiscard]] bool is_boundary() const
{
return meta & BOUNDARY_BIT;
}
bool clear_boundary_bit()
{
return meta &= ~BOUNDARY_BIT;
}
};
} // namespace snmalloc

8
src/mem/corealloc.h
View File

@@ -332,7 +332,7 @@ namespace snmalloc
SNMALLOC_ASSERT(
address_cast(start_of_slab) ==
address_cast(chunk_record->meta_common.chunk));
chunk_record->meta_common.chunk_address());
#if defined(__CHERI_PURE_CAPABILITY__) && !defined(SNMALLOC_CHECK_CLIENT)
// Zero the whole slab. For CHERI we at least need to clear the freelist
@@ -340,9 +340,9 @@ namespace snmalloc
// the freelist order as for SNMALLOC_CHECK_CLIENT. Zeroing the whole slab
// may be more friendly to hw because it does not involve pointer chasing
// and is amenable to prefetching.
SharedStateHandle::Pal::zero(
chunk_record->meta_common.chunk.unsafe_ptr(),
snmalloc::sizeclass_to_slab_size(sizeclass));
chunk_record->meta_common
.template zero_chunk<typename SharedStateHandle::Pal>(
snmalloc::sizeclass_to_slab_size(sizeclass));
#endif
#ifdef SNMALLOC_TRACING

188
src/mem/metaslab.h
View File

@@ -1,5 +1,6 @@
#pragma once
#include "../backend/metatypes.h"
#include "../ds/helpers.h"
#include "../ds/seqset.h"
#include "../mem/remoteallocator.h"
@@ -8,18 +9,6 @@
namespace snmalloc
{
/**
* A guaranteed type-stable sub-structure of all metadata referenced by the
* Pagemap. Use-specific structures (Metaslab, ChunkRecord) are expected to
* have this at offset zero so that, even in the face of concurrent mutation
* and reuse of the memory backing that metadata, the types of these fields
* remain fixed.
*/
struct MetaCommon
{
capptr::Chunk<void> chunk;
};
// The Metaslab represent the status of a single slab.
class alignas(CACHELINE_SIZE) Metaslab
{
@@ -216,140 +205,12 @@ namespace snmalloc
}
};
static_assert(std::is_standard_layout_v<Metaslab>);
#if defined(USE_METADATA_CONCEPT)
static_assert(ConceptMetadataStruct<Metaslab>);
#endif
static_assert(
offsetof(Metaslab, meta_common) == 0,
"ChunkRecord and Metaslab must share a common prefix");
/**
* Entry stored in the pagemap.
*/
class MetaEntry
{
template<typename Pagemap>
friend class BuddyChunkRep;
/**
* The pointer to the metaslab, the bottom bit is used to indicate if this
* is the first chunk in a PAL allocation, that cannot be combined with
* the preceeding chunk.
*/
uintptr_t meta{0};
/**
* Bit used to indicate this should not be considered part of the previous
* PAL allocation.
*
* Some platforms cannot treat different PalAllocs as a single allocation.
* This is true on CHERI as the combined permission might not be
* representable. It is also true on Windows as you cannot Commit across
* multiple continuous VirtualAllocs.
*/
static constexpr address_t BOUNDARY_BIT = 1;
/**
* A bit-packed pointer to the owning allocator (if any), and the sizeclass
* of this chunk. The sizeclass here is itself a union between two cases:
*
* * log_2(size), at least MIN_CHUNK_BITS, for large allocations.
*
* * a value in [0, NUM_SMALL_SIZECLASSES] for small allocations. These
* may be directly passed to the sizeclass (not slab_sizeclass) functions of
* sizeclasstable.h
*
*/
uintptr_t remote_and_sizeclass{0};
public:
constexpr MetaEntry() = default;
/**
* Constructor, provides the remote and sizeclass embedded in a single
* pointer-sized word. This format is not guaranteed to be stable and so
* the second argument of this must always be the return value from
* `get_remote_and_sizeclass`.
*/
SNMALLOC_FAST_PATH
MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass)
: meta(unsafe_to_uintptr<Metaslab>(meta)),
remote_and_sizeclass(remote_and_sizeclass)
{}
SNMALLOC_FAST_PATH
MetaEntry(
Metaslab* meta,
RemoteAllocator* remote,
sizeclass_t sizeclass = sizeclass_t())
: meta(unsafe_to_uintptr<Metaslab>(meta))
{
/* remote might be nullptr; cast to uintptr_t before offsetting */
remote_and_sizeclass = pointer_offset(
unsafe_to_uintptr<RemoteAllocator>(remote), sizeclass.raw());
}
/**
* Return the Metaslab metadata associated with this chunk, guarded by an
* assert that this chunk is being used as a slab (i.e., has an associated
* owning allocator).
*/
[[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const
{
SNMALLOC_ASSERT(get_remote() != nullptr);
return unsafe_from_uintptr<Metaslab>(meta & ~BOUNDARY_BIT);
}
/**
* Return the remote and sizeclass in an implementation-defined encoding.
* This is not guaranteed to be stable across snmalloc releases and so the
* only safe use for this is to pass it to the two-argument constructor of
* this class.
*/
[[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const
{
return remote_and_sizeclass;
}
[[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const
{
return unsafe_from_uintptr<RemoteAllocator>(
pointer_align_down<REMOTE_WITH_BACKEND_MARKER_ALIGN>(
remote_and_sizeclass));
}
[[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const
{
// TODO: perhaps remove static_cast with resolution of
// https://github.com/CTSRD-CHERI/llvm-project/issues/588
return sizeclass_t::from_raw(
static_cast<size_t>(remote_and_sizeclass) &
(REMOTE_WITH_BACKEND_MARKER_ALIGN - 1));
}
MetaEntry(const MetaEntry&) = delete;
MetaEntry& operator=(const MetaEntry& other)
{
// Don't overwrite the boundary bit with the other's
meta = (other.meta & ~BOUNDARY_BIT) | address_cast(meta & BOUNDARY_BIT);
remote_and_sizeclass = other.remote_and_sizeclass;
return *this;
}
void set_boundary()
{
meta |= BOUNDARY_BIT;
}
[[nodiscard]] bool is_boundary() const
{
return meta & BOUNDARY_BIT;
}
bool clear_boundary_bit()
{
return meta &= ~BOUNDARY_BIT;
}
};
sizeof(Metaslab) == PAGEMAP_METADATA_STRUCT_SIZE,
"Metaslab is expected to be the largest pagemap metadata record");
struct MetaslabCache
{
@@ -363,4 +224,41 @@ namespace snmalloc
uint16_t unused = 0;
uint16_t length = 0;
};
/*
* MetaEntry methods that deal with RemoteAllocator* and sizeclass_t are here,
* so that the backend does not need to know the details and can, instead,
* just provide the storage space.
*/
SNMALLOC_FAST_PATH_INLINE
MetaEntry::MetaEntry(
Metaslab* meta,
RemoteAllocator* remote,
sizeclass_t sizeclass = sizeclass_t())
: meta(reinterpret_cast<uintptr_t>(meta))
{
/* remote might be nullptr; cast to uintptr_t before offsetting */
remote_and_sizeclass =
pointer_offset(reinterpret_cast<uintptr_t>(remote), sizeclass.raw());
}
[[nodiscard]] SNMALLOC_FAST_PATH_INLINE RemoteAllocator*
MetaEntry::get_remote() const
{
return reinterpret_cast<RemoteAllocator*>(
pointer_align_down<REMOTE_WITH_BACKEND_MARKER_ALIGN>(
remote_and_sizeclass));
}
[[nodiscard]] SNMALLOC_FAST_PATH_INLINE sizeclass_t
MetaEntry::get_sizeclass() const
{
// TODO: perhaps remove static_cast with resolution of
// https://github.com/CTSRD-CHERI/llvm-project/issues/588
return sizeclass_t::from_raw(
static_cast<size_t>(remote_and_sizeclass) &
(REMOTE_WITH_BACKEND_MARKER_ALIGN - 1));
}
} // namespace snmalloc