Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

RFC, NFC: refactor ranges to be nested templates (#535)

Browse Source 
This way, we don't have to specify a Parent when we're just interested in
Pipe-ing things together.

We could have called these inner classes Apply and left the Pipe implementation
alone, but it's probably better to call them Type and adjust the Pipe code.
This commit is contained in:
Nathaniel Filardo
2022-06-06 12:04:37 +01:00
committed by GitHub
parent 9464556129
commit 392acff4d5
11 changed files with 452 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/snmalloc/backend/meta_protected_range.h
View File

@@ -38,7 +38,7 @@ namespace snmalloc
Pagemap,
MinSizeBits>,
LogRange<2>,
GlobalRange<>>;
GlobalRange>;
static constexpr size_t page_size_bits =
bits::next_pow2_bits_const(PAL::page_size);
@@ -53,9 +53,9 @@ namespace snmalloc
GlobalR,
LargeBuddyRange<GlobalCacheSizeBits, bits::BITS - 1, Pagemap>,
LogRange<3>,
GlobalRange<>,
GlobalRange,
CommitRange<PAL>,
StatsRange<>>;
StatsRange>;
// Controls the padding around the meta-data range.
// The larger the padding range the more randomisation that
@@ -81,8 +81,8 @@ namespace snmalloc
Pagemap,
page_size_bits>,
LogRange<4>,
GlobalRange<>,
StatsRange<>>;
GlobalRange,
StatsRange>;
// Local caching of object range
using ObjectRange = Pipe<
@@ -101,7 +101,7 @@ namespace snmalloc
LocalCacheSizeBits - SubRangeRatioBits,
bits::BITS - 1,
Pagemap>,
SmallBuddyRange<>>;
SmallBuddyRange>;
public:
using Stats = StatsCombiner<CentralObjectRange, CentralMetaRange>;
@@ -119,6 +119,6 @@ namespace snmalloc
// Don't want to add the SmallBuddyRange to the CentralMetaRange as that
// would require committing memory inside the main global lock.
using GlobalMetaRange =
Pipe<CentralMetaRange, SmallBuddyRange<>, GlobalRange<>>;
Pipe<CentralMetaRange, SmallBuddyRange, GlobalRange>;
};
} // namespace snmalloc
} // namespace snmalloc

10
src/snmalloc/backend/standard_range.h
View File

@@ -35,10 +35,10 @@ namespace snmalloc
Pagemap,
MinSizeBits>,
LogRange<2>,
GlobalRange<>>;
GlobalRange>;
// Track stats of the committed memory
using Stats = Pipe<GlobalR, CommitRange<PAL>, StatsRange<>>;
using Stats = Pipe<GlobalR, CommitRange<PAL>, StatsRange>;
private:
static constexpr size_t page_size_bits =
@@ -53,11 +53,11 @@ namespace snmalloc
LocalCacheSizeBits,
Pagemap,
page_size_bits>,
SmallBuddyRange<>>;
SmallBuddyRange>;
public:
// Expose a global range for the initial allocation of meta-data.
using GlobalMetaRange = Pipe<ObjectRange, GlobalRange<>>;
using GlobalMetaRange = Pipe<ObjectRange, GlobalRange>;
// Where we get user allocations from.
ObjectRange object_range;
@@ -69,4 +69,4 @@ namespace snmalloc
return object_range;
}
};
} // namespace snmalloc
} // namespace snmalloc

74
src/snmalloc/backend_helpers/commitrange.h
View File

@@ -5,46 +5,44 @@
namespace snmalloc
{
template<typename PAL, typename ParentRange = EmptyRange>
class CommitRange : public ContainsParent<ParentRange>
template<typename PAL>
struct CommitRange
{
using ContainsParent<ParentRange>::parent;
public:
/**
* We use a nested Apply type to enable a Pipe operation.
*/
template<typename ParentRange2>
using Apply = CommitRange<PAL, ParentRange2>;
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe;
constexpr CommitRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
template<typename ParentRange>
class Type : public ContainsParent<ParentRange>
{
SNMALLOC_ASSERT_MSG(
(size % PAL::page_size) == 0,
"size ({}) must be a multiple of page size ({})",
size,
PAL::page_size);
auto range = parent.alloc_range(size);
if (range != nullptr)
PAL::template notify_using<NoZero>(range.unsafe_ptr(), size);
return range;
}
using ContainsParent<ParentRange>::parent;
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
SNMALLOC_ASSERT_MSG(
(size % PAL::page_size) == 0,
"size ({}) must be a multiple of page size ({})",
size,
PAL::page_size);
PAL::notify_not_using(base.unsafe_ptr(), size);
parent.dealloc_range(base, size);
}
public:
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe;
constexpr Type() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
SNMALLOC_ASSERT_MSG(
(size % PAL::page_size) == 0,
"size ({}) must be a multiple of page size ({})",
size,
PAL::page_size);
auto range = parent.alloc_range(size);
if (range != nullptr)
PAL::template notify_using<NoZero>(range.unsafe_ptr(), size);
return range;
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
SNMALLOC_ASSERT_MSG(
(size % PAL::page_size) == 0,
"size ({}) must be a multiple of page size ({})",
size,
PAL::page_size);
PAL::notify_not_using(base.unsafe_ptr(), size);
parent.dealloc_range(base, size);
}
};
};
} // namespace snmalloc

61
src/snmalloc/backend_helpers/globalrange.h
View File

@@ -9,40 +9,37 @@ namespace snmalloc
* Makes the supplied ParentRange into a global variable,
* and protects access with a lock.
*/
template<typename ParentRange = EmptyRange>
class GlobalRange : public StaticParent<ParentRange>
struct GlobalRange
{
using StaticParent<ParentRange>::parent;
/**
* This is infrequently used code, a spin lock simplifies the code
* considerably, and should never be on the fast path.
*/
SNMALLOC_REQUIRE_CONSTINIT static inline FlagWord spin_lock{};
public:
/**
* We use a nested Apply type to enable a Pipe operation.
*/
template<typename ParentRange2>
using Apply = GlobalRange<ParentRange2>;
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = true;
constexpr GlobalRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
template<typename ParentRange = EmptyRange>
class Type : public StaticParent<ParentRange>
{
FlagLock lock(spin_lock);
return parent.alloc_range(size);
}
using StaticParent<ParentRange>::parent;
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
FlagLock lock(spin_lock);
parent.dealloc_range(base, size);
}
/**
* This is infrequently used code, a spin lock simplifies the code
* considerably, and should never be on the fast path.
*/
SNMALLOC_REQUIRE_CONSTINIT static inline FlagWord spin_lock{};
public:
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = true;
constexpr Type() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
FlagLock lock(spin_lock);
return parent.alloc_range(size);
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
FlagLock lock(spin_lock);
parent.dealloc_range(base, size);
}
};
};
} // namespace snmalloc

329
src/snmalloc/backend_helpers/largebuddyrange.h
View File

@@ -187,12 +187,9 @@ namespace snmalloc
size_t REFILL_SIZE_BITS,
size_t MAX_SIZE_BITS,
SNMALLOC_CONCEPT(IsWritablePagemap) Pagemap,
size_t MIN_REFILL_SIZE_BITS = 0,
typename ParentRange = EmptyRange>
class LargeBuddyRange : public ContainsParent<ParentRange>
size_t MIN_REFILL_SIZE_BITS = 0>
class LargeBuddyRange
{
using ContainsParent<ParentRange>::parent;
static_assert(
REFILL_SIZE_BITS <= MAX_SIZE_BITS, "REFILL_SIZE_BITS > MAX_SIZE_BITS");
static_assert(
@@ -210,186 +207,182 @@ namespace snmalloc
static constexpr size_t MIN_REFILL_SIZE =
bits::one_at_bit(MIN_REFILL_SIZE_BITS);
/**
* The size of memory requested so far.
*
* This is used to determine the refill size.
*/
size_t requested_total = 0;
/**
* Buddy allocator used to represent this range of memory.
*/
Buddy<BuddyChunkRep<Pagemap>, MIN_CHUNK_BITS, MAX_SIZE_BITS> buddy_large;
/**
* The parent might not support deallocation if this buddy allocator covers
* the whole range. Uses template insanity to make this work.
*/
template<bool exists = MAX_SIZE_BITS != (bits::BITS - 1)>
std::enable_if_t<exists>
parent_dealloc_range(capptr::Chunk<void> base, size_t size)
{
static_assert(
MAX_SIZE_BITS != (bits::BITS - 1), "Don't set SFINAE parameter");
parent.dealloc_range(base, size);
}
void dealloc_overflow(capptr::Chunk<void> overflow)
{
if constexpr (MAX_SIZE_BITS != (bits::BITS - 1))
{
if (overflow != nullptr)
{
parent.dealloc_range(overflow, bits::one_at_bit(MAX_SIZE_BITS));
}
}
else
{
if (overflow != nullptr)
abort();
}
}
/**
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(capptr::Chunk<void> base, size_t length)
{
range_to_pow_2_blocks<MIN_CHUNK_BITS>(
base, length, [this](capptr::Chunk<void> base, size_t align, bool) {
auto overflow = capptr::Chunk<void>(reinterpret_cast<void*>(
buddy_large.add_block(base.unsafe_uintptr(), align)));
dealloc_overflow(overflow);
});
}
capptr::Chunk<void> refill(size_t size)
{
if (ParentRange::Aligned)
{
// Use amount currently requested to determine refill size.
// This will gradually increase the usage of the parent range.
// So small examples can grow local caches slowly, and larger
// examples will grow them by the refill size.
//
// The heuristic is designed to allocate the following sequence for
// 16KiB requests 16KiB, 16KiB, 32Kib, 64KiB, ..., REFILL_SIZE/2,
// REFILL_SIZE, REFILL_SIZE, ... Hence if this if they are coming from a
// contiguous aligned range, then they could be consolidated. This
// depends on the ParentRange behaviour.
size_t refill_size = bits::min(REFILL_SIZE, requested_total);
refill_size = bits::max(refill_size, MIN_REFILL_SIZE);
refill_size = bits::max(refill_size, size);
refill_size = bits::next_pow2(refill_size);
auto refill_range = parent.alloc_range(refill_size);
if (refill_range != nullptr)
{
requested_total += refill_size;
add_range(pointer_offset(refill_range, size), refill_size - size);
}
return refill_range;
}
// Note the unaligned parent path does not use
// requested_total in the heuristic for the initial size
// this is because the request needs to introduce alignment.
// Currently the unaligned variant is not used as a local cache.
// So the gradual growing of refill_size is not needed.
// Need to overallocate to get the alignment right.
bool overflow = false;
size_t needed_size = bits::umul(size, 2, overflow);
if (overflow)
{
return nullptr;
}
auto refill_size = bits::max(needed_size, REFILL_SIZE);
while (needed_size <= refill_size)
{
auto refill = parent.alloc_range(refill_size);
if (refill != nullptr)
{
requested_total += refill_size;
add_range(refill, refill_size);
SNMALLOC_ASSERT(refill_size < bits::one_at_bit(MAX_SIZE_BITS));
static_assert(
(REFILL_SIZE < bits::one_at_bit(MAX_SIZE_BITS)) ||
ParentRange::Aligned,
"Required to prevent overflow.");
return alloc_range(size);
}
refill_size >>= 1;
}
return nullptr;
}
public:
/**
* We use a nested Apply type to enable a Pipe operation.
*/
template<typename ParentRange2>
using Apply = LargeBuddyRange<
REFILL_SIZE_BITS,
MAX_SIZE_BITS,
Pagemap,
MIN_REFILL_SIZE_BITS,
ParentRange2>;
static constexpr bool Aligned = true;
static constexpr bool ConcurrencySafe = false;
constexpr LargeBuddyRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
template<typename ParentRange = EmptyRange>
class Type : public ContainsParent<ParentRange>
{
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
SNMALLOC_ASSERT(bits::is_pow2(size));
using ContainsParent<ParentRange>::parent;
if (size >= (bits::one_at_bit(MAX_SIZE_BITS) - 1))
/**
* The size of memory requested so far.
*
* This is used to determine the refill size.
*/
size_t requested_total = 0;
/**
* Buddy allocator used to represent this range of memory.
*/
Buddy<BuddyChunkRep<Pagemap>, MIN_CHUNK_BITS, MAX_SIZE_BITS> buddy_large;
/**
* The parent might not support deallocation if this buddy allocator
* covers the whole range. Uses template insanity to make this work.
*/
template<bool exists = MAX_SIZE_BITS != (bits::BITS - 1)>
std::enable_if_t<exists>
parent_dealloc_range(capptr::Chunk<void> base, size_t size)
{
static_assert(
MAX_SIZE_BITS != (bits::BITS - 1), "Don't set SFINAE parameter");
parent.dealloc_range(base, size);
}
void dealloc_overflow(capptr::Chunk<void> overflow)
{
if constexpr (MAX_SIZE_BITS != (bits::BITS - 1))
{
if (overflow != nullptr)
{
parent.dealloc_range(overflow, bits::one_at_bit(MAX_SIZE_BITS));
}
}
else
{
if (overflow != nullptr)
abort();
}
}
/**
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(capptr::Chunk<void> base, size_t length)
{
range_to_pow_2_blocks<MIN_CHUNK_BITS>(
base, length, [this](capptr::Chunk<void> base, size_t align, bool) {
auto overflow = capptr::Chunk<void>(reinterpret_cast<void*>(
buddy_large.add_block(base.unsafe_uintptr(), align)));
dealloc_overflow(overflow);
});
}
capptr::Chunk<void> refill(size_t size)
{
if (ParentRange::Aligned)
return parent.alloc_range(size);
{
// Use amount currently requested to determine refill size.
// This will gradually increase the usage of the parent range.
// So small examples can grow local caches slowly, and larger
// examples will grow them by the refill size.
//
// The heuristic is designed to allocate the following sequence for
// 16KiB requests 16KiB, 16KiB, 32Kib, 64KiB, ..., REFILL_SIZE/2,
// REFILL_SIZE, REFILL_SIZE, ... Hence if this if they are coming from
// a contiguous aligned range, then they could be consolidated. This
// depends on the ParentRange behaviour.
size_t refill_size = bits::min(REFILL_SIZE, requested_total);
refill_size = bits::max(refill_size, MIN_REFILL_SIZE);
refill_size = bits::max(refill_size, size);
refill_size = bits::next_pow2(refill_size);
auto refill_range = parent.alloc_range(refill_size);
if (refill_range != nullptr)
{
requested_total += refill_size;
add_range(pointer_offset(refill_range, size), refill_size - size);
}
return refill_range;
}
// Note the unaligned parent path does not use
// requested_total in the heuristic for the initial size
// this is because the request needs to introduce alignment.
// Currently the unaligned variant is not used as a local cache.
// So the gradual growing of refill_size is not needed.
// Need to overallocate to get the alignment right.
bool overflow = false;
size_t needed_size = bits::umul(size, 2, overflow);
if (overflow)
{
return nullptr;
}
auto refill_size = bits::max(needed_size, REFILL_SIZE);
while (needed_size <= refill_size)
{
auto refill = parent.alloc_range(refill_size);
if (refill != nullptr)
{
requested_total += refill_size;
add_range(refill, refill_size);
SNMALLOC_ASSERT(refill_size < bits::one_at_bit(MAX_SIZE_BITS));
static_assert(
(REFILL_SIZE < bits::one_at_bit(MAX_SIZE_BITS)) ||
ParentRange::Aligned,
"Required to prevent overflow.");
return alloc_range(size);
}
refill_size >>= 1;
}
return nullptr;
}
auto result = capptr::Chunk<void>(
reinterpret_cast<void*>(buddy_large.remove_block(size)));
public:
static constexpr bool Aligned = true;
if (result != nullptr)
return result;
static constexpr bool ConcurrencySafe = false;
return refill(size);
}
constexpr Type() = default;
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
SNMALLOC_ASSERT(bits::is_pow2(size));
if constexpr (MAX_SIZE_BITS != (bits::BITS - 1))
capptr::Chunk<void> alloc_range(size_t size)
{
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
SNMALLOC_ASSERT(bits::is_pow2(size));
if (size >= (bits::one_at_bit(MAX_SIZE_BITS) - 1))
{
parent_dealloc_range(base, size);
return;
if (ParentRange::Aligned)
return parent.alloc_range(size);
return nullptr;
}
auto result = capptr::Chunk<void>(
reinterpret_cast<void*>(buddy_large.remove_block(size)));
if (result != nullptr)
return result;
return refill(size);
}
auto overflow = capptr::Chunk<void>(reinterpret_cast<void*>(
buddy_large.add_block(base.unsafe_uintptr(), size)));
dealloc_overflow(overflow);
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
SNMALLOC_ASSERT(bits::is_pow2(size));
if constexpr (MAX_SIZE_BITS != (bits::BITS - 1))
{
if (size >= (bits::one_at_bit(MAX_SIZE_BITS) - 1))
{
parent_dealloc_range(base, size);
return;
}
}
auto overflow = capptr::Chunk<void>(reinterpret_cast<void*>(
buddy_large.add_block(base.unsafe_uintptr(), size)));
dealloc_overflow(overflow);
}
};
};
} // namespace snmalloc

78
src/snmalloc/backend_helpers/logrange.h
View File

@@ -11,51 +11,49 @@ namespace snmalloc
*
* ParentRange is what the range is logging calls to.
*/
template<size_t RangeName, typename ParentRange = EmptyRange>
class LogRange : public ContainsParent<ParentRange>
template<size_t RangeName>
struct LogRange
{
using ContainsParent<ParentRange>::parent;
public:
/**
* We use a nested Apply type to enable a Pipe operation.
*/
template<typename ParentRange2>
using Apply = LogRange<RangeName, ParentRange2>;
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe;
constexpr LogRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
template<typename ParentRange = EmptyRange>
class Type : public ContainsParent<ParentRange>
{
#ifdef SNMALLOC_TRACING
message<1024>("Call alloc_range({}) on {}", size, RangeName);
#endif
auto range = parent.alloc_range(size);
#ifdef SNMALLOC_TRACING
message<1024>(
"{} = alloc_range({}) in {}", range.unsafe_ptr(), size, RangeName);
#endif
return range;
}
using ContainsParent<ParentRange>::parent;
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
public:
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe;
constexpr Type() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
#ifdef SNMALLOC_TRACING
message<1024>(
"dealloc_range({}, {}}) on {}", base.unsafe_ptr(), size, RangeName);
message<1024>("Call alloc_range({}) on {}", size, RangeName);
#endif
parent.dealloc_range(base, size);
auto range = parent.alloc_range(size);
#ifdef SNMALLOC_TRACING
message<1024>(
"Done dealloc_range({}, {}})! on {}",
base.unsafe_ptr(),
size,
RangeName);
message<1024>(
"{} = alloc_range({}) in {}", range.unsafe_ptr(), size, RangeName);
#endif
}
return range;
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
#ifdef SNMALLOC_TRACING
message<1024>(
"dealloc_range({}, {}}) on {}", base.unsafe_ptr(), size, RangeName);
#endif
parent.dealloc_range(base, size);
#ifdef SNMALLOC_TRACING
message<1024>(
"Done dealloc_range({}, {}})! on {}",
base.unsafe_ptr(),
size,
RangeName);
#endif
}
};
};
} // namespace snmalloc
} // namespace snmalloc

55
src/snmalloc/backend_helpers/pagemapregisterrange.h
View File

@@ -8,40 +8,37 @@ namespace snmalloc
{
template<
SNMALLOC_CONCEPT(IsWritablePagemapWithRegister) Pagemap,
bool CanConsolidate = true,
typename ParentRange = EmptyRange>
class PagemapRegisterRange : public ContainsParent<ParentRange>
bool CanConsolidate = true>
struct PagemapRegisterRange
{
using ContainsParent<ParentRange>::parent;
public:
/**
* We use a nested Apply type to enable a Pipe operation.
*/
template<typename ParentRange2>
using Apply = PagemapRegisterRange<Pagemap, CanConsolidate, ParentRange2>;
constexpr PagemapRegisterRange() = default;
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe;
capptr::Chunk<void> alloc_range(size_t size)
template<typename ParentRange = EmptyRange>
class Type : public ContainsParent<ParentRange>
{
auto base = parent.alloc_range(size);
using ContainsParent<ParentRange>::parent;
if (base != nullptr)
Pagemap::register_range(address_cast(base), size);
public:
constexpr Type() = default;
if (!CanConsolidate)
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe;
capptr::Chunk<void> alloc_range(size_t size)
{
// Mark start of allocation in pagemap.
auto& entry = Pagemap::get_metaentry_mut(address_cast(base));
entry.set_boundary();
}
auto base = parent.alloc_range(size);
return base;
}
if (base != nullptr)
Pagemap::register_range(address_cast(base), size);
if (!CanConsolidate)
{
// Mark start of allocation in pagemap.
auto& entry = Pagemap::get_metaentry_mut(address_cast(base));
entry.set_boundary();
}
return base;
}
};
};
} // namespace snmalloc

2
src/snmalloc/backend_helpers/range_helpers.h
View File

@@ -61,7 +61,7 @@ namespace snmalloc
{
public:
using result =
typename PipeImpl<typename Fun::template Apply<First>, Rest...>::result;
typename PipeImpl<typename Fun::template Type<First>, Rest...>::result;
};
/**

159
src/snmalloc/backend_helpers/smallbuddyrange.h
View File

@@ -143,100 +143,97 @@ namespace snmalloc
}
};
template<typename ParentRange = EmptyRange>
class SmallBuddyRange : public ContainsParent<ParentRange>
struct SmallBuddyRange
{
using ContainsParent<ParentRange>::parent;
static constexpr size_t MIN_BITS =
bits::next_pow2_bits_const(sizeof(FreeChunk));
Buddy<BuddyInplaceRep, MIN_BITS, MIN_CHUNK_BITS> buddy_small;
/**
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(capptr::Chunk<void> base, size_t length)
template<typename ParentRange = EmptyRange>
class Type : public ContainsParent<ParentRange>
{
range_to_pow_2_blocks<MIN_BITS>(
base, length, [this](capptr::Chunk<void> base, size_t align, bool) {
capptr::Chunk<void> overflow =
buddy_small.add_block(base.as_reinterpret<FreeChunk>(), align)
.template as_reinterpret<void>();
if (overflow != nullptr)
parent.dealloc_range(overflow, bits::one_at_bit(MIN_CHUNK_BITS));
});
}
using ContainsParent<ParentRange>::parent;
capptr::Chunk<void> refill(size_t size)
{
auto refill = parent.alloc_range(MIN_CHUNK_SIZE);
static constexpr size_t MIN_BITS =
bits::next_pow2_bits_const(sizeof(FreeChunk));
if (refill != nullptr)
add_range(pointer_offset(refill, size), MIN_CHUNK_SIZE - size);
Buddy<BuddyInplaceRep, MIN_BITS, MIN_CHUNK_BITS> buddy_small;
return refill;
}
public:
/**
* We use a nested Apply type to enable a Pipe operation.
*/
template<typename ParentRange2>
using Apply = SmallBuddyRange<ParentRange2>;
static constexpr bool Aligned = true;
static_assert(ParentRange::Aligned, "ParentRange must be aligned");
static constexpr bool ConcurrencySafe = false;
constexpr SmallBuddyRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
if (size >= MIN_CHUNK_SIZE)
/**
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(capptr::Chunk<void> base, size_t length)
{
return parent.alloc_range(size);
range_to_pow_2_blocks<MIN_BITS>(
base, length, [this](capptr::Chunk<void> base, size_t align, bool) {
capptr::Chunk<void> overflow =
buddy_small.add_block(base.as_reinterpret<FreeChunk>(), align)
.template as_reinterpret<void>();
if (overflow != nullptr)
parent.dealloc_range(overflow, bits::one_at_bit(MIN_CHUNK_BITS));
});
}
auto result = buddy_small.remove_block(size);
if (result != nullptr)
capptr::Chunk<void> refill(size_t size)
{
result->left = nullptr;
result->right = nullptr;
auto refill = parent.alloc_range(MIN_CHUNK_SIZE);
if (refill != nullptr)
add_range(pointer_offset(refill, size), MIN_CHUNK_SIZE - size);
return refill;
}
public:
static constexpr bool Aligned = true;
static_assert(ParentRange::Aligned, "ParentRange must be aligned");
static constexpr bool ConcurrencySafe = false;
constexpr Type() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
if (size >= MIN_CHUNK_SIZE)
{
return parent.alloc_range(size);
}
auto result = buddy_small.remove_block(size);
if (result != nullptr)
{
result->left = nullptr;
result->right = nullptr;
return result.template as_reinterpret<void>();
}
return refill(size);
}
capptr::Chunk<void> alloc_range_with_leftover(size_t size)
{
SNMALLOC_ASSERT(size <= MIN_CHUNK_SIZE);
auto rsize = bits::next_pow2(size);
auto result = alloc_range(rsize);
if (result == nullptr)
return nullptr;
auto remnant = pointer_offset(result, size);
add_range(remnant, rsize - size);
return result.template as_reinterpret<void>();
}
return refill(size);
}
capptr::Chunk<void> alloc_range_with_leftover(size_t size)
{
SNMALLOC_ASSERT(size <= MIN_CHUNK_SIZE);
auto rsize = bits::next_pow2(size);
auto result = alloc_range(rsize);
if (result == nullptr)
return nullptr;
auto remnant = pointer_offset(result, size);
add_range(remnant, rsize - size);
return result.template as_reinterpret<void>();
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
if (size >= MIN_CHUNK_SIZE)
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
parent.dealloc_range(base, size);
return;
}
if (size >= MIN_CHUNK_SIZE)
{
parent.dealloc_range(base, size);
return;
}
add_range(base, size);
}
add_range(base, size);
}
};
};
} // namespace snmalloc

83
src/snmalloc/backend_helpers/statsrange.h
View File

@@ -10,58 +10,55 @@ namespace snmalloc
/**
* Used to measure memory usage.
*/
template<typename ParentRange = EmptyRange>
class StatsRange : public ContainsParent<ParentRange>
struct StatsRange
{
using ContainsParent<ParentRange>::parent;
static inline std::atomic<size_t> current_usage{};
static inline std::atomic<size_t> peak_usage{};
public:
/**
* We use a nested Apply type to enable a Pipe operation.
*/
template<typename ParentRange2>
using Apply = StatsRange<ParentRange2>;
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe;
constexpr StatsRange() = default;
capptr::Chunk<void> alloc_range(size_t size)
template<typename ParentRange = EmptyRange>
class Type : public ContainsParent<ParentRange>
{
auto result = parent.alloc_range(size);
if (result != nullptr)
using ContainsParent<ParentRange>::parent;
static inline std::atomic<size_t> current_usage{};
static inline std::atomic<size_t> peak_usage{};
public:
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe;
constexpr Type() = default;
capptr::Chunk<void> alloc_range(size_t size)
{
auto prev = current_usage.fetch_add(size);
auto curr = peak_usage.load();
while (curr < prev + size)
auto result = parent.alloc_range(size);
if (result != nullptr)
{
if (peak_usage.compare_exchange_weak(curr, prev + size))
break;
auto prev = current_usage.fetch_add(size);
auto curr = peak_usage.load();
while (curr < prev + size)
{
if (peak_usage.compare_exchange_weak(curr, prev + size))
break;
}
}
return result;
}
return result;
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
current_usage -= size;
parent.dealloc_range(base, size);
}
void dealloc_range(capptr::Chunk<void> base, size_t size)
{
current_usage -= size;
parent.dealloc_range(base, size);
}
size_t get_current_usage()
{
return current_usage.load();
}
size_t get_current_usage()
{
return current_usage.load();
}
size_t get_peak_usage()
{
return peak_usage.load();
}
size_t get_peak_usage()
{
return peak_usage.load();
}
};
};
template<typename StatsR1, typename StatsR2>

62
src/snmalloc/backend_helpers/subrange.h
View File

@@ -9,43 +9,41 @@ namespace snmalloc
* 2^RATIO_BITS. Will not return a the block at the start or
* the end of the large allocation.
*/
template<typename PAL, size_t RATIO_BITS, typename ParentRange = EmptyRange>
class SubRange : public ContainsParent<ParentRange>
template<typename PAL, size_t RATIO_BITS>
struct SubRange
{
using ContainsParent<ParentRange>::parent;
public:
/**
* We use a nested Apply type to enable a Pipe operation.
*/
template<typename ParentRange2>
using Apply = SubRange<PAL, RATIO_BITS, ParentRange2>;
constexpr SubRange() = default;
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe;
capptr::Chunk<void> alloc_range(size_t sub_size)
template<typename ParentRange = EmptyRange>
class Type : public ContainsParent<ParentRange>
{
SNMALLOC_ASSERT(bits::is_pow2(sub_size));
using ContainsParent<ParentRange>::parent;
auto full_size = sub_size << RATIO_BITS;
auto overblock = parent.alloc_range(full_size);
if (overblock == nullptr)
return nullptr;
public:
constexpr Type() = default;
size_t offset_mask = full_size - sub_size;
// Don't use first or last block in the larger reservation
// Loop required to get uniform distribution.
size_t offset;
do
static constexpr bool Aligned = ParentRange::Aligned;
static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe;
capptr::Chunk<void> alloc_range(size_t sub_size)
{
offset = get_entropy64<PAL>() & offset_mask;
} while ((offset == 0) || (offset == offset_mask));
SNMALLOC_ASSERT(bits::is_pow2(sub_size));
return pointer_offset(overblock, offset);
}
auto full_size = sub_size << RATIO_BITS;
auto overblock = parent.alloc_range(full_size);
if (overblock == nullptr)
return nullptr;
size_t offset_mask = full_size - sub_size;
// Don't use first or last block in the larger reservation
// Loop required to get uniform distribution.
size_t offset;
do
{
offset = get_entropy64<PAL>() & offset_mask;
} while ((offset == 0) || (offset == offset_mask));
return pointer_offset(overblock, offset);
}
};
};
} // namespace snmalloc