Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expose Entropy

Browse Source 
Define various parts of random that can be used to make the layout of
memory more random.  Thread this through the allocator.

Expose the concept as part of the Pal. Subsequent commits will expose
that on different platforms.
This commit is contained in:
Matthew Parkinson
2021-03-26 16:22:32 +00:00
committed by Matthew Parkinson
parent c5b65d07b8
commit f15dc6ee2e
8 changed files with 186 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/mem/alloc.h
View File

@@ -98,6 +98,7 @@ namespace snmalloc
{
LargeAlloc<MemoryProvider> large_allocator;
ChunkMap chunk_map;
LocalEntropy entropy;
/**
* Per size class bumpptr for building new free lists
@@ -683,6 +684,11 @@ namespace snmalloc
if (isFake)
return;
// Entropy must be first, so that all data-structures can use the key
// it generates.
// This must occur before any freelists are constructed.
entropy.init<typename MemoryProvider::Pal>();
init_message_queue();
message_queue().invariant();
@@ -745,10 +751,10 @@ namespace snmalloc
Slab* slab = Metaslab::get_slab(bp);
while (pointer_align_up(bp, SLAB_SIZE) != bp)
{
Slab::alloc_new_list(bp, ffl, rsize);
Slab::alloc_new_list(bp, ffl, rsize, entropy);
while (!ffl.empty())
{
small_dealloc_offseted_inner(super, slab, ffl.take(), i);
small_dealloc_offseted_inner(super, slab, ffl.take(entropy), i);
}
}
}
@@ -762,7 +768,7 @@ namespace snmalloc
auto slab = Metaslab::get_slab(head);
do
{
auto curr = small_fast_free_lists[i].take();
auto curr = small_fast_free_lists[i].take(entropy);
small_dealloc_offseted_inner(super, slab, curr, i);
} while (!small_fast_free_lists[i].empty());
@@ -1016,7 +1022,7 @@ namespace snmalloc
{
stats().alloc_request(size);
stats().sizeclass_alloc(sizeclass);
void* p = remove_cache_friendly_offset(fl.take(), sizeclass);
void* p = remove_cache_friendly_offset(fl.take(entropy), sizeclass);
if constexpr (zero_mem == YesZero)
{
MemoryProvider::Pal::zero(p, sizeclass_to_size(sizeclass));
@@ -1061,7 +1067,7 @@ namespace snmalloc
auto meta = reinterpret_cast<Metaslab*>(sl.get_next());
auto& ffl = small_fast_free_lists[sizeclass];
return Metaslab::alloc<zero_mem, typename MemoryProvider::Pal>(
meta, ffl, rsize);
meta, ffl, rsize, entropy);
}
return small_alloc_rare<zero_mem>(sizeclass, size);
}
@@ -1125,9 +1131,9 @@ namespace snmalloc
auto rsize = sizeclass_to_size(sizeclass);
auto& ffl = small_fast_free_lists[sizeclass];
SNMALLOC_ASSERT(ffl.empty());
Slab::alloc_new_list(bp, ffl, rsize);
Slab::alloc_new_list(bp, ffl, rsize, entropy);
auto p = remove_cache_friendly_offset(ffl.take(), sizeclass);
auto p = remove_cache_friendly_offset(ffl.take(entropy), sizeclass);
if constexpr (zero_mem == YesZero)
{
@@ -1213,7 +1219,7 @@ namespace snmalloc
SNMALLOC_FAST_PATH void small_dealloc_offseted_inner(
Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass)
{
if (likely(Slab::dealloc_fast(slab, super, p)))
if (likely(Slab::dealloc_fast(slab, super, p, entropy)))
return;
small_dealloc_offseted_slow(super, slab, p, sizeclass);
@@ -1224,7 +1230,7 @@ namespace snmalloc
{
bool was_full = super->is_full();
SlabList* sl = &small_classes[sizeclass];
Superslab::Action a = Slab::dealloc_slow(slab, sl, super, p);
Superslab::Action a = Slab::dealloc_slow(slab, sl, super, p, entropy);
if (likely(a == Superslab::NoSlabReturn))
return;
stats().sizeclass_dealloc_slab(sizeclass);

2
src/mem/chunkmap.h
View File

@@ -72,7 +72,7 @@ namespace snmalloc
* internal tree node in the non-flat pagemap).
*/
# define SNMALLOC_MAX_FLATPAGEMAP_SIZE \
(pal_supports<LazyCommit> ? 256ULL * 1024 * 1024 : PAGEMAP_NODE_SIZE)
(pal_supports<LazyCommit, Pal> ? 256ULL * 1024 * 1024 : PAGEMAP_NODE_SIZE)
#endif
static constexpr bool CHUNKMAP_USE_FLATPAGEMAP =
SNMALLOC_MAX_FLATPAGEMAP_SIZE >=

96
src/mem/entropy.h Normal file
View File

@@ -0,0 +1,96 @@
#include "../ds/address.h"
#include <cstdint>
#include <random>
#include <type_traits>
namespace snmalloc
{
template<typename PAL>
std::enable_if_t<pal_supports<Entropy, PAL>, uint64_t> get_entropy64()
{
return PAL::get_entropy64();
}
template<typename PAL>
std::enable_if_t<!pal_supports<Entropy, PAL>, uint64_t> get_entropy64()
{
std::random_device rd;
uint64_t a = rd();
return (a << 32) ^ rd();
}
class LocalEntropy
{
uint64_t bit_source;
uint64_t local_key;
uint64_t local_counter;
address_t constant_key;
public:
template<typename PAL>
void init()
{
local_key = get_entropy64<PAL>();
local_counter = get_entropy64<PAL>();
if constexpr (bits::BITS == 64)
constant_key = get_next();
else
constant_key = get_next() & 0xffff'ffff;
bit_source = get_next();
}
/**
* Returns a bit.
*
* The bit returned is cycled every 64 calls.
* This is a very cheap source of some randomness.
* Returns the bottom bit.
*/
uint32_t next_bit()
{
uint64_t bottom_bit = bit_source & 1;
bit_source = (bottom_bit << 63) | (bit_source >> 1);
return bottom_bit & 1;
}
/**
* A key that is not changed or used to create other keys
*
* This is for use when there is no storage for the key.
*/
address_t get_constant_key()
{
return constant_key;
}
/**
* Source of random 64bit values
*
* Has a 2^64 period.
*
* Applies a Feistel cipher to a counter
*/
uint64_t get_next()
{
uint64_t c = ++local_counter;
uint64_t bottom;
for (int i = 0; i < 2; i++)
{
bottom = c & 0xffff'fffff;
c = (c << 32) | (((bottom * local_key) ^ c) >> 32);
}
return c;
}
/**
* Refresh `next_bit` source of bits.
*
* This loads new entropy into the `next_bit` values.
*/
void refresh_bits()
{
bit_source = get_next();
}
};
} // namespace snmalloc

82
src/mem/freelist.h
View File

@@ -9,6 +9,7 @@
#include "../ds/dllist.h"
#include "../ds/helpers.h"
#include "allocconfig.h"
#include "entropy.h"
#include <cstdint>
@@ -16,13 +17,6 @@ namespace snmalloc
{
#ifdef CHECK_CLIENT
static constexpr std::size_t PRESERVE_BOTTOM_BITS = 16;
/**
* The key that is used to encode free list pointers.
* This should be randomised at startup in the future.
*/
inline static address_t global_key = static_cast<std::size_t>(
bits::is64() ? 0x5a59'DEAD'BEEF'5A59 : 0x5A59'BEEF);
#endif
/**
@@ -85,7 +79,7 @@ namespace snmalloc
#ifdef CHECK_CLIENT
template<typename T = FreeObject>
static std::enable_if_t<do_encode, T*>
encode(uint16_t local_key, T* next_object)
encode(uint16_t local_key, T* next_object, LocalEntropy& entropy)
{
// Simple involutional encoding. The bottom half of each word is
// multiplied by a function of both global and local keys (the latter,
@@ -95,7 +89,7 @@ namespace snmalloc
auto next = address_cast(next_object);
constexpr address_t MASK = bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1;
// Mix in local_key
address_t key = (local_key + 1) * global_key;
address_t key = (local_key + 1) * entropy.get_constant_key();
next ^= (((next & MASK) + 1) * key) &
~(bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1);
return reinterpret_cast<FreeObject*>(next);
@@ -104,20 +98,21 @@ namespace snmalloc
template<typename T = FreeObject>
static std::enable_if_t<!do_encode, T*>
encode(uint16_t local_key, T* next_object)
encode(uint16_t local_key, T* next_object, LocalEntropy& entropy)
{
UNUSED(local_key);
UNUSED(entropy);
return next_object;
}
void store(FreeObject* value, uint16_t local_key)
void store(FreeObject* value, uint16_t local_key, LocalEntropy& entropy)
{
reference = encode(local_key, value);
reference = encode(local_key, value, entropy);
}
FreeObject* read(uint16_t local_key)
FreeObject* read(uint16_t local_key, LocalEntropy& entropy)
{
return encode(local_key, reference);
return encode(local_key, reference, entropy);
}
};
@@ -142,9 +137,9 @@ namespace snmalloc
/**
* Read the next pointer handling any required decoding of the pointer
*/
FreeObject* read_next(uint16_t key)
FreeObject* read_next(uint16_t key, LocalEntropy& entropy)
{
return next_object.read(key);
return next_object.read(key, entropy);
}
};
@@ -223,14 +218,14 @@ namespace snmalloc
/**
* Moves the iterator on, and returns the current value.
*/
void* take()
void* take(LocalEntropy& entropy)
{
#ifdef CHECK_CLIENT
check_client(
!different_slab(prev, curr), "Heap corruption - free list corrupted!");
#endif
auto c = curr;
update_cursor(curr->read_next(get_prev()));
update_cursor(curr->read_next(get_prev(), entropy));
return c;
}
};
@@ -266,7 +261,6 @@ namespace snmalloc
// In the empty case end[i] == &head[i]
// This enables branch free enqueuing.
EncodeFreeObjectReference* end[2];
uint32_t interleave;
#ifdef CHECK_CLIENT
// The bottom 16 bits of the previous pointer
uint16_t prev[2];
@@ -300,18 +294,6 @@ namespace snmalloc
static constexpr uint16_t HEAD_KEY = 1;
/**
* Rotate the bits for interleaving.
*
* Returns the bottom bit.
*/
uint32_t next_interleave()
{
uint32_t bottom_bit = interleave & 1;
interleave = (bottom_bit << 31) | (interleave >> 1);
return bottom_bit;
}
public:
FreeListBuilder()
{
@@ -324,8 +306,6 @@ namespace snmalloc
*/
void open(void* p)
{
interleave = 0xDEADBEEF; // TODO RANDOM
SNMALLOC_ASSERT(empty());
#ifdef CHECK_CLIENT
prev[0] = HEAD_KEY;
@@ -337,8 +317,6 @@ namespace snmalloc
#endif
end[0] = &head[0];
end[1] = &head[1];
SNMALLOC_ASSERT(debug_length() == 0);
}
/**
@@ -352,15 +330,15 @@ namespace snmalloc
/**
* Adds an element to the builder
*/
void add(void* n)
void add(void* n, LocalEntropy& entropy)
{
SNMALLOC_ASSERT(
!different_slab(end[0], n) || !different_slab(end[1], n) || empty());
FreeObject* next = FreeObject::make(n);
uint32_t index = next_interleave();
auto index = entropy.next_bit();
end[index]->store(next, get_prev(index));
end[index]->store(next, get_prev(index), entropy);
end[index] = &(next->next_object);
#ifdef CHECK_CLIENT
prev[index] = curr[index];
@@ -374,18 +352,18 @@ namespace snmalloc
* If this is needed in a non-debug setting then
* we should look at redesigning the queue.
*/
size_t debug_length()
size_t debug_length(LocalEntropy& entropy)
{
size_t count = 0;
for (size_t i = 0; i < 2; i++)
{
uint16_t local_prev = HEAD_KEY;
EncodeFreeObjectReference* iter = &head[i];
FreeObject* prev_obj = iter->read(local_prev);
FreeObject* prev_obj = iter->read(local_prev, entropy);
uint16_t local_curr = initial_key(prev_obj) & 0xffff;
while (end[i] != iter)
{
FreeObject* next = iter->read(local_prev);
FreeObject* next = iter->read(local_prev, entropy);
check_client(!different_slab(next, prev_obj), "Heap corruption");
local_prev = local_curr;
local_curr = address_cast(next) & 0xffff;
@@ -412,7 +390,7 @@ namespace snmalloc
*
* It is used with preserve_queue disabled by close.
*/
FreeListIter terminate(bool preserve_queue = true)
FreeListIter terminate(LocalEntropy& entropy, bool preserve_queue = true)
{
SNMALLOC_ASSERT(end[1] != &head[0]);
SNMALLOC_ASSERT(end[0] != &head[1]);
@@ -420,25 +398,25 @@ namespace snmalloc
// If second list is empty, then append is trivial.
if (end[1] == &head[1])
{
end[0]->store(nullptr, get_prev(0));
return {head[0].read(HEAD_KEY)};
end[0]->store(nullptr, get_prev(0), entropy);
return {head[0].read(HEAD_KEY, entropy)};
}
end[1]->store(nullptr, get_prev(1));
end[1]->store(nullptr, get_prev(1), entropy);
// Append 1 to 0
auto mid = head[1].read(HEAD_KEY);
end[0]->store(mid, get_prev(0));
auto mid = head[1].read(HEAD_KEY, entropy);
end[0]->store(mid, get_prev(0), entropy);
// Re-code first link in second list (if there is one).
// The first link in the second list will be encoded with initial_key,
// But that needs to be changed to the curr of the first list.
if (mid != nullptr)
{
auto mid_next = mid->read_next(initial_key(mid) & 0xffff);
mid->next_object.store(mid_next, get_curr(0));
auto mid_next = mid->read_next(initial_key(mid) & 0xffff, entropy);
mid->next_object.store(mid_next, get_curr(0), entropy);
}
auto h = head[0].read(HEAD_KEY);
auto h = head[0].read(HEAD_KEY, entropy);
// If we need to continue adding to the builder
// Set up the second list as empty,
@@ -467,9 +445,9 @@ namespace snmalloc
* Close a free list, and set the iterator parameter
* to iterate it.
*/
void close(FreeListIter& dst)
void close(FreeListIter& dst, LocalEntropy& entropy)
{
dst = terminate(false);
dst = terminate(entropy, false);
init();
}

22
src/mem/metaslab.h
View File

@@ -160,14 +160,19 @@ namespace snmalloc
* eventually annotate that pointer with additional information.
*/
template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
static SNMALLOC_FAST_PATH void*
alloc(Metaslab* self, FreeListIter& fast_free_list, size_t rsize)
static SNMALLOC_FAST_PATH void* alloc(
Metaslab* self,
FreeListIter& fast_free_list,
size_t rsize,
LocalEntropy& entropy)
{
SNMALLOC_ASSERT(rsize == sizeclass_to_size(self->sizeclass()));
SNMALLOC_ASSERT(!self->is_full());
self->free_queue.close(fast_free_list);
void* n = fast_free_list.take();
self->free_queue.close(fast_free_list, entropy);
void* n = fast_free_list.take(entropy);
entropy.refresh_bits();
// Treat stealing the free list as allocating it all.
self->remove();
@@ -176,7 +181,7 @@ namespace snmalloc
void* p = remove_cache_friendly_offset(n, self->sizeclass());
SNMALLOC_ASSERT(is_start_of_object(self, p));
self->debug_slab_invariant(Metaslab::get_slab(p));
self->debug_slab_invariant(Metaslab::get_slab(p), entropy);
if constexpr (zero_mem == YesZero)
{
@@ -193,14 +198,14 @@ namespace snmalloc
return p;
}
void debug_slab_invariant(Slab* slab)
void debug_slab_invariant(Slab* slab, LocalEntropy& entropy)
{
#if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS)
bool is_short = Metaslab::is_short(slab);
if (is_full())
{
size_t count = free_queue.debug_length();
size_t count = free_queue.debug_length(entropy);
SNMALLOC_ASSERT(count < threshold_for_waking_slab(is_short));
return;
}
@@ -216,7 +221,7 @@ namespace snmalloc
SNMALLOC_ASSERT(SLAB_SIZE > accounted_for);
// Account for list size
size_t count = free_queue.debug_length();
size_t count = free_queue.debug_length(entropy);
accounted_for += count * size;
SNMALLOC_ASSERT(count <= get_slab_capacity(sizeclass(), is_short));
@@ -234,6 +239,7 @@ namespace snmalloc
SNMALLOC_ASSERT(SLAB_SIZE == accounted_for);
#else
UNUSED(slab);
UNUSED(entropy);
#endif
}
};

33
src/mem/slab.h
View File

@@ -29,8 +29,11 @@ namespace snmalloc
* worth of allocations, or one if the allocation size is larger than a
* page.
*/
static SNMALLOC_FAST_PATH void
alloc_new_list(void*& bumpptr, FreeListIter& fast_free_list, size_t rsize)
static SNMALLOC_FAST_PATH void alloc_new_list(
void*& bumpptr,
FreeListIter& fast_free_list,
size_t rsize,
LocalEntropy& entropy)
{
void* slab_end = pointer_align_up<SLAB_SIZE>(pointer_offset(bumpptr, 1));
@@ -48,14 +51,14 @@ namespace snmalloc
void* newbumpptr = pointer_offset(bumpptr, rsize * offset);
while (newbumpptr < slab_end)
{
b.add(newbumpptr);
b.add(newbumpptr, entropy);
newbumpptr = pointer_offset(newbumpptr, rsize * start_index.size());
}
}
bumpptr = slab_end;
SNMALLOC_ASSERT(!b.empty());
b.close(fast_free_list);
b.close(fast_free_list, entropy);
}
// Returns true, if it deallocation can proceed without changing any status
@@ -65,7 +68,7 @@ namespace snmalloc
// This is pre-factored to take an explicit self parameter so that we can
// eventually annotate that pointer with additional information.
static SNMALLOC_FAST_PATH bool
dealloc_fast(Slab* self, Superslab* super, void* p)
dealloc_fast(Slab* self, Superslab* super, void* p, LocalEntropy& entropy)
{
Metaslab& meta = super->get_meta(self);
SNMALLOC_ASSERT(!meta.is_unused());
@@ -74,7 +77,7 @@ namespace snmalloc
return false;
// Update the head and the next pointer in the free list.
meta.free_queue.add(p);
meta.free_queue.add(p, entropy);
return true;
}
@@ -86,11 +89,15 @@ namespace snmalloc
//
// This is pre-factored to take an explicit self parameter so that we can
// eventually annotate that pointer with additional information.
static SNMALLOC_SLOW_PATH typename Superslab::Action
dealloc_slow(Slab* self, SlabList* sl, Superslab* super, void* p)
static SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow(
Slab* self,
SlabList* sl,
Superslab* super,
void* p,
LocalEntropy& entropy)
{
Metaslab& meta = super->get_meta(self);
meta.debug_slab_invariant(self);
meta.debug_slab_invariant(self, entropy);
if (meta.is_full())
{
@@ -106,7 +113,7 @@ namespace snmalloc
return super->dealloc_slab(self);
}
meta.free_queue.add(p);
meta.free_queue.add(p, entropy);
// Remove trigger threshold from how many we need before we have fully
// freed the slab.
meta.needed() =
@@ -114,7 +121,7 @@ namespace snmalloc
// Push on the list of slabs for this sizeclass.
sl->insert_prev(&meta);
meta.debug_slab_invariant(self);
meta.debug_slab_invariant(self, entropy);
return Superslab::NoSlabReturn;
}
@@ -123,11 +130,11 @@ namespace snmalloc
// Check free list is well-formed on platforms with
// integers as pointers.
FreeListIter fl;
meta.free_queue.close(fl);
meta.free_queue.close(fl, entropy);
while (!fl.empty())
{
fl.take();
fl.take(entropy);
count++;
}
#endif

6
src/pal/pal.h
View File

@@ -66,12 +66,6 @@ namespace snmalloc
Pal::error(str);
}
/**
* Query whether the PAL supports a specific feature.
*/
template<PalFeatures F, SNMALLOC_CONCEPT(ConceptPAL) PAL = Pal>
constexpr static bool pal_supports = (PAL::pal_features & F) == F;
// Used to keep Superslab metadata committed.
static constexpr size_t OS_PAGE_SIZE = Pal::page_size;

10
src/pal/pal_consts.h
View File

@@ -41,6 +41,10 @@ namespace snmalloc
* should be pre-allocated.
*/
NoAllocation = (1 << 3),
/**
* This Pal provides a source of Entropy
*/
Entropy = (1 << 4),
};
/**
* Flag indicating whether requested memory should be zeroed.
@@ -125,4 +129,10 @@ namespace snmalloc
}
}
};
/**
* Query whether the PAL supports a specific feature.
*/
template<PalFeatures F, typename PAL>
constexpr static bool pal_supports = (PAL::pal_features & F) == F;
} // namespace snmalloc