Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implementing full permutation of slab

Browse Source 
Allocate slab is randomly in all possible permutations. This increases
the entropy of the order considerably.  This uses an algorithm to build
a random cycle in a slab, and then use this to build the free list.

We disable the per-slab randomisation in the non-CHECK_CLIENT builds.
This commit is contained in:
Matthew Parkinson
2021-04-04 20:42:05 +01:00
committed by Matthew Parkinson
parent 7202f9e091
commit 9ca73b8153
4 changed files with 171 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/mem/entropy.h
View File

@@ -27,6 +27,8 @@ namespace snmalloc
uint64_t local_key;
uint64_t local_counter;
address_t constant_key;
uint64_t fresh_bits;
uint64_t count;
public:
template<typename PAL>
@@ -93,5 +95,44 @@ namespace snmalloc
{
bit_source = get_next();
}
/**
* Pseudo random bit source.
*
* Does not cycle as frequently as `next_bit`.
*/
uint16_t next_fresh_bits(size_t n)
{
if (count <= n)
{
fresh_bits = get_next();
count = 64;
}
uint16_t result =
static_cast<uint16_t>(fresh_bits & (bits::one_at_bit(n) - 1));
fresh_bits >>= n;
count -= n;
return result;
}
/***
* Approximation of a uniform distribution
*
* Biases high numbers. A proper uniform distribution
* was too expensive. This maps a uniform distribution
* over the next power of two (2^m), and for numbers that
* are drawn larger then n-1, they are mapped onto uniform
* top range of n.
*/
uint16_t sample(uint16_t n)
{
size_t i = bits::next_pow2_bits(n);
uint16_t bits = next_fresh_bits(i);
uint16_t result = bits;
// Put over flowing bits at the top.
if (bits >= n)
result = n - (1 + bits - n);
return result;
}
};
} // namespace snmalloc

138
src/mem/freelist.h
View File

@@ -251,23 +251,28 @@ namespace snmalloc
* The fields are paired up to give better codegen as then they are offset
* by a power of 2, and the bit extract from the interleaving seed can
* be shifted to calculate the relevant offset to index the fields.
*
* If RANDOM is set to false, then the code does not perform any
* randomisation.
*/
template<typename S = uint32_t>
template<bool RANDOM = true, typename S = uint32_t>
class FreeListBuilder
{
static constexpr size_t LENGTH = RANDOM ? 2 : 1;
// Pointer to the first element.
EncodeFreeObjectReference head[2];
EncodeFreeObjectReference head[LENGTH];
// Pointer to the reference to the last element.
// In the empty case end[i] == &head[i]
// This enables branch free enqueuing.
EncodeFreeObjectReference* end[2];
EncodeFreeObjectReference* end[LENGTH];
#ifdef CHECK_CLIENT
// The bottom 16 bits of the previous pointer
uint16_t prev[2];
uint16_t prev[LENGTH];
// The bottom 16 bits of the current pointer
// This needs to be stored for the empty case
// where it is `initial_key()` for the slab.
uint16_t curr[2];
uint16_t curr[LENGTH];
#endif
public:
S s;
@@ -307,16 +312,16 @@ namespace snmalloc
void open(void* p)
{
SNMALLOC_ASSERT(empty());
for (size_t i = 0; i < LENGTH; i++)
{
#ifdef CHECK_CLIENT
prev[0] = HEAD_KEY;
curr[0] = initial_key(p) & 0xffff;
prev[1] = HEAD_KEY;
curr[1] = initial_key(p) & 0xffff;
prev[i] = HEAD_KEY;
curr[i] = initial_key(p) & 0xffff;
#else
UNUSED(p);
UNUSED(p);
#endif
end[0] = &head[0];
end[1] = &head[1];
end[i] = &head[i];
}
}
/**
@@ -324,7 +329,22 @@ namespace snmalloc
*/
bool empty()
{
return end[0] == &head[0] && end[1] == &head[1];
for (size_t i = 0; i < LENGTH; i++)
{
if (end[i] != &head[i])
return false;
}
return true;
}
bool debug_different_slab(void* n)
{
for (size_t i = 0; i < LENGTH; i++)
{
if (!different_slab(end[i], n))
return false;
}
return true;
}
/**
@@ -332,11 +352,10 @@ namespace snmalloc
*/
void add(void* n, LocalEntropy& entropy)
{
SNMALLOC_ASSERT(
!different_slab(end[0], n) || !different_slab(end[1], n) || empty());
SNMALLOC_ASSERT(!debug_different_slab(n) || empty());
FreeObject* next = FreeObject::make(n);
auto index = entropy.next_bit();
auto index = RANDOM ? entropy.next_bit() : 0;
end[index]->store(next, get_prev(index), entropy);
end[index] = &(next->next_object);
@@ -355,7 +374,7 @@ namespace snmalloc
size_t debug_length(LocalEntropy& entropy)
{
size_t count = 0;
for (size_t i = 0; i < 2; i++)
for (size_t i = 0; i < LENGTH; i++)
{
uint16_t local_prev = HEAD_KEY;
EncodeFreeObjectReference* iter = &head[i];
@@ -392,53 +411,60 @@ namespace snmalloc
*/
FreeListIter terminate(LocalEntropy& entropy, bool preserve_queue = true)
{
SNMALLOC_ASSERT(end[1] != &head[0]);
SNMALLOC_ASSERT(end[0] != &head[1]);
// If second list is empty, then append is trivial.
if (end[1] == &head[1])
if constexpr (RANDOM)
{
end[0]->store(nullptr, get_prev(0), entropy);
return {head[0].read(HEAD_KEY, entropy)};
}
SNMALLOC_ASSERT(end[1] != &head[0]);
SNMALLOC_ASSERT(end[0] != &head[1]);
end[1]->store(nullptr, get_prev(1), entropy);
// If second list is non-empty, perform append.
if (end[1] != &head[1])
{
end[1]->store(nullptr, get_prev(1), entropy);
// Append 1 to 0
auto mid = head[1].read(HEAD_KEY, entropy);
end[0]->store(mid, get_prev(0), entropy);
// Re-code first link in second list (if there is one).
// The first link in the second list will be encoded with initial_key,
// But that needs to be changed to the curr of the first list.
if (mid != nullptr)
{
auto mid_next = mid->read_next(initial_key(mid) & 0xffff, entropy);
mid->next_object.store(mid_next, get_curr(0), entropy);
}
// Append 1 to 0
auto mid = head[1].read(HEAD_KEY, entropy);
end[0]->store(mid, get_prev(0), entropy);
// Re-code first link in second list (if there is one).
// The first link in the second list will be encoded with initial_key,
// But that needs to be changed to the curr of the first list.
if (mid != nullptr)
{
auto mid_next = mid->read_next(initial_key(mid) & 0xffff, entropy);
mid->next_object.store(mid_next, get_curr(0), entropy);
}
auto h = head[0].read(HEAD_KEY, entropy);
auto h = head[0].read(HEAD_KEY, entropy);
// If we need to continue adding to the builder
// Set up the second list as empty,
// and extend the first list to cover all of the second.
if (preserve_queue && h != nullptr)
{
// If we need to continue adding to the builder
// Set up the second list as empty,
// and extend the first list to cover all of the second.
if (preserve_queue && h != nullptr)
{
#ifdef CHECK_CLIENT
prev[0] = prev[1];
curr[0] = curr[1];
prev[0] = prev[1];
curr[0] = curr[1];
#endif
end[0] = end[1];
end[0] = end[1];
#ifdef CHECK_CLIENT
prev[1] = HEAD_KEY;
curr[1] = initial_key(h) & 0xffff;
prev[1] = HEAD_KEY;
curr[1] = initial_key(h) & 0xffff;
#endif
end[1] = &(head[1]);
end[1] = &(head[1]);
}
SNMALLOC_ASSERT(end[1] != &head[0]);
SNMALLOC_ASSERT(end[0] != &head[1]);
return {h};
}
}
else
{
UNUSED(preserve_queue);
}
SNMALLOC_ASSERT(end[1] != &head[0]);
SNMALLOC_ASSERT(end[0] != &head[1]);
return {h};
end[0]->store(nullptr, get_prev(0), entropy);
return {head[0].read(HEAD_KEY, entropy)};
}
/**
@@ -456,8 +482,10 @@ namespace snmalloc
*/
void init()
{
end[0] = &head[0];
end[1] = &head[1];
for (size_t i = 0; i < LENGTH; i++)
{
end[i] = &head[i];
}
}
};
} // namespace snmalloc

6
src/mem/metaslab.h
View File

@@ -50,7 +50,11 @@ namespace snmalloc
*
* Spare 32bits are used for the fields in MetaslabEnd.
*/
FreeListBuilder<MetaslabEnd> free_queue;
#ifdef CHECK_CLIENT
FreeListBuilder<true, MetaslabEnd> free_queue;
#else
FreeListBuilder<false, MetaslabEnd> free_queue;
#endif
uint16_t& needed()
{

54
src/mem/slab.h
View File

@@ -37,24 +37,54 @@ namespace snmalloc
{
void* slab_end = pointer_align_up<SLAB_SIZE>(pointer_offset(bumpptr, 1));
FreeListBuilder b;
FreeListBuilder<false> b;
SNMALLOC_ASSERT(b.empty());
b.open(bumpptr);
// This code needs generalising, but currently applies
// various offsets with a stride of seven to increase chance of catching
// accidental OOB write.
std::array<size_t, 7> start_index = {3, 5, 0, 2, 4, 1, 6};
for (size_t offset : start_index)
#ifdef CHECK_CLIENT
// Structure to represent the temporary list elements
struct PreAllocObject
{
void* newbumpptr = pointer_offset(bumpptr, rsize * offset);
while (newbumpptr < slab_end)
{
b.add(newbumpptr, entropy);
newbumpptr = pointer_offset(newbumpptr, rsize * start_index.size());
}
PreAllocObject* next;
};
// The following code implements Sattolo's algorithm for generating
// random cyclic permutations. This implementation is in the opposite
// direction, so that the original space does not need initialising. This
// is described as outside-in without citation on Wikipedia, appears to be
// Folklore algorithm.
PreAllocObject* curr = pointer_offset<PreAllocObject>(bumpptr, 0);
curr->next = curr;
uint16_t count = 1;
for (PreAllocObject* p = pointer_offset<PreAllocObject>(bumpptr, rsize);
p < slab_end;
p = pointer_offset<PreAllocObject>(p, rsize))
{
size_t insert_index = entropy.sample(count);
p->next = std::exchange(
pointer_offset<PreAllocObject>(bumpptr, insert_index * rsize)->next,
p);
count++;
}
// Pick entry into space, and then build linked list by traversing cycle
// to the start.
auto start_index = entropy.sample(count);
auto start_ptr =
pointer_offset<PreAllocObject>(bumpptr, start_index * rsize);
auto curr_ptr = start_ptr;
do
{
b.add(curr_ptr, entropy);
curr_ptr = curr_ptr->next;
} while (curr_ptr != start_ptr);
#else
for (void* p = bumpptr; p < slab_end; p = pointer_offset<void>(p, rsize))
{
b.add(p, entropy);
}
#endif
// This code consumes everything up to slab_end.
bumpptr = slab_end;
SNMALLOC_ASSERT(!b.empty());