Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Randomise slab filling (#397)

Browse Source 
# Free List builder track length

This commit makes the free list builder track the length of the lists in
the Random case.

# Refactor free list creation.

Minor refactoring to share code between the new free list and existing
path.

# Randomise slab filling

Knowing when a slab is going to become full makes it easier to by pass
the free list entries as protection for OOB writes.  This commit
randomises when a slab will become full.

This commit changes two things

* the free list builder can return some fraction of the deallocations
  on a slab.
* when there is a single free slab, we can with some probability
  allocate an additional slab.

These two combine to make it difficult to predict when a slab will be
free.

# Apply suggestions from code review

Co-authored-by: Nathaniel Wesley Filardo <nfilardo@microsoft.com>
This commit is contained in:
Matthew Parkinson
2021-10-07 15:51:18 +01:00
committed by GitHub
parent 8ac2adc4e5
commit bba66e4f7e
3 changed files with 119 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
src/mem/corealloc.h
View File

@@ -188,7 +188,7 @@ namespace snmalloc
static SNMALLOC_FAST_PATH void alloc_new_list(
CapPtr<void, CBChunk>& bumpptr,
FreeListIter& fast_free_list,
Metaslab* meta,
size_t rsize,
size_t slab_size,
LocalEntropy& entropy)
@@ -197,8 +197,7 @@ namespace snmalloc
auto& key = entropy.get_free_list_key();
FreeListBuilder<false> b;
SNMALLOC_ASSERT(b.empty());
auto& b = meta->free_queue;
#ifdef SNMALLOC_CHECK_CLIENT
// Structure to represent the temporary list elements
@@ -243,7 +242,7 @@ namespace snmalloc
auto curr_ptr = start_ptr;
do
{
b.add(FreeObject::make(curr_ptr.as_void()), key);
b.add(FreeObject::make(curr_ptr.as_void()), key, entropy);
curr_ptr = curr_ptr->next;
} while (curr_ptr != start_ptr);
#else
@@ -256,16 +255,14 @@ namespace snmalloc
#endif
// This code consumes everything up to slab_end.
bumpptr = slab_end;
SNMALLOC_ASSERT(!b.empty());
b.close(fast_free_list, key);
}
ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass)
{
auto& key = entropy.get_free_list_key();
FreeListIter fl;
meta->free_queue.close(fl, key);
auto more = meta->free_queue.close(fl, key);
UNUSED(more);
void* p = finish_alloc_no_zero(fl.take(key), sizeclass);
#ifdef SNMALLOC_CHECK_CLIENT
@@ -278,6 +275,21 @@ namespace snmalloc
count++;
}
// Check the list contains all the elements
SNMALLOC_ASSERT(
(count + more) == snmalloc::sizeclass_to_slab_object_count(sizeclass));
if (more > 0)
{
auto no_more = meta->free_queue.close(fl, key);
SNMALLOC_ASSERT(no_more == 0);
UNUSED(no_more);
while (!fl.empty())
{
fl.take(key);
count++;
}
}
SNMALLOC_ASSERT(
count == snmalloc::sizeclass_to_slab_object_count(sizeclass));
#endif
@@ -582,15 +594,32 @@ namespace snmalloc
// Look to see if we can grab a free list.
auto& sl = alloc_classes[sizeclass].queue;
if (likely(!(sl.is_empty())))
if (likely(alloc_classes[sizeclass].length > 0))
{
#ifdef SNMALLOC_CHECK_CLIENT
// Occassionally don't use the last list.
if (
unlikely(alloc_classes[sizeclass].length == 1) &&
(entropy.next_bit() == 0))
{
return small_alloc_slow<zero_mem>(sizeclass, fast_free_list, rsize);
}
#endif
auto meta = sl.pop();
// Drop length of sl, and empty count if it was empty.
alloc_classes[sizeclass].length--;
if (meta->needed() == 0)
alloc_classes[sizeclass].unused--;
auto p = Metaslab::alloc(meta, fast_free_list, entropy, sizeclass);
auto [p, still_active] =
Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass);
if (still_active)
{
alloc_classes[sizeclass].length++;
sl.insert(meta);
}
return finish_alloc<zero_mem, SharedStateHandle>(p, sizeclass);
}
@@ -641,16 +670,20 @@ namespace snmalloc
return nullptr;
}
// Build a free list for the slab
alloc_new_list(slab, fast_free_list, rsize, slab_size, entropy);
// Set meta slab to empty.
meta->initialise(sizeclass);
auto& key = entropy.get_free_list_key();
// Build a free list for the slab
alloc_new_list(slab, meta, rsize, slab_size, entropy);
// take an allocation from the free list
auto p = fast_free_list.take(key);
auto [p, still_active] =
Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass);
if (still_active)
{
alloc_classes[sizeclass].length++;
alloc_classes[sizeclass].queue.insert(meta);
}
return finish_alloc<zero_mem, SharedStateHandle>(p, sizeclass);
}

57
src/mem/freelist.h
View File

@@ -301,6 +301,8 @@ namespace snmalloc
// This enables branch free enqueuing.
std::array<CapPtr<FreeObject, CBAlloc>*, LENGTH> end{nullptr};
std::array<uint16_t, RANDOM ? 2 : 0> length{};
public:
constexpr FreeListBuilder()
{
@@ -336,6 +338,10 @@ namespace snmalloc
index = 0;
end[index] = FreeObject::store_next(end[index], n, key);
if constexpr (RANDOM)
{
length[index]++;
}
}
/**
@@ -388,41 +394,42 @@ namespace snmalloc
/**
* Close a free list, and set the iterator parameter
* to iterate it.
*
* In the RANDOM case, it may return only part of the freelist.
*
* The return value is how many entries are still contained in the builder.
*/
SNMALLOC_FAST_PATH void close(FreeListIter& fl, const FreeListKey& key)
SNMALLOC_FAST_PATH uint16_t close(FreeListIter& fl, const FreeListKey& key)
{
uint32_t i;
if constexpr (RANDOM)
{
SNMALLOC_ASSERT(end[1] != &head[0]);
SNMALLOC_ASSERT(end[0] != &head[1]);
// If second list is non-empty, perform append.
if (end[1] != &head[1])
{
// The start token has been corrupted.
// TOCTTOU issue, but small window here.
read_head(1, key)->check_prev(get_fake_signed_prev(1, key));
terminate_list(1, key);
// Append 1 to 0
FreeObject::store_next(end[0], read_head(1, key), key);
SNMALLOC_ASSERT(end[1] != &head[0]);
SNMALLOC_ASSERT(end[0] != &head[1]);
}
else
{
terminate_list(0, key);
}
// Select longest list.
i = length[0] > length[1] ? 0 : 1;
}
else
{
terminate_list(0, key);
i = 0;
}
fl = {read_head(0, key), get_fake_signed_prev(0, key)};
init();
terminate_list(i, key);
fl = {read_head(i, key), get_fake_signed_prev(i, key)};
end[i] = &head[i];
if constexpr (RANDOM)
{
length[i] = 0;
return length[1 - i];
}
else
{
return 0;
}
}
/**
@@ -433,6 +440,10 @@ namespace snmalloc
for (size_t i = 0; i < LENGTH; i++)
{
end[i] = &head[i];
if (RANDOM)
{
length[i] = 0;
}
}
}

48
src/mem/metaslab.h
View File

@@ -71,7 +71,7 @@ namespace snmalloc
// returned all the elements, but this is a slab that is still being bump
// allocated from. Hence, the bump allocator slab will never be returned
// for use in another size class.
set_sleeping(sizeclass);
set_sleeping(sizeclass, 0);
}
/**
@@ -96,14 +96,28 @@ namespace snmalloc
return sleeping();
}
SNMALLOC_FAST_PATH void set_sleeping(sizeclass_t sizeclass)
/**
* Try to set this metaslab to sleep. If the remaining elements are fewer
* than the threshold, then it will actually be set to the sleeping state,
* and will return true, otherwise it will return false.
*/
SNMALLOC_FAST_PATH bool
set_sleeping(sizeclass_t sizeclass, uint16_t remaining)
{
SNMALLOC_ASSERT(free_queue.empty());
auto threshold = threshold_for_waking_slab(sizeclass);
if (remaining >= threshold)
{
// Set needed to at least one, possibly more so we only use
// a slab when it has a reasonable amount of free elements
auto allocated = sizeclass_to_slab_object_count(sizeclass);
needed() = allocated - remaining;
sleeping() = false;
return false;
}
// Set needed to at least one, possibly more so we only use
// a slab when it has a reasonable amount of free elements
needed() = threshold_for_waking_slab(sizeclass);
sleeping() = true;
needed() = threshold - remaining;
return true;
}
SNMALLOC_FAST_PATH void set_not_sleeping(sizeclass_t sizeclass)
@@ -129,9 +143,18 @@ namespace snmalloc
}
/**
* TODO
* Allocates a free list from the meta data.
*
* Returns a freshly allocated object of the correct size, and a bool that
* specifies if the metaslab should be placed in the queue for that
* sizeclass.
*
* If Randomisation is not used, it will always return false for the second
* component, but with randomisation, it may only return part of the
* available objects for this metaslab.
*/
static SNMALLOC_FAST_PATH CapPtr<FreeObject, CBAlloc> alloc(
static SNMALLOC_FAST_PATH std::pair<CapPtr<FreeObject, CBAlloc>, bool>
alloc_free_list(
Metaslab* meta,
FreeListIter& fast_free_list,
LocalEntropy& entropy,
@@ -140,7 +163,7 @@ namespace snmalloc
auto& key = entropy.get_free_list_key();
FreeListIter tmp_fl;
meta->free_queue.close(tmp_fl, key);
auto remaining = meta->free_queue.close(tmp_fl, key);
auto p = tmp_fl.take(key);
fast_free_list = tmp_fl;
@@ -150,12 +173,13 @@ namespace snmalloc
UNUSED(entropy);
#endif
// Treat stealing the free list as allocating it all.
// This marks the slab as sleeping, and sets a wakeup
// when sufficient deallocations have occurred to this slab.
meta->set_sleeping(sizeclass);
// Takes how many deallocations were not grabbed on this call
// This will be zero if there is no randomisation.
auto sleeping = meta->set_sleeping(sizeclass, remaining);
return p;
return {p, !sleeping};
}
};