Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move link out of object space.

Browse Source 
The link object was previously stored in a disused object.  This is
good for reducing meta-data, but if we want to reduce the meta-data
corruption potential, then this is not a good design choice.

This commit moves it into the Metaslab.
This commit is contained in:
Matthew Parkinson
2021-03-12 21:11:43 +00:00
committed by Matthew Parkinson
parent e7dce55f19
commit 59edf294d0
5 changed files with 75 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/ds/cdllist.h
View File

@@ -110,6 +110,17 @@ namespace snmalloc
#endif
}
/**
* Nulls the previous pointer
*
* The Meta-slab uses nullptr in prev to mean that it is not part of a
* size class list.
**/
void null_prev()
{
prev = nullptr;
}
SNMALLOC_FAST_PATH CDLLNode* get_prev()
{
return prev;

8
src/mem/alloc.h
View File

@@ -1074,18 +1074,14 @@ namespace snmalloc
size_t rsize = sizeclass_to_size(sizeclass);
auto& sl = small_classes[sizeclass];
Slab* slab;
if (likely(!sl.is_empty()))
{
stats().alloc_request(size);
stats().sizeclass_alloc(sizeclass);
SlabLink* link = sl.get_next();
slab = get_slab(link);
auto meta = reinterpret_cast<Metaslab*>(sl.get_next());
auto& ffl = small_fast_free_lists[sizeclass];
return slab->alloc<zero_mem, typename MemoryProvider::Pal>(
sl, ffl, rsize);
return meta->alloc<zero_mem, typename MemoryProvider::Pal>(ffl, rsize);
}
return small_alloc_rare<zero_mem, allow_reserve>(sizeclass, size);
}

85
src/mem/metaslab.h
View File

@@ -9,6 +9,12 @@ namespace snmalloc
{
class Slab;
struct FreeListHead
{
// Use a value with bottom bit set for empty list.
void* value = nullptr;
};
using SlabList = CDLLNode;
using SlabLink = CDLLNode;
@@ -23,14 +29,13 @@ namespace snmalloc
// The Metaslab represent the status of a single slab.
// This can be either a short or a standard slab.
class Metaslab
class Metaslab : public SlabLink
{
public:
/**
* Pointer to first free entry in this slab
*
* The list will be (allocated - needed - 1) long. The -1 is
* for the `link` element which is not in the free list.
* The list will be (allocated - needed) long.
*/
void* head = nullptr;
@@ -38,7 +43,7 @@ namespace snmalloc
* How many entries are not in the free list of slab, i.e.
* how many entries are needed to fully free this slab.
*
* In the case of a fully allocated slab, where link==1 needed
* In the case of a fully allocated slab, where prev==0 needed
* will be 1. This enables 'return_object' to detect the slow path
* case with a single operation subtract and test.
*/
@@ -49,11 +54,6 @@ namespace snmalloc
*/
uint16_t allocated = 0;
// When a slab has free space it will be on the has space list for
// that size class. We use an empty block in this slab to be the
// doubly linked node into that size class's free list.
Mod<SLAB_SIZE, uint16_t> link;
uint8_t sizeclass;
// Initially zero to encode the superslabs relative list of slabs.
uint8_t next = 0;
@@ -77,7 +77,7 @@ namespace snmalloc
bool is_full()
{
auto result = link == 1;
auto result = get_prev() == nullptr;
SNMALLOC_ASSERT(!result || head == nullptr);
return result;
}
@@ -85,16 +85,10 @@ namespace snmalloc
SNMALLOC_FAST_PATH void set_full()
{
SNMALLOC_ASSERT(head == nullptr);
SNMALLOC_ASSERT(link != 1);
link = 1;
// Set needed to 1, so that "return_object" will return true after calling
// set_full
needed = 1;
}
SlabLink* get_link(Slab* slab)
{
return reinterpret_cast<SlabLink*>(pointer_offset(slab, link));
null_prev();
}
/// Value used to check for corruptions in a block
@@ -157,6 +151,51 @@ namespace snmalloc
pointer_diff(p, pointer_align_up<SLAB_SIZE>(pointer_offset(p, 1))));
}
/**
* Takes a free list out of a slabs meta data.
* Returns the link as the allocation, and places the free list into the
* `fast_free_list` for further allocations.
*/
template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
SNMALLOC_FAST_PATH void* alloc(FreeListHead& fast_free_list, size_t rsize)
{
SNMALLOC_ASSERT(rsize == sizeclass_to_size(sizeclass));
SNMALLOC_ASSERT(!is_full());
auto slab = get_slab(head);
debug_slab_invariant(slab);
// Use first element as the allocation
void* p = head;
// Put the rest in allocators small_class fast free list.
fast_free_list.value = Metaslab::follow_next(p);
head = nullptr;
// Treat stealing the free list as allocating it all.
needed = allocated;
remove();
set_full();
p = remove_cache_friendly_offset(p, sizeclass);
SNMALLOC_ASSERT(is_start_of_object(p));
debug_slab_invariant(slab);
if constexpr (zero_mem == YesZero)
{
if (rsize < PAGE_ALIGNED_SIZE)
PAL::zero(p, rsize);
else
PAL::template zero<true>(p, rsize);
}
else
{
UNUSED(rsize);
}
return p;
}
/**
* Check bump-free-list-segment for cycles
*
@@ -207,7 +246,6 @@ namespace snmalloc
if (is_full())
{
// There is no free list to validate
// 'link' value is not important if full.
return;
}
@@ -236,8 +274,6 @@ namespace snmalloc
// Account for free elements in free list
accounted_for += size;
SNMALLOC_ASSERT(SLAB_SIZE >= accounted_for);
// We should never reach the link node in the free list.
SNMALLOC_ASSERT(curr != pointer_offset(slab, link));
// Iterate bump/free list segment
curr = follow_next(curr);
@@ -250,16 +286,7 @@ namespace snmalloc
// Account for to be bump allocated space
accounted_for += SLAB_SIZE - bumpptr;
if (bumpptr != SLAB_SIZE)
{
// The link should be the first allocation as we
// haven't completely filled this block at any point.
SNMALLOC_ASSERT(link == get_initial_offset(sizeclass, is_short));
}
SNMALLOC_ASSERT(!is_full());
// Add the link node.
accounted_for += size;
// All space accounted for
SNMALLOC_ASSERT(SLAB_SIZE == accounted_for);

72
src/mem/slab.h
View File

@@ -4,12 +4,6 @@
namespace snmalloc
{
struct FreeListHead
{
// Use a value with bottom bit set for empty list.
void* value = nullptr;
};
class Slab
{
private:
@@ -26,62 +20,6 @@ namespace snmalloc
return super->get_meta(this);
}
SlabLink* get_link()
{
return get_meta().get_link(this);
}
/**
* Takes a free list out of a slabs meta data.
* Returns the link as the allocation, and places the free list into the
* `fast_free_list` for further allocations.
*/
template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
SNMALLOC_FAST_PATH void*
alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize)
{
// Read the head from the metadata stored in the superslab.
Metaslab& meta = get_meta();
SNMALLOC_ASSERT(meta.link != 1);
SNMALLOC_ASSERT(rsize == sizeclass_to_size(meta.sizeclass));
SNMALLOC_ASSERT(
sl.get_next() == (SlabLink*)pointer_offset(this, meta.link));
SNMALLOC_ASSERT(!meta.is_full());
meta.debug_slab_invariant(this);
// Put everything in allocators small_class free list.
fast_free_list.value = meta.head;
meta.head = nullptr;
// Return the link as the node for this allocation.
void* link = pointer_offset(this, meta.link);
void* p = remove_cache_friendly_offset(link, meta.sizeclass);
// Treat stealing the free list as allocating it all.
meta.needed = meta.allocated;
meta.set_full();
sl.get_next()->remove();
SNMALLOC_ASSERT(meta.is_start_of_object(p));
meta.debug_slab_invariant(this);
if constexpr (zero_mem == YesZero)
{
if (rsize < PAGE_ALIGNED_SIZE)
PAL::zero(p, rsize);
else
PAL::template zero<true>(p, rsize);
}
else
{
UNUSED(rsize);
}
return p;
}
/**
* Given a bumpptr and a fast_free_list head reference, builds a new free
* list, and stores it in the fast_free_list. It will only create a page
@@ -158,21 +96,19 @@ namespace snmalloc
return super->dealloc_slab(this);
}
// Update the head and the sizeclass link.
uint16_t index = pointer_to_index(p);
SNMALLOC_ASSERT(meta.head == nullptr);
// SNMALLOC_ASSERT(meta.fully_allocated(is_short()));
meta.link = index;
meta.head = p;
Metaslab::store_next(p, nullptr);
meta.needed = meta.allocated - 1;
// Push on the list of slabs for this sizeclass.
sl->insert_prev(meta.get_link(this));
sl->insert_prev(&meta);
meta.debug_slab_invariant(this);
return Superslab::NoSlabReturn;
}
// Remove from the sizeclass list and dealloc on the superslab.
meta.get_link(this)->remove();
meta.remove();
if (is_short())
return super->dealloc_short_slab();

6
src/mem/superslab.h
View File

@@ -173,8 +173,7 @@ namespace snmalloc
meta[0].allocated = static_cast<uint16_t>(
(SLAB_SIZE - get_initial_offset(sizeclass, true)) /
sizeclass_to_size(sizeclass));
meta[0].link = 1;
meta[0].needed = 1;
meta[0].set_full();
meta[0].sizeclass = static_cast<uint8_t>(sizeclass);
used++;
@@ -198,8 +197,7 @@ namespace snmalloc
meta[h].allocated = static_cast<uint16_t>(
(SLAB_SIZE - get_initial_offset(sizeclass, false)) /
sizeclass_to_size(sizeclass));
meta[h].needed = 1;
meta[h].link = 1;
meta[h].set_full();
meta[h].sizeclass = static_cast<uint8_t>(sizeclass);
head = h + n + 1;