SP: mediumslab: reduce capptr_export calls

Mediumslabs are strung on a dllist and used to feed the allocator there.  If we
ensure that these (and the root pointer to the list itself) are already
exported, then our alloc paths can bound these to arrive at exposible pointers.

The dealloc paths, where we might want a non-exported pointer, already have one,
as they have gone through amplification to get an arena-bounded pointer.

The sole wrinkle in this plan is that we might need a pointer without platform
constraints to manipulate the memory map for page-based zeroing.  Since we have
ample room in the Mediumslab header (a few kilobytes end up being used for
padding; the curious should see b8b5f305 and 3d3b0487), just cache therein a
copy of the CBChunk-bound pointer used in Mediumslab::init() for ::alloc().
This commit is contained in:
Nathaniel Filardo
2021-03-13 15:20:12 +00:00
committed by Nathaniel Wesley Filardo
parent eff76309f5
commit cf50fc5e55
2 changed files with 26 additions and 12 deletions

View File

@@ -638,7 +638,7 @@ namespace snmalloc
};
SlabList small_classes[NUM_SMALL_CLASSES];
DLList<Mediumslab, CapPtrCBChunk> medium_classes[NUM_MEDIUM_CLASSES];
DLList<Mediumslab, CapPtrCBChunkE> medium_classes[NUM_MEDIUM_CLASSES];
DLList<Superslab, CapPtrCBChunk> super_available;
DLList<Superslab, CapPtrCBChunk> super_only_short_available;
@@ -1383,7 +1383,7 @@ namespace snmalloc
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
auto sc = &medium_classes[medium_class];
CapPtr<Mediumslab, CBChunk> slab = sc->get_head();
CapPtr<Mediumslab, CBChunkE> slab = sc->get_head();
CapPtr<void, CBAllocE> p;
if (slab != nullptr)
@@ -1424,11 +1424,14 @@ namespace snmalloc
Mediumslab::init(newslab, public_state(), sizeclass, rsize);
chunkmap().set_slab(newslab);
auto newslab_export = capptr_export(newslab);
p = Mediumslab::alloc<zero_mem, typename MemoryProvider::Pal>(
newslab, rsize);
newslab_export, rsize);
if (!Mediumslab::full(newslab))
sc->insert(newslab);
sc->insert(newslab_export);
}
stats().alloc_request(size);
@@ -1515,7 +1518,12 @@ namespace snmalloc
{
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
auto sc = &medium_classes[medium_class];
sc->remove(slab_bounded);
/*
* This unsafety lets us avoid applying platform constraints to a
* pointer we are just about to drop on the floor; remove() uses its
* argument but does not persist it.
*/
sc->remove(CapPtr<Mediumslab, CBChunkE>(slab_bounded.unsafe_capptr));
}
chunkmap().clear_slab(slab_bounded);
@@ -1527,7 +1535,7 @@ namespace snmalloc
{
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
auto sc = &medium_classes[medium_class];
sc->insert(slab_bounded);
sc->insert(capptr_export(slab_bounded));
}
}

View File

@@ -12,12 +12,16 @@ namespace snmalloc
// This is the view of a 16 mb area when it is being used to allocate
// medium sized classes: 64 kb to 16 mb, non-inclusive.
private:
friend DLList<Mediumslab, CapPtrCBChunk>;
friend DLList<Mediumslab, CapPtrCBChunkE>;
// Keep the allocator pointer on a separate cache line. It is read by
// other threads, and does not change, so we avoid false sharing.
alignas(CACHELINE_SIZE) CapPtr<Mediumslab, CBChunk> next;
CapPtr<Mediumslab, CBChunk> prev;
alignas(CACHELINE_SIZE) CapPtr<Mediumslab, CBChunkE> next;
CapPtr<Mediumslab, CBChunkE> prev;
// Store a pointer to ourselves without platform constraints applied,
// as we need this to be able to zero memory by manipulating the VM map
CapPtr<void, CBChunk> self_chunk;
uint16_t free;
uint8_t head;
@@ -78,6 +82,7 @@ namespace snmalloc
// initialise the allocation stack.
if ((self->kind != Medium) || (self->sizeclass != sc))
{
self->self_chunk = self.as_void();
self->sizeclass = static_cast<uint8_t>(sc);
uint16_t ssize = static_cast<uint16_t>(rsize >> 8);
self->kind = Medium;
@@ -89,6 +94,7 @@ namespace snmalloc
else
{
SNMALLOC_ASSERT(self->free == medium_slab_free(sc));
SNMALLOC_ASSERT(self->self_chunk == self.as_void());
}
}
@@ -99,7 +105,7 @@ namespace snmalloc
template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
static CapPtr<void, CBAllocE>
alloc(CapPtr<Mediumslab, CBChunk> self, size_t size)
alloc(CapPtr<Mediumslab, CBChunkE> self, size_t size)
{
SNMALLOC_ASSERT(!full(self));
@@ -108,11 +114,11 @@ namespace snmalloc
self->free--;
if constexpr (zero_mem == YesZero)
pal_zero<PAL>(p, size);
pal_zero<PAL>(Aal::capptr_rebound(self->self_chunk, p), size);
else
UNUSED(size);
return capptr_export(Aal::capptr_bound<void, CBAlloc>(p, size));
return Aal::capptr_bound<void, CBAllocE>(p, size);
}
static bool