SP: mediumslab: reduce capptr_export calls
Mediumslabs are strung on a dllist and used to feed the allocator there. If we ensure that these (and the root pointer to the list itself) are already exported, then our alloc paths can bound these to arrive at exposible pointers. The dealloc paths, where we might want a non-exported pointer, already have one, as they have gone through amplification to get an arena-bounded pointer. The sole wrinkle in this plan is that we might need a pointer without platform constraints to manipulate the memory map for page-based zeroing. Since we have ample room in the Mediumslab header (a few kilobytes end up being used for padding; the curious should seeb8b5f305and3d3b0487), just cache therein a copy of the CBChunk-bound pointer used in Mediumslab::init() for ::alloc().
This commit is contained in:
committed by
Nathaniel Wesley Filardo
parent
eff76309f5
commit
cf50fc5e55
@@ -638,7 +638,7 @@ namespace snmalloc
|
||||
};
|
||||
|
||||
SlabList small_classes[NUM_SMALL_CLASSES];
|
||||
DLList<Mediumslab, CapPtrCBChunk> medium_classes[NUM_MEDIUM_CLASSES];
|
||||
DLList<Mediumslab, CapPtrCBChunkE> medium_classes[NUM_MEDIUM_CLASSES];
|
||||
|
||||
DLList<Superslab, CapPtrCBChunk> super_available;
|
||||
DLList<Superslab, CapPtrCBChunk> super_only_short_available;
|
||||
@@ -1383,7 +1383,7 @@ namespace snmalloc
|
||||
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
|
||||
|
||||
auto sc = &medium_classes[medium_class];
|
||||
CapPtr<Mediumslab, CBChunk> slab = sc->get_head();
|
||||
CapPtr<Mediumslab, CBChunkE> slab = sc->get_head();
|
||||
CapPtr<void, CBAllocE> p;
|
||||
|
||||
if (slab != nullptr)
|
||||
@@ -1424,11 +1424,14 @@ namespace snmalloc
|
||||
|
||||
Mediumslab::init(newslab, public_state(), sizeclass, rsize);
|
||||
chunkmap().set_slab(newslab);
|
||||
|
||||
auto newslab_export = capptr_export(newslab);
|
||||
|
||||
p = Mediumslab::alloc<zero_mem, typename MemoryProvider::Pal>(
|
||||
newslab, rsize);
|
||||
newslab_export, rsize);
|
||||
|
||||
if (!Mediumslab::full(newslab))
|
||||
sc->insert(newslab);
|
||||
sc->insert(newslab_export);
|
||||
}
|
||||
|
||||
stats().alloc_request(size);
|
||||
@@ -1515,7 +1518,12 @@ namespace snmalloc
|
||||
{
|
||||
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
|
||||
auto sc = &medium_classes[medium_class];
|
||||
sc->remove(slab_bounded);
|
||||
/*
|
||||
* This unsafety lets us avoid applying platform constraints to a
|
||||
* pointer we are just about to drop on the floor; remove() uses its
|
||||
* argument but does not persist it.
|
||||
*/
|
||||
sc->remove(CapPtr<Mediumslab, CBChunkE>(slab_bounded.unsafe_capptr));
|
||||
}
|
||||
|
||||
chunkmap().clear_slab(slab_bounded);
|
||||
@@ -1527,7 +1535,7 @@ namespace snmalloc
|
||||
{
|
||||
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
|
||||
auto sc = &medium_classes[medium_class];
|
||||
sc->insert(slab_bounded);
|
||||
sc->insert(capptr_export(slab_bounded));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,12 +12,16 @@ namespace snmalloc
|
||||
// This is the view of a 16 mb area when it is being used to allocate
|
||||
// medium sized classes: 64 kb to 16 mb, non-inclusive.
|
||||
private:
|
||||
friend DLList<Mediumslab, CapPtrCBChunk>;
|
||||
friend DLList<Mediumslab, CapPtrCBChunkE>;
|
||||
|
||||
// Keep the allocator pointer on a separate cache line. It is read by
|
||||
// other threads, and does not change, so we avoid false sharing.
|
||||
alignas(CACHELINE_SIZE) CapPtr<Mediumslab, CBChunk> next;
|
||||
CapPtr<Mediumslab, CBChunk> prev;
|
||||
alignas(CACHELINE_SIZE) CapPtr<Mediumslab, CBChunkE> next;
|
||||
CapPtr<Mediumslab, CBChunkE> prev;
|
||||
|
||||
// Store a pointer to ourselves without platform constraints applied,
|
||||
// as we need this to be able to zero memory by manipulating the VM map
|
||||
CapPtr<void, CBChunk> self_chunk;
|
||||
|
||||
uint16_t free;
|
||||
uint8_t head;
|
||||
@@ -78,6 +82,7 @@ namespace snmalloc
|
||||
// initialise the allocation stack.
|
||||
if ((self->kind != Medium) || (self->sizeclass != sc))
|
||||
{
|
||||
self->self_chunk = self.as_void();
|
||||
self->sizeclass = static_cast<uint8_t>(sc);
|
||||
uint16_t ssize = static_cast<uint16_t>(rsize >> 8);
|
||||
self->kind = Medium;
|
||||
@@ -89,6 +94,7 @@ namespace snmalloc
|
||||
else
|
||||
{
|
||||
SNMALLOC_ASSERT(self->free == medium_slab_free(sc));
|
||||
SNMALLOC_ASSERT(self->self_chunk == self.as_void());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,7 +105,7 @@ namespace snmalloc
|
||||
|
||||
template<ZeroMem zero_mem, SNMALLOC_CONCEPT(ConceptPAL) PAL>
|
||||
static CapPtr<void, CBAllocE>
|
||||
alloc(CapPtr<Mediumslab, CBChunk> self, size_t size)
|
||||
alloc(CapPtr<Mediumslab, CBChunkE> self, size_t size)
|
||||
{
|
||||
SNMALLOC_ASSERT(!full(self));
|
||||
|
||||
@@ -108,11 +114,11 @@ namespace snmalloc
|
||||
self->free--;
|
||||
|
||||
if constexpr (zero_mem == YesZero)
|
||||
pal_zero<PAL>(p, size);
|
||||
pal_zero<PAL>(Aal::capptr_rebound(self->self_chunk, p), size);
|
||||
else
|
||||
UNUSED(size);
|
||||
|
||||
return capptr_export(Aal::capptr_bound<void, CBAlloc>(p, size));
|
||||
return Aal::capptr_bound<void, CBAllocE>(p, size);
|
||||
}
|
||||
|
||||
static bool
|
||||
|
||||
Reference in New Issue
Block a user