diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 5eaaab7..fcdf501 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -638,7 +638,7 @@ namespace snmalloc }; SlabList small_classes[NUM_SMALL_CLASSES]; - DLList medium_classes[NUM_MEDIUM_CLASSES]; + DLList medium_classes[NUM_MEDIUM_CLASSES]; DLList super_available; DLList super_only_short_available; @@ -1383,7 +1383,7 @@ namespace snmalloc sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; auto sc = &medium_classes[medium_class]; - CapPtr slab = sc->get_head(); + CapPtr slab = sc->get_head(); CapPtr p; if (slab != nullptr) @@ -1424,11 +1424,14 @@ namespace snmalloc Mediumslab::init(newslab, public_state(), sizeclass, rsize); chunkmap().set_slab(newslab); + + auto newslab_export = capptr_export(newslab); + p = Mediumslab::alloc( - newslab, rsize); + newslab_export, rsize); if (!Mediumslab::full(newslab)) - sc->insert(newslab); + sc->insert(newslab_export); } stats().alloc_request(size); @@ -1515,7 +1518,12 @@ namespace snmalloc { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; auto sc = &medium_classes[medium_class]; - sc->remove(slab_bounded); + /* + * This unsafety lets us avoid applying platform constraints to a + * pointer we are just about to drop on the floor; remove() uses its + * argument but does not persist it. + */ + sc->remove(CapPtr(slab_bounded.unsafe_capptr)); } chunkmap().clear_slab(slab_bounded); @@ -1527,7 +1535,7 @@ namespace snmalloc { sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; auto sc = &medium_classes[medium_class]; - sc->insert(slab_bounded); + sc->insert(capptr_export(slab_bounded)); } } diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h index ec3e83c..1877961 100644 --- a/src/mem/mediumslab.h +++ b/src/mem/mediumslab.h @@ -12,12 +12,16 @@ namespace snmalloc // This is the view of a 16 mb area when it is being used to allocate // medium sized classes: 64 kb to 16 mb, non-inclusive. private: - friend DLList; + friend DLList; // Keep the allocator pointer on a separate cache line. It is read by // other threads, and does not change, so we avoid false sharing. - alignas(CACHELINE_SIZE) CapPtr next; - CapPtr prev; + alignas(CACHELINE_SIZE) CapPtr next; + CapPtr prev; + + // Store a pointer to ourselves without platform constraints applied, + // as we need this to be able to zero memory by manipulating the VM map + CapPtr self_chunk; uint16_t free; uint8_t head; @@ -78,6 +82,7 @@ namespace snmalloc // initialise the allocation stack. if ((self->kind != Medium) || (self->sizeclass != sc)) { + self->self_chunk = self.as_void(); self->sizeclass = static_cast(sc); uint16_t ssize = static_cast(rsize >> 8); self->kind = Medium; @@ -89,6 +94,7 @@ namespace snmalloc else { SNMALLOC_ASSERT(self->free == medium_slab_free(sc)); + SNMALLOC_ASSERT(self->self_chunk == self.as_void()); } } @@ -99,7 +105,7 @@ namespace snmalloc template static CapPtr - alloc(CapPtr self, size_t size) + alloc(CapPtr self, size_t size) { SNMALLOC_ASSERT(!full(self)); @@ -108,11 +114,11 @@ namespace snmalloc self->free--; if constexpr (zero_mem == YesZero) - pal_zero(p, size); + pal_zero(Aal::capptr_rebound(self->self_chunk, p), size); else UNUSED(size); - return capptr_export(Aal::capptr_bound(p, size)); + return Aal::capptr_bound(p, size); } static bool