diff --git a/src/mem/alloc.h b/src/mem/alloc.h index e4e47d8..5b51954 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -763,16 +763,19 @@ namespace snmalloc for (size_t i = 0; i < NUM_SMALL_CLASSES; i++) { - while (!small_fast_free_lists[i].empty()) + if (!small_fast_free_lists[i].empty()) { - auto curr = small_fast_free_lists[i].take(); + auto head = small_fast_free_lists[i].peek(); + auto super = Superslab::get(head); + auto slab = Metaslab::get_slab(head); + do + { + auto curr = small_fast_free_lists[i].take(); + small_dealloc_offseted_inner(super, slab, curr, i); + } while (!small_fast_free_lists[i].empty()); - Superslab* super = Superslab::get(curr); - Slab* slab = Metaslab::get_slab(curr); - small_dealloc_offseted_inner(super, slab, curr, i); + test(small_classes[i]); } - - test(small_classes[i]); } for (auto& medium_class : medium_classes) @@ -828,10 +831,9 @@ namespace snmalloc // Destined for my slabs Superslab* super = Superslab::get(p); -#ifdef CHECK_CLIENT - if (p->trunc_target_id() != (super->get_allocator()->trunc_id())) - error("Detected memory corruption. Potential use-after-free"); -#endif + check_client( + p->trunc_target_id() == super->get_allocator()->trunc_id(), + "Detected memory corruption. Potential use-after-free"); void* start = remove_cache_friendly_offset(p, p->sizeclass()); dealloc_not_large_local(super, start, p, p->sizeclass()); @@ -1178,13 +1180,9 @@ namespace snmalloc SNMALLOC_FAST_PATH void small_dealloc_unchecked(Superslab* super, void* p, sizeclass_t sizeclass) { -#ifdef CHECK_CLIENT - uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p)); - if (chunkmap_slab_kind != CMSuperslab) - { - error("Claimed small deallocation is not in a Superslab"); - } -#endif + check_client( + chunkmap().get(address_cast(p)) == CMSuperslab, + "Claimed small deallocation is not in a Superslab"); small_dealloc_checked_chunkmap(super, p, sizeclass); } @@ -1193,13 +1191,9 @@ namespace snmalloc Superslab* super, void* p, sizeclass_t sizeclass) { Slab* slab = Metaslab::get_slab(p); -#ifdef CHECK_CLIENT - Metaslab& meta = super->get_meta(slab); - if (sizeclass != meta.sizeclass) - { - error("Claimed small deallocation with mismatching size class"); - } -#endif + check_client( + sizeclass == super->get_meta(slab).sizeclass, + "Claimed small deallocation with mismatching size class"); small_dealloc_checked_sizeclass(super, slab, p, sizeclass); } @@ -1207,12 +1201,9 @@ namespace snmalloc SNMALLOC_FAST_PATH void small_dealloc_checked_sizeclass( Superslab* super, Slab* slab, void* p, sizeclass_t sizeclass) { -#ifdef CHECK_CLIENT - if (!Metaslab::is_start_of_object(&Slab::get_meta(slab), p)) - { - error("Not deallocating start of an object"); - } -#endif + check_client( + Metaslab::is_start_of_object(&Slab::get_meta(slab), p), + "Not deallocating start of an object"); small_dealloc_start(super, slab, p, sizeclass); } @@ -1363,13 +1354,9 @@ namespace snmalloc void medium_dealloc_unchecked(Mediumslab* slab, void* p, sizeclass_t sizeclass) { -#ifdef CHECK_CLIENT - uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p)); - if (chunkmap_slab_kind != CMMediumslab) - { - error("Claimed medium deallocation is not in a Mediumslab"); - } -#endif + check_client( + chunkmap().get(address_cast(p)) == CMMediumslab, + "Claimed medium deallocation is not in a Mediumslab"); medium_dealloc_checked_chunkmap(slab, p, sizeclass); } @@ -1378,12 +1365,9 @@ namespace snmalloc void medium_dealloc_checked_chunkmap( Mediumslab* slab, void* p, sizeclass_t sizeclass) { -#ifdef CHECK_CLIENT - if (slab->get_sizeclass() != sizeclass) - { - error("Claimed medium deallocation of the wrong sizeclass"); - } -#endif + check_client( + slab->get_sizeclass() == sizeclass, + "Claimed medium deallocation of the wrong sizeclass"); medium_dealloc_checked_sizeclass(slab, p, sizeclass); } @@ -1392,14 +1376,11 @@ namespace snmalloc void medium_dealloc_checked_sizeclass( Mediumslab* slab, void* p, sizeclass_t sizeclass) { -#ifdef CHECK_CLIENT - if (!is_multiple_of_sizeclass( - sizeclass_to_size(sizeclass), - pointer_diff(p, pointer_offset(slab, SUPERSLAB_SIZE)))) - { - error("Not deallocating start of an object"); - } -#endif + check_client( + is_multiple_of_sizeclass( + sizeclass_to_size(sizeclass), + pointer_diff(p, pointer_offset(slab, SUPERSLAB_SIZE))), + "Not deallocating start of an object"); medium_dealloc_start(slab, p, sizeclass); } @@ -1482,37 +1463,22 @@ namespace snmalloc void large_dealloc_unchecked(void* p, size_t size) { - size_t claimed_chunkmap_slab_kind = bits::next_pow2_bits(size); - uint8_t chunkmap_slab_kind; + uint8_t claimed_chunkmap_slab_kind = + static_cast(bits::next_pow2_bits(size)); -#ifdef CHECK_CLIENT - chunkmap_slab_kind = chunkmap().get(address_cast(p)); - if (chunkmap_slab_kind < CMLargeMin) - { - error("Claimed large deallocation is not in a large slab"); - } - if (chunkmap_slab_kind != claimed_chunkmap_slab_kind) - { - error("Claimed large deallocation with wrong size class"); - } -#else - // Trusting sort, aren't we? - chunkmap_slab_kind = static_cast(claimed_chunkmap_slab_kind); -#endif + check_client( + chunkmap().get(address_cast(p)) == claimed_chunkmap_slab_kind, + "Claimed large deallocation with wrong size class"); - large_dealloc_checked_sizeclass(p, size, chunkmap_slab_kind); + large_dealloc_checked_sizeclass(p, size, claimed_chunkmap_slab_kind); } void large_dealloc_checked_sizeclass( void* p, size_t size, uint8_t chunkmap_slab_kind) { -#ifdef CHECK_CLIENT - Superslab* super = Superslab::get(p); - if (address_cast(super) != address_cast(p)) - { - error("Not deallocating start of an object"); - } -#endif + check_client( + address_cast(Superslab::get(p)) == address_cast(p), + "Not deallocating start of an object"); SNMALLOC_ASSERT(bits::one_at_bit(chunkmap_slab_kind) >= SUPERSLAB_SIZE); large_dealloc_start(p, size, chunkmap_slab_kind); diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 5bb03fb..2b25512 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -8,6 +8,22 @@ namespace snmalloc // calling the API correctly. #if !defined(NDEBUG) && !defined(CHECK_CLIENT) # define CHECK_CLIENT +#endif + + SNMALLOC_FAST_PATH void check_client_impl(bool test, const char* const str) + { +#ifdef CHECK_CLIENT + if (unlikely(!test)) + error(str); +#else + UNUSED(test); + UNUSED(str); +#endif + } +#ifdef CHECK_CLIENT +# define check_client(test, str) check_client_impl(test, str) +#else +# define check_client(test, str) #endif // 0 intermediate bits results in power of 2 small allocs. 1 intermediate diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 773759b..5b65afc 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -139,10 +139,9 @@ namespace snmalloc # ifndef NDEBUG if (next != nullptr) { - if (unlikely(different_slab(prev, next))) - { - error("Heap corruption - free list corrupted!"); - } + check_client( + !different_slab(prev, next), + "Heap corruption - free list corrupted!"); } # endif prev = address_cast(curr); @@ -162,10 +161,8 @@ namespace snmalloc void move_next() { #ifdef CHECK_CLIENT - if (unlikely(different_slab(prev, curr))) - { - error("Heap corruption - free list corrupted!"); - } + check_client( + !different_slab(prev, curr), "Heap corruption - free list corrupted!"); #endif update_cursor(curr->read_next(get_prev())); } @@ -221,6 +218,14 @@ namespace snmalloc return front.get_curr() == nullptr; } + /** + * Returns current head without affecting the iterator. + */ + void* peek() + { + return front.get_curr(); + } + /** * Moves the iterator on, and returns the current value. */ @@ -258,7 +263,7 @@ namespace snmalloc */ void* peek_head() { - return front.get_curr(); + return peek(); } /** diff --git a/src/mem/slab.h b/src/mem/slab.h index f8d4f62..295bba8 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -81,10 +81,7 @@ namespace snmalloc dealloc_fast(Slab* self, Superslab* super, void* p) { Metaslab& meta = super->get_meta(self); -#ifdef CHECK_CLIENT - if (meta.is_unused()) - error("Detected potential double free."); -#endif + SNMALLOC_ASSERT(!meta.is_unused()); if (unlikely(meta.return_object())) return false;