diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 4ed76a6..cbb1410 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -875,6 +875,60 @@ namespace snmalloc #endif } + /** + * If result parameter is non-null, then is_empty is passsed into the + * the location pointed to by result. + * + * Otherwise, asserts that it is empty. + **/ + void debug_is_empty(bool* result) + { + auto test = [&result](bool value) { + if (!value) + { + if (result != nullptr) + *result = false; + else + error("debug_is_empty: found non-empty allocator"); + } + }; + + // Destroy the message queue so that it has no stub message. + Remote* p = message_queue().destroy(); + + while (p != nullptr) + { + Remote* n = p->non_atomic_next; + handle_dealloc_remote(p); + p = n; + } + + for (size_t i = 0; i < NUM_SMALL_CLASSES; i++) + { + auto prev = small_fast_free_lists[i].value; + small_fast_free_lists[i].value = nullptr; + while (prev != nullptr) + { + auto n = Metaslab::follow_next(prev); + dealloc(prev); + prev = n; + } + + test(small_classes[i].is_empty()); + } + + for (size_t i = 0; i < NUM_MEDIUM_CLASSES; i++) + { + test(medium_classes[i].is_empty()); + } + + test(super_available.is_empty()); + test(super_only_short_available.is_empty()); + + // Place the static stub message on the queue. + init_message_queue(); + } + template static uintptr_t external_pointer(void* p, sizeclass_t sizeclass, size_t end_point) @@ -1116,7 +1170,7 @@ namespace snmalloc if (void* replacement = Replacement(this)) { return reinterpret_cast(replacement) - ->template small_alloc_slow(sizeclass); + ->template small_alloc_inner(sizeclass); } stats().sizeclass_alloc(sizeclass); diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index 9db7d35..8cb05c2 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -120,32 +120,18 @@ namespace snmalloc auto* alloc = Parent::iterate(); bool done = false; + bool okay = true; - size_t non_empty_count = 0; while (!done) { done = true; alloc = Parent::iterate(); - non_empty_count = 0; + okay = true; while (alloc != nullptr) { - // Destroy the message queue so that it has no stub message. - Remote* p = alloc->message_queue().destroy(); - - while (p != nullptr) - { - Remote* next = p->non_atomic_next; - alloc->handle_dealloc_remote(p); - p = next; - } - // Check that the allocator has freed all memory. - if (!alloc->stats().is_empty()) - non_empty_count++; - - // Place the static stub message on the queue. - alloc->init_message_queue(); + alloc->debug_is_empty(&okay); // Post all remotes, including forwarded ones. If any allocator posts, // repeat the loop. @@ -162,13 +148,18 @@ namespace snmalloc if (result != nullptr) { - *result = non_empty_count == 0; + *result = okay; return; } - if (non_empty_count != 0) + if (!okay) { - error("debug_check_empty: found non-empty allocators"); + alloc = Parent::iterate(); + while (alloc != nullptr) + { + alloc->debug_is_empty(nullptr); + alloc = Parent::iterate(alloc); + } } #else UNUSED(result);