diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index f8b7251..f348542 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -97,7 +97,7 @@ namespace snmalloc // // dealloc() can reject attempts to free such MetaEntry-s due to the // zero sizeclass. - MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr, 0); + MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr); Pagemap::set_metaentry(local_state, address_cast(base), 1, t); return; } diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 323c566..a83d2a8 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -68,9 +68,9 @@ namespace snmalloc #endif // Maximum size of an object that uses sizeclasses. - static constexpr size_t MAX_SIZECLASS_BITS = 16; - static constexpr size_t MAX_SIZECLASS_SIZE = - bits::one_at_bit(MAX_SIZECLASS_BITS); + static constexpr size_t MAX_SMALL_SIZECLASS_BITS = 16; + static constexpr size_t MAX_SMALL_SIZECLASS_SIZE = + bits::one_at_bit(MAX_SMALL_SIZECLASS_BITS); // Number of slots for remote deallocation. static constexpr size_t REMOTE_SLOT_BITS = 8; diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h index 937a3a9..18896c6 100644 --- a/src/mem/allocstats.h +++ b/src/mem/allocstats.h @@ -179,7 +179,7 @@ namespace snmalloc #endif } - void sizeclass_alloc(sizeclass_t sc) + void sizeclass_alloc(smallsizeclass_t sc) { UNUSED(sc); @@ -189,7 +189,7 @@ namespace snmalloc #endif } - void sizeclass_dealloc(sizeclass_t sc) + void sizeclass_dealloc(smallsizeclass_t sc) { UNUSED(sc); @@ -209,7 +209,7 @@ namespace snmalloc #endif } - void sizeclass_alloc_slab(sizeclass_t sc) + void sizeclass_alloc_slab(smallsizeclass_t sc) { UNUSED(sc); @@ -219,7 +219,7 @@ namespace snmalloc #endif } - void sizeclass_dealloc_slab(sizeclass_t sc) + void sizeclass_dealloc_slab(smallsizeclass_t sc) { UNUSED(sc); @@ -266,7 +266,7 @@ namespace snmalloc #endif } - void remote_free(sizeclass_t sc) + void remote_free(smallsizeclass_t sc) { UNUSED(sc); @@ -282,7 +282,7 @@ namespace snmalloc #endif } - void remote_receive(sizeclass_t sc) + void remote_receive(smallsizeclass_t sc) { UNUSED(sc); @@ -374,7 +374,7 @@ namespace snmalloc << "Count" << csv.endl; } - for (sizeclass_t i = 0; i < N; i++) + for (smallsizeclass_t i = 0; i < N; i++) { if (sizeclass[i].count.is_unused()) continue; @@ -387,15 +387,15 @@ namespace snmalloc sizeclass[i].print(csv, sizeclass_to_size(i)); } - for (uint8_t i = 0; i < LARGE_N; i++) - { - if ((large_push_count[i] == 0) && (large_pop_count[i] == 0)) - continue; + // for (uint8_t i = 0; i < LARGE_N; i++) + // { + // if ((large_push_count[i] == 0) && (large_pop_count[i] == 0)) + // continue; - csv << "LargeBucketedStats" << dumpid << allocatorid << (i + N) - << large_sizeclass_to_size(i) << large_push_count[i] - << large_pop_count[i] << csv.endl; - } + // csv << "LargeBucketedStats" << dumpid << allocatorid << (i + N) + // << large_sizeclass_to_size(i) << large_push_count[i] + // << large_pop_count[i] << csv.endl; + // } size_t low = 0; size_t high = 0; diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index 286d7d0..f2d69e9 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -182,7 +182,7 @@ namespace snmalloc typename SharedStateHandle::LocalState& local_state, ChunkAllocatorLocalState& chunk_alloc_local_state, sizeclass_t sizeclass, - sizeclass_t slab_sizeclass, // TODO sizeclass_t + chunksizeclass_t slab_sizeclass, size_t slab_size, RemoteAllocator* remote) { diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 1d714f5..5811079 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -56,7 +56,7 @@ namespace snmalloc /** * Per size class list of active slabs for this allocator. */ - MetaslabCache alloc_classes[NUM_SIZECLASSES]; + MetaslabCache alloc_classes[NUM_SMALL_SIZECLASSES]; /** * Local cache for the Chunk allocator. @@ -196,7 +196,9 @@ namespace snmalloc }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( - domesticate, size, [&](sizeclass_t sizeclass, freelist::Iter<>* fl) { + domesticate, + size, + [&](smallsizeclass_t sizeclass, freelist::Iter<>* fl) { return small_alloc(sizeclass, *fl); }); } @@ -285,7 +287,7 @@ namespace snmalloc bumpptr = slab_end; } - ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) + ChunkRecord* clear_slab(Metaslab* meta, smallsizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); freelist::Iter<> fl; @@ -347,7 +349,7 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH void dealloc_local_slabs(sizeclass_t sizeclass) + SNMALLOC_SLOW_PATH void dealloc_local_slabs(smallsizeclass_t sizeclass) { // Return unused slabs of sizeclass_t back to global allocator alloc_classes[sizeclass].available.filter([this, @@ -400,7 +402,7 @@ namespace snmalloc // TODO: Handle message queue on this path? Metaslab* meta = entry.get_metaslab(); - sizeclass_t sizeclass = entry.get_sizeclass(); + smallsizeclass_t sizeclass = entry.get_sizeclass().as_small(); UNUSED(entropy); if (meta->is_sleeping()) @@ -557,11 +559,11 @@ namespace snmalloc get_backend_local_state(), chunk_local_state); #ifndef NDEBUG - for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) + for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) { size_t size = sizeclass_to_size(i); - sizeclass_t sc1 = size_to_sizeclass(size); - sizeclass_t sc2 = size_to_sizeclass_const(size); + smallsizeclass_t sc1 = size_to_sizeclass(size); + smallsizeclass_t sc2 = size_to_sizeclass_const(size); size_t size1 = sizeclass_to_size(sc1); size_t size2 = sizeclass_to_size(sc2); @@ -662,7 +664,8 @@ namespace snmalloc SNMALLOC_ASSERT(!meta->is_unused()); check_client( - Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), + Metaslab::is_start_of_object( + entry.get_sizeclass().as_small(), address_cast(p)), "Not deallocating start of an object"); auto cp = p.as_static>(); @@ -677,7 +680,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH capptr::Alloc - small_alloc(sizeclass_t sizeclass, freelist::Iter<>& fast_free_list) + small_alloc(smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { // Look to see if we can grab a free list. auto& sl = alloc_classes[sizeclass].available; @@ -737,8 +740,8 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH capptr::Alloc - small_alloc_slow(sizeclass_t sizeclass, freelist::Iter<>& fast_free_list) + SNMALLOC_SLOW_PATH capptr::Alloc small_alloc_slow( + smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -755,7 +758,7 @@ namespace snmalloc snmalloc::ChunkAllocator::alloc_chunk( get_backend_local_state(), chunk_local_state, - sizeclass, + sizeclass_t::from_small_class(sizeclass), slab_sizeclass, slab_size, public_state()); @@ -831,7 +834,8 @@ namespace snmalloc [&](capptr::Alloc p) { dealloc_local_object(p); }); // We may now have unused slabs, return to the global allocator. - for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) + for (smallsizeclass_t sizeclass = 0; sizeclass < NUM_SMALL_SIZECLASSES; + sizeclass++) { dealloc_local_slabs(sizeclass); } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index fa9d2c7..f994a4b 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -183,7 +183,7 @@ namespace snmalloc auto [chunk, meta] = ChunkAllocator::alloc_chunk( core_alloc->get_backend_local_state(), core_alloc->chunk_local_state, - bits::next_pow2_bits(size), // TODO + size_to_sizeclass_full(size), large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), SharedStateHandle::fake_large_remote); @@ -211,21 +211,20 @@ namespace snmalloc template SNMALLOC_FAST_PATH capptr::Alloc small_alloc(size_t size) { - // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate( core_alloc->backend_state_ptr(), p); }; auto slowpath = [&]( - sizeclass_t sizeclass, + smallsizeclass_t sizeclass, freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { if (likely(core_alloc != nullptr)) { return core_alloc->handle_message_queue( []( CoreAlloc* core_alloc, - sizeclass_t sizeclass, + smallsizeclass_t sizeclass, freelist::Iter<>* fl) { return core_alloc->template small_alloc(sizeclass, *fl); }, @@ -234,7 +233,7 @@ namespace snmalloc fl); } return lazy_init( - [&](CoreAlloc*, sizeclass_t sizeclass) { + [&](CoreAlloc*, smallsizeclass_t sizeclass) { return small_alloc(sizeclass_to_size(sizeclass)); }, sizeclass); @@ -429,7 +428,8 @@ namespace snmalloc #else // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. - if (likely((size - 1) <= (sizeclass_to_size(NUM_SIZECLASSES - 1) - 1))) + if (likely( + (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) { // Small allocations are more likely. Improve // branch prediction by placing this case first. @@ -446,7 +446,6 @@ namespace snmalloc template SNMALLOC_FAST_PATH ALLOCATOR void* alloc() { - // TODO optimise return alloc(size); } @@ -455,7 +454,6 @@ namespace snmalloc #ifdef SNMALLOC_PASS_THROUGH external_alloc::free(p_raw); #else - // TODO: // Care is needed so that dealloc(nullptr) works before init // The backend allocator must ensure that a minimal page map exists // before init, that maps null to a remote_deallocator that will never @@ -508,16 +506,10 @@ namespace snmalloc } // Large deallocation or null. - if (likely(p_tame != nullptr)) + // also checks for managed by page map. + if (likely((p_tame != nullptr) && !entry.get_sizeclass().is_default())) { - size_t entry_sizeclass = entry.get_sizeclass(); - - // Check this is managed by this pagemap. - // - // TODO: Should this be tested even in the !CHECK_CLIENT case? Things - // go fairly pear-shaped, with the ASM's ranges[] getting cross-linked - // with a ChunkAllocator's chunk_stack[0], which seems bad. - check_client(entry_sizeclass != 0, "Not allocated by snmalloc."); + size_t entry_sizeclass = entry.get_sizeclass().as_large(); size_t size = bits::one_at_bit(entry_sizeclass); size_t slab_sizeclass = @@ -558,6 +550,11 @@ namespace snmalloc return; } + // If p_tame is not null, then dealloc has been call on something + // it shouldn't be called on. + // TODO: Should this be tested even in the !CHECK_CLIENT case? + check_client(p_tame == nullptr, "Not allocated by snmalloc."); + # ifdef SNMALLOC_TRACING std::cout << "nullptr deallocation" << std::endl; # endif @@ -611,14 +608,7 @@ namespace snmalloc MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p_raw)); - if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) - return sizeclass_to_size(entry.get_sizeclass()); - - // Sizeclass zero is for large is actually zero - if (likely(entry.get_sizeclass() != 0)) - return bits::one_at_bit(entry.get_sizeclass()); - - return 0; + return sizeclass_full_to_size(entry.get_sizeclass()); #endif } @@ -630,61 +620,65 @@ namespace snmalloc * the potential pointer space. */ template - void* external_pointer(void* p_raw) + void* external_pointer(void* p) + { + // Note that each case uses `pointer_offset`, so that on + // CHERI it is monotone with respect to the capability. + // Note that the returned pointer could be outside the CHERI + // bounds of `p`, and thus not something that can be followed. + if constexpr (location == Start) + { + size_t index = index_in_object(p); + return pointer_offset(p, 0 - index); + } + else if constexpr (location == End) + { + return pointer_offset(p, remaining_bytes(p) - 1); + } + else + { + return pointer_offset(p, remaining_bytes(p)); + } + } + + /** + * Returns the number of remaining bytes in an object. + * + * auto p = (char*)malloc(size) + * remaining_bytes(p + n) == size - n provided n < size + */ + size_t remaining_bytes(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - // TODO What's the domestication policy here? At the moment we just - // probe the pagemap with the raw address, without checks. There could - // be implicit domestication through the `SharedStateHandle::Pagemap` or - // we could just leave well enough alone. - - capptr::AllocWild p = capptr_from_client(p_raw); - MetaEntry entry = SharedStateHandle::Pagemap::template get_metaentry( core_alloc->backend_state_ptr(), address_cast(p)); + auto sizeclass = entry.get_sizeclass(); - if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) - { - auto rsize = sizeclass_to_size(sizeclass); - auto offset = address_cast(p) & (sizeclass_to_slab_size(sizeclass) - 1); - auto start_offset = round_by_sizeclass(sizeclass, offset); - if constexpr (location == Start) - { - UNUSED(rsize); - return capptr_reveal_wild(pointer_offset(p, start_offset - offset)); - } - else if constexpr (location == End) - return capptr_reveal_wild( - pointer_offset(p, rsize + start_offset - offset - 1)); - else - return capptr_reveal_wild( - pointer_offset(p, rsize + start_offset - offset)); - } - - // Sizeclass zero of a large allocation is used for not managed by us. - if (likely(sizeclass != 0)) - { - // This is a large allocation, find start by masking. - auto rsize = bits::one_at_bit(sizeclass); - auto start = pointer_align_down(p, rsize); - if constexpr (location == Start) - return capptr_reveal_wild(start); - else if constexpr (location == End) - return capptr_reveal_wild(pointer_offset(start, rsize - 1)); - else - return capptr_reveal_wild(pointer_offset(start, rsize)); - } + return snmalloc::remaining_bytes(sizeclass, address_cast(p)); #else - UNUSED(p_raw); + return pointer_diff(p, reinterpret_cast(UINTPTR_MAX)); #endif + } - if constexpr ((location == End) || (location == OnePastEnd)) - // We don't know the End, so return MAX_PTR - return reinterpret_cast(UINTPTR_MAX); - else - // We don't know the Start, so return MIN_PTR - return nullptr; + /** + * Returns the byte offset into an object. + * + * auto p = (char*)malloc(size) + * index_in_object(p + n) == n provided n < size + */ + size_t index_in_object(const void* p) + { +#ifndef SNMALLOC_PASS_THROUGH + MetaEntry entry = + SharedStateHandle::Pagemap::template get_metaentry( + core_alloc->backend_state_ptr(), address_cast(p)); + + auto sizeclass = entry.get_sizeclass(); + return snmalloc::index_in_object(sizeclass, address_cast(p)); +#else + return reinterpret_cast(p); +#endif } /** diff --git a/src/mem/localcache.h b/src/mem/localcache.h index b7c2be7..6e98bb8 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -10,10 +10,10 @@ namespace snmalloc { - using Stats = AllocStats; + using Stats = AllocStats; inline static SNMALLOC_FAST_PATH capptr::Alloc - finish_alloc_no_zero(freelist::HeadPtr p, sizeclass_t sizeclass) + finish_alloc_no_zero(freelist::HeadPtr p, smallsizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -23,7 +23,7 @@ namespace snmalloc template inline static SNMALLOC_FAST_PATH capptr::Alloc - finish_alloc(freelist::HeadPtr p, sizeclass_t sizeclass) + finish_alloc(freelist::HeadPtr p, smallsizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); @@ -45,7 +45,7 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - freelist::Iter<> small_fast_free_lists[NUM_SIZECLASSES] = {}; + freelist::Iter<> small_fast_free_lists[NUM_SMALL_SIZECLASSES] = {}; // This is the entropy for a particular thread. LocalEntropy entropy; @@ -83,7 +83,7 @@ namespace snmalloc return capptr_domesticate(local_state, p); }; - for (size_t i = 0; i < NUM_SIZECLASSES; i++) + for (size_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) { // TODO could optimise this, to return the whole list in one append // call. @@ -108,7 +108,7 @@ namespace snmalloc alloc(Domesticator domesticate, size_t size, Slowpath slowpath) { auto& key = entropy.get_free_list_key(); - sizeclass_t sizeclass = size_to_sizeclass(size); + smallsizeclass_t sizeclass = size_to_sizeclass(size); stats.alloc_request(size); stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 3576166..8ece19a 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -73,7 +73,7 @@ namespace snmalloc /** * Initialise Metaslab for a slab. */ - void initialise(sizeclass_t sizeclass) + void initialise(smallsizeclass_t sizeclass) { free_queue.init(); // Set up meta data as if the entire slab has been turned into a free @@ -112,7 +112,7 @@ namespace snmalloc * and will return true, otherwise it will return false. */ SNMALLOC_FAST_PATH bool - set_sleeping(sizeclass_t sizeclass, uint16_t remaining) + set_sleeping(smallsizeclass_t sizeclass, uint16_t remaining) { auto threshold = threshold_for_waking_slab(sizeclass); if (remaining >= threshold) @@ -130,7 +130,7 @@ namespace snmalloc return true; } - SNMALLOC_FAST_PATH void set_not_sleeping(sizeclass_t sizeclass) + SNMALLOC_FAST_PATH void set_not_sleeping(smallsizeclass_t sizeclass) { auto allocated = sizeclass_to_slab_object_count(sizeclass); needed() = allocated - threshold_for_waking_slab(sizeclass); @@ -145,9 +145,9 @@ namespace snmalloc } static SNMALLOC_FAST_PATH bool - is_start_of_object(sizeclass_t sizeclass, address_t p) + is_start_of_object(smallsizeclass_t sizeclass, address_t p) { - return is_multiple_of_sizeclass( + return divisible_by_sizeclass( sizeclass, p - (bits::align_down(p, sizeclass_to_slab_size(sizeclass)))); } @@ -170,7 +170,7 @@ namespace snmalloc Metaslab* meta, freelist::Iter<>& fast_free_list, LocalEntropy& entropy, - sizeclass_t sizeclass) + smallsizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); @@ -213,8 +213,8 @@ namespace snmalloc * * * log_2(size), at least MIN_CHUNK_BITS, for large allocations. * - * * a value in [0, NUM_SIZECLASSES] for small allocations. These may be - * directly passed to the sizeclass (not slab_sizeclass) functions of + * * a value in [0, NUM_SMALL_SIZECLASSES] for small allocations. These + * may be directly passed to the sizeclass (not slab_sizeclass) functions of * sizeclasstable.h * */ @@ -235,12 +235,15 @@ namespace snmalloc {} SNMALLOC_FAST_PATH - MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass) + MetaEntry( + Metaslab* meta, + RemoteAllocator* remote, + sizeclass_t sizeclass = sizeclass_t()) : meta(meta) { /* remote might be nullptr; cast to uintptr_t before offsetting */ remote_and_sizeclass = - pointer_offset(reinterpret_cast(remote), sizeclass); + pointer_offset(reinterpret_cast(remote), sizeclass.raw()); } /** @@ -285,8 +288,9 @@ namespace snmalloc { // TODO: perhaps remove static_cast with resolution of // https://github.com/CTSRD-CHERI/llvm-project/issues/588 - return static_cast(remote_and_sizeclass) & - (alignof(RemoteAllocator) - 1); + return sizeclass_t::from_raw( + static_cast(remote_and_sizeclass) & + (alignof(RemoteAllocator) - 1)); } }; diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 8056c53..5520593 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -12,17 +12,8 @@ namespace snmalloc { // Remotes need to be aligned enough that the bottom bits have enough room for // all the size classes, both large and small. - // - // Including large classes in this calculation might seem remarkably strange, - // since large allocations don't have associated Remotes, that is, their - // remote is taken to be 0. However, if there are very few small size - // classes and many large classes, the attempt to align that 0 down by the - // alignment of a Remote might result in a nonzero value. - static constexpr size_t REMOTE_MIN_ALIGN = bits::max( - CACHELINE_SIZE, - bits::max( - bits::next_pow2_const(NUM_SIZECLASSES + 1), - bits::next_pow2_const(NUM_LARGE_CLASSES + 1))); + static constexpr size_t REMOTE_MIN_ALIGN = + bits::max(CACHELINE_SIZE, SIZECLASS_REP_SIZE); /** * Global key for all remote lists. diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 46cb428..2ecb056 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -55,7 +55,7 @@ namespace snmalloc SNMALLOC_FAST_PATH bool reserve_space(const MetaEntry& entry) { auto size = - static_cast(sizeclass_to_size(entry.get_sizeclass())); + static_cast(sizeclass_full_to_size(entry.get_sizeclass())); bool result = capacity > size; if (result) diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index ebd091f..899d5c2 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -5,22 +5,28 @@ #include "../ds/helpers.h" #include "allocconfig.h" +/** + * This file contains all the code for transforming transforming sizes to + * sizeclasses and back. It also contains various sizeclass pre-calculated + * tables for operations based on size class such as `modulus` and `divisible + * by`, and constants for the slab based allocator. + * + * TODO: Due to the current structure for constexpr evaluation this file does + * not well delimit internal versus external APIs. Some refactoring should be + * done. + */ + namespace snmalloc { - // Both usings should compile - // We use size_t as it generates better code. - using sizeclass_t = size_t; - // using sizeclass_t = uint8_t; - using sizeclass_compress_t = uint8_t; + using smallsizeclass_t = size_t; + using chunksizeclass_t = size_t; - constexpr static uintptr_t SIZECLASS_MASK = 0xFF; - - constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) + constexpr static inline smallsizeclass_t size_to_sizeclass_const(size_t size) { // Don't use sizeclasses that are not a multiple of the alignment. // For example, 24 byte allocations can be // problematic for some data due to alignment issues. - auto sc = static_cast( + auto sc = static_cast( bits::to_exp_mant_const(size)); SNMALLOC_ASSERT(sc == static_cast(sc)); @@ -28,20 +34,98 @@ namespace snmalloc return sc; } - static inline size_t large_sizeclass_to_size(uint8_t large_class) - { - // TODO. Remove - UNUSED(large_class); - abort(); - // return bits::one_at_bit(large_class + SUPERSLAB_BITS); - } + static constexpr size_t NUM_SMALL_SIZECLASSES = + size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE); - static constexpr size_t NUM_SIZECLASSES = - size_to_sizeclass_const(MAX_SIZECLASS_SIZE); - - // Large classes range from [SUPERSLAB, ADDRESS_SPACE).// TODO + // Large classes range from [MAX_SMALL_SIZECLASS_SIZE, ADDRESS_SPACE). static constexpr size_t NUM_LARGE_CLASSES = - Pal::address_bits - MAX_SIZECLASS_BITS; + Pal::address_bits - MAX_SMALL_SIZECLASS_BITS; + + // How many bits are required to represent either a large or a small + // sizeclass. + static constexpr size_t TAG_SIZECLASS_BITS = bits::max( + bits::next_pow2_bits_const(NUM_SMALL_SIZECLASSES + 1), + bits::next_pow2_bits_const(NUM_LARGE_CLASSES + 1)); + + // Number of bits required to represent a tagged sizeclass that can be + // either small or large. + static constexpr size_t SIZECLASS_REP_SIZE = + bits::one_at_bit(TAG_SIZECLASS_BITS + 1); + + /** + * Encapsulates a tagged union of large and small sizeclasses. + * + * Used in various lookup tables to make efficient code that handles + * all objects allocated by snmalloc. + */ + class sizeclass_t + { + static constexpr size_t TAG = bits::one_at_bit(TAG_SIZECLASS_BITS); + + size_t value{0}; + + constexpr sizeclass_t(size_t value) : value(value) {} + + public: + constexpr sizeclass_t() = default; + + constexpr static sizeclass_t from_small_class(smallsizeclass_t sc) + { + SNMALLOC_ASSERT(sc < TAG); + // Note could use `+` or `|`. Using `+` as will combine nicely with array + // offset. + return {TAG + sc}; + } + + /** + * Takes the number of leading zero bits from the actual large size-1. + * See size_to_sizeclass_full + */ + constexpr static sizeclass_t from_large_class(size_t large_class) + { + SNMALLOC_ASSERT(large_class < TAG); + return {large_class}; + } + + constexpr static sizeclass_t from_raw(size_t raw) + { + return {raw}; + } + + constexpr size_t index() + { + return value & (TAG - 1); + } + + constexpr smallsizeclass_t as_small() + { + SNMALLOC_ASSERT(is_small()); + return value & (TAG - 1); + } + + constexpr chunksizeclass_t as_large() + { + SNMALLOC_ASSERT(!is_small()); + return bits::BITS - (value & (TAG - 1)); + } + + constexpr size_t raw() + { + return value; + } + + constexpr bool is_small() + { + return (value & TAG) != 0; + } + + constexpr bool is_default() + { + return value == 0; + } + }; + + using sizeclass_compress_t = uint8_t; inline SNMALLOC_FAST_PATH static size_t aligned_size(size_t alignment, size_t size) @@ -64,9 +148,9 @@ namespace snmalloc // the slab. size_t slab_mask; // Table of constants for reciprocal division for each sizeclass. - size_t div_mult; - // Table of constants for reciprocal modulus for each sizeclass. size_t mod_mult; + // Table of constants for reciprocal modulus for each sizeclass. + size_t mod_zero_mult; }; /** @@ -81,64 +165,118 @@ namespace snmalloc struct SizeClassTable { - ModArray fast; - ModArray slow; + ModArray fast_; + ModArray slow_; - constexpr SizeClassTable() : fast(), slow() + [[nodiscard]] constexpr sizeclass_data_fast& fast(sizeclass_t index) { - for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + return fast_[index.raw()]; + } + + [[nodiscard]] constexpr sizeclass_data_fast fast(sizeclass_t index) const + { + return fast_[index.raw()]; + } + + [[nodiscard]] constexpr sizeclass_data_fast& fast_small(smallsizeclass_t sc) + { + return fast_[sizeclass_t::from_small_class(sc).raw()]; + } + + [[nodiscard]] constexpr sizeclass_data_fast + fast_small(smallsizeclass_t sc) const + { + return fast_[sizeclass_t::from_small_class(sc).raw()]; + } + + [[nodiscard]] constexpr sizeclass_data_slow& slow(sizeclass_t index) + { + return slow_[index.raw()]; + } + + [[nodiscard]] constexpr sizeclass_data_slow slow(sizeclass_t index) const + { + return slow_[index.raw()]; + } + + constexpr SizeClassTable() : fast_(), slow_() + { + for (sizeclass_compress_t sizeclass = 0; + sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { + auto& meta = fast_small(sizeclass); + size_t rsize = bits::from_exp_mant(sizeclass); - fast[sizeclass].size = rsize; + meta.size = rsize; size_t slab_bits = bits::max( bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS); - fast[sizeclass].slab_mask = bits::one_at_bit(slab_bits) - 1; + meta.slab_mask = bits::one_at_bit(slab_bits) - 1; - slow[sizeclass].capacity = - static_cast((fast[sizeclass].slab_mask + 1) / rsize); + auto& meta_slow = slow(sizeclass_t::from_small_class(sizeclass)); + meta_slow.capacity = + static_cast((meta.slab_mask + 1) / rsize); - slow[sizeclass].waking = + meta_slow.waking = #ifdef SNMALLOC_CHECK_CLIENT - static_cast(slow[sizeclass].capacity / 4); + static_cast(meta_slow.capacity / 4); #else - static_cast(bits::min((slow[sizeclass].capacity / 4), 32)); + static_cast(bits::min((meta_slow.capacity / 4), 32)); #endif } - for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + for (sizeclass_compress_t sizeclass = 0; + sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { - fast[sizeclass].div_mult = // TODO is MAX_SIZECLASS_BITS right? - (bits::one_at_bit(bits::BITS - 24) / - (fast[sizeclass].size / MIN_ALLOC_SIZE)); - if (!bits::is_pow2(fast[sizeclass].size)) - fast[sizeclass].div_mult++; + // Calculate reciprocal modulus constant like reciprocal division, but + // constant is choosen to overflow and only leave the modulus as the + // result. + auto& meta = fast_small(sizeclass); + meta.mod_mult = bits::one_at_bit(bits::BITS - 1) / meta.size; + meta.mod_mult *= 2; - fast[sizeclass].mod_mult = - (bits::one_at_bit(bits::BITS - 1) / fast[sizeclass].size); - if (!bits::is_pow2(fast[sizeclass].size)) - fast[sizeclass].mod_mult++; - // Shift multiplier, so that the result of division completely - // overflows, and thus the top SUPERSLAB_BITS will be zero if the mod is - // zero. - fast[sizeclass].mod_mult *= 2; + if (bits::is_pow2(meta.size)) + { + // Set to zero, so masking path is taken if power of 2. + meta.mod_mult = 0; + } + + size_t zero = 0; + meta.mod_zero_mult = (~zero / meta.size) + 1; + } + + // Set up table for large classes. + // Note skipping sizeclass == 0 as this is size == 0, so the tables can be + // all zero. + for (size_t sizeclass = 1; sizeclass < bits::BITS; sizeclass++) + { + auto lsc = sizeclass_t::from_large_class(sizeclass); + fast(lsc).size = bits::one_at_bit(lsc.as_large()); + + // Use slab mask as 0 for power of two sizes. + fast(lsc).slab_mask = 0; } } }; static inline constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); - constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) + constexpr static inline size_t sizeclass_to_size(smallsizeclass_t sizeclass) { - return sizeclass_metadata.fast[sizeclass].size; + return sizeclass_metadata.fast_small(sizeclass).size; } - inline static size_t sizeclass_to_slab_size(sizeclass_t sizeclass) + static inline size_t sizeclass_full_to_size(sizeclass_t sizeclass) { - return sizeclass_metadata.fast[sizeclass].slab_mask + 1; + return sizeclass_metadata.fast(sizeclass).size; + } + + inline static size_t sizeclass_to_slab_size(smallsizeclass_t sizeclass) + { + return sizeclass_metadata.fast_small(sizeclass).slab_mask + 1; } /** @@ -148,19 +286,20 @@ namespace snmalloc * * It also increases entropy, when we have randomisation. */ - inline uint16_t threshold_for_waking_slab(sizeclass_t sizeclass) + inline uint16_t threshold_for_waking_slab(smallsizeclass_t sizeclass) { - return sizeclass_metadata.slow[sizeclass].waking; + return sizeclass_metadata.slow(sizeclass_t::from_small_class(sizeclass)) + .waking; } - inline static size_t sizeclass_to_slab_sizeclass(sizeclass_t sizeclass) + inline static size_t sizeclass_to_slab_sizeclass(smallsizeclass_t sizeclass) { size_t ssize = sizeclass_to_slab_size(sizeclass); return bits::next_pow2_bits(ssize) - MIN_CHUNK_BITS; } - inline static size_t slab_sizeclass_to_size(sizeclass_t sizeclass) + inline static size_t slab_sizeclass_to_size(chunksizeclass_t sizeclass) { return bits::one_at_bit(MIN_CHUNK_BITS + sizeclass); } @@ -170,74 +309,71 @@ namespace snmalloc * which must be shifted into the index space of slab_sizeclass-es. */ inline static size_t - metaentry_chunk_sizeclass_to_slab_sizeclass(sizeclass_t sizeclass) + metaentry_chunk_sizeclass_to_slab_sizeclass(chunksizeclass_t sizeclass) { return sizeclass - MIN_CHUNK_BITS; } inline constexpr static uint16_t - sizeclass_to_slab_object_count(sizeclass_t sizeclass) + sizeclass_to_slab_object_count(smallsizeclass_t sizeclass) { - return sizeclass_metadata.slow[sizeclass].capacity; + return sizeclass_metadata.slow(sizeclass_t::from_small_class(sizeclass)) + .capacity; } - inline static size_t round_by_sizeclass(sizeclass_t sc, size_t offset) + inline static size_t mod_by_sizeclass(smallsizeclass_t sc, size_t offset) { - // Only works up to certain offsets, exhaustively tested upto - // SUPERSLAB_SIZE. - // SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); + // Only works up to certain offsets, exhaustively tested by rounding.cc + auto meta = sizeclass_metadata.fast_small(sc); - auto rsize = sizeclass_to_size(sc); + // Powers of two should use straigt mask. + SNMALLOC_ASSERT(meta.mod_mult != 0); if constexpr (sizeof(offset) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. - // The code is using reciprocal division. If SUPERSLABS - // get larger then we should review this code. For 24 bits, there are in - // sufficient bits to do this completely efficiently as 24 * 3 is larger - // than 64 bits. But we can pre-round by MIN_ALLOC_SIZE which gets us an - // extra 4 * 3 bits, and thus achievable in 64bit multiplication. - // static_assert( - // SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); - - // TODO 24 hack - static_assert(bits::BITS >= 24, "About to attempt a negative shift"); - static_assert( - (8 * sizeof(offset)) >= (bits::BITS - 24), - "About to shift further than the type"); - return (((offset >> MIN_ALLOC_BITS) * - sizeclass_metadata.fast[sc].div_mult) >> - (bits::BITS - 24)) * - rsize; + // Could be made nicer with 128bit multiply (umulh): + // https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/ + auto bits_l = bits::BITS / 2; + auto bits_h = bits::BITS - bits_l; + return ( + ((((offset + 1) * meta.mod_mult) >> (bits_l)) * meta.size) >> bits_h); } else // Use 32-bit division as considerably faster than 64-bit, and // everything fits into 32bits here. - return static_cast(offset / rsize) * rsize; + return static_cast(offset % meta.size); } - inline static bool is_multiple_of_sizeclass(sizeclass_t sc, size_t offset) + inline static size_t index_in_object(sizeclass_t sc, address_t addr) { - // Only works up to certain offsets, exhaustively tested upto - // SUPERSLAB_SIZE. - // SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); + if (sizeclass_metadata.fast(sc).mod_mult == 0) + { + return addr & (sizeclass_metadata.fast(sc).size - 1); + } + + address_t offset = addr & (sizeclass_to_slab_size(sc.as_small()) - 1); + return mod_by_sizeclass(sc.as_small(), offset); + } + + inline static size_t remaining_bytes(sizeclass_t sc, address_t addr) + { + return sizeclass_metadata.fast(sc).size - index_in_object(sc, addr); + } + + inline static bool divisible_by_sizeclass(smallsizeclass_t sc, size_t offset) + { + // Only works up to certain offsets, exhaustively tested by rounding.cc if constexpr (sizeof(offset) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. - // The code is using reciprocal division. If SUPERSLABS - // get larger then we should review this code. The modulus code - // has fewer restrictions than division, as it only requires the - // square of the offset to be representable. - // TODO 24 hack. Redo the maths given the multiple - // slab sizes - static_assert(bits::BITS >= 25); - static constexpr size_t MASK = - ~(bits::one_at_bit(bits::BITS - 1 - 24) - 1); - - return ((offset * sizeclass_metadata.fast[sc].mod_mult) & MASK) == 0; + // This is based on: + // https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/ + auto mod_zero_mult = sizeclass_metadata.fast_small(sc).mod_zero_mult; + return (offset * mod_zero_mult) < mod_zero_mult; } else // Use 32-bit division as considerably faster than 64-bit, and @@ -262,10 +398,10 @@ namespace snmalloc return (s - 1) >> MIN_ALLOC_BITS; } - static inline sizeclass_t size_to_sizeclass(size_t size) + static inline smallsizeclass_t size_to_sizeclass(size_t size) { constexpr static size_t sizeclass_lookup_size = - sizeclass_lookup_index(MAX_SIZECLASS_SIZE); + sizeclass_lookup_index(MAX_SMALL_SIZECLASS_SIZE); /** * This struct is used to statically initialise a table for looking up @@ -278,10 +414,11 @@ namespace snmalloc constexpr SizeClassLookup() { size_t curr = 1; - for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + for (sizeclass_compress_t sizeclass = 0; + sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { - for (; curr <= sizeclass_metadata.fast[sizeclass].size; + for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; curr += 1 << MIN_ALLOC_BITS) { auto i = sizeclass_lookup_index(curr); @@ -307,9 +444,29 @@ namespace snmalloc return 0; } + /** + * A compressed size representation, + * either a small size class with the 7th bit set + * or a large class with the 7th bit not set. + * Large classes are stored as a mask shift. + * size = (~0 >> lc) + 1; + * Thus large size class 0, has size 0. + * And large size class 33, has size 2^31 + */ + static inline sizeclass_t size_to_sizeclass_full(size_t size) + { + if ((size - 1) < sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1)) + { + return sizeclass_t::from_small_class(size_to_sizeclass(size)); + } + // bits::clz is undefined on 0, but we have size == 1 has already been + // handled here. We conflate 0 and sizes larger than we can allocate. + return sizeclass_t::from_large_class(bits::clz(size - 1)); + } + inline SNMALLOC_FAST_PATH static size_t round_size(size_t size) { - if (size > sizeclass_to_size(NUM_SIZECLASSES - 1)) + if (size > sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1)) { return bits::next_pow2(size); } diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 82bbf9a..c193dc2 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -136,8 +136,7 @@ namespace auto& alloc = ThreadAlloc::get(); void* p = const_cast(ptr); - if (unlikely( - pointer_diff(ptr, alloc.external_pointer(p)) < len)) + if (unlikely(alloc.remaining_bytes(ptr) < len)) { if constexpr (FailFast) { diff --git a/src/override/rust.cc b/src/override/rust.cc index b54be6b..23f0c68 100644 --- a/src/override/rust.cc +++ b/src/override/rust.cc @@ -32,7 +32,8 @@ rust_realloc(void* ptr, size_t alignment, size_t old_size, size_t new_size) size_t aligned_old_size = aligned_size(alignment, old_size), aligned_new_size = aligned_size(alignment, new_size); if ( - size_to_sizeclass(aligned_old_size) == size_to_sizeclass(aligned_new_size)) + size_to_sizeclass_full(aligned_old_size).raw() == + size_to_sizeclass_full(aligned_new_size).raw()) return ptr; void* p = ThreadAlloc::get().alloc(aligned_new_size); if (p) diff --git a/src/test/func/first_operation/first_operation.cc b/src/test/func/first_operation/first_operation.cc index 0337b22..aa22635 100644 --- a/src/test/func/first_operation/first_operation.cc +++ b/src/test/func/first_operation/first_operation.cc @@ -166,7 +166,7 @@ int main(int, char**) f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::MAX_SIZECLASS_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) { auto shifted = [exp](size_t v) { return v << exp; }; diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index d6e5a58..80667f3 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -147,7 +147,7 @@ int main(int argc, char** argv) our_free(nullptr); - for (sizeclass_t sc = 0; sc < (MAX_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); printf("malloc: %zu\n", size); @@ -161,12 +161,13 @@ int main(int argc, char** argv) our_free(nullptr); - for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); bool overflow = false; - for (size_t n = 1; bits::umul(size, n, overflow) <= MAX_SIZECLASS_SIZE; + for (size_t n = 1; + bits::umul(size, n, overflow) <= MAX_SMALL_SIZECLASS_SIZE; n *= 5) { if (overflow) @@ -178,13 +179,13 @@ int main(int argc, char** argv) test_calloc(0, size, SUCCESS, false); } - for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); test_realloc(our_malloc(size), ((size_t)-1) / 2, ENOMEM, true); - for (sizeclass_t sc2 = 0; sc2 < NUM_SIZECLASSES; sc2++) + for (smallsizeclass_t sc2 = 0; sc2 < NUM_SMALL_SIZECLASSES; sc2++) { const size_t size2 = sizeclass_to_size(sc2); test_realloc(our_malloc(size), size2, SUCCESS, false); @@ -192,13 +193,13 @@ int main(int argc, char** argv) } } - for (sizeclass_t sc = 0; sc < (MAX_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); test_realloc(our_malloc(size), ((size_t)-1) / 2, ENOMEM, true); - for (sizeclass_t sc2 = 0; sc2 < (MAX_SIZECLASS_BITS + 4); sc2++) + for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); printf("size1: %zu, size2:%zu\n", size, size2); @@ -213,10 +214,10 @@ int main(int argc, char** argv) test_posix_memalign(((size_t)-1) / 2, 0, EINVAL, true); test_posix_memalign(OS_PAGE_SIZE, sizeof(uintptr_t) / 2, EINVAL, true); - for (size_t align = sizeof(uintptr_t); align < MAX_SIZECLASS_SIZE * 8; + for (size_t align = sizeof(uintptr_t); align < MAX_SMALL_SIZECLASS_SIZE * 8; align <<= 1) { - for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES - 6; sc++) + for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES - 6; sc++) { const size_t size = sizeclass_to_size(sc); test_posix_memalign(size, align, SUCCESS, false); diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index a0daef7..6185c5a 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -236,11 +236,18 @@ void test_external_pointer() // Malloc does not have an external pointer querying mechanism. auto& alloc = ThreadAlloc::get(); - for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (uint8_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) { size_t size = sizeclass_to_size(sc); void* p1 = alloc.alloc(size); + if (size != alloc.alloc_size(p1)) + { + std::cout << "Requested size: " << size + << " alloc_size: " << alloc.alloc_size(p1) << std::endl; + abort(); + } + for (size_t offset = 0; offset < size; offset += 17) { void* p2 = pointer_offset(p1, offset); @@ -248,8 +255,9 @@ void test_external_pointer() void* p4 = alloc.external_pointer(p2); if (p1 != p3) { - std::cout << "size: " << size << " offset: " << offset << " p1: " << p1 - << " p3: " << p3 << std::endl; + std::cout << "size: " << size << " alloc_size: " << alloc.alloc_size(p1) + << " offset: " << offset << " p1: " << p1 << " p3: " << p3 + << std::endl; } SNMALLOC_CHECK(p1 == p3); if ((size_t)p4 != (size_t)p1 + size - 1) @@ -272,7 +280,11 @@ void check_offset(void* base, void* interior) auto& alloc = ThreadAlloc::get(); void* calced_base = alloc.external_pointer((void*)interior); if (calced_base != (void*)base) + { + std::cout << "Calced base: " << calced_base << " actual base: " << base + << " for interior: " << interior << std::endl; abort(); + } } void check_external_pointer_large(size_t* base) @@ -301,7 +313,7 @@ void test_external_pointer_large() for (size_t i = 0; i < count; i++) { - size_t b = MAX_SIZECLASS_BITS + 3; + size_t b = MAX_SMALL_SIZECLASS_BITS + 3; size_t rand = r.next() & ((1 << b) - 1); size_t size = (1 << 24) + rand; total_size += size; @@ -383,7 +395,7 @@ void test_calloc_large_bug() // Some PALS have special paths for PAGE aligned zeroing of large // allocations. This is a large allocation that is intentionally // not a multiple of page size. - const size_t size = (MAX_SIZECLASS_SIZE << 3) - 7; + const size_t size = (MAX_SMALL_SIZECLASS_SIZE << 3) - 7; void* p1 = alloc.alloc(size); SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size); @@ -415,7 +427,7 @@ void test_static_sized_allocs() { // For each small, medium, and large class, do each kind dealloc. This is // mostly to ensure that all of these forms compile. - for (size_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (size_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) { // test_static_sized_alloc(); // test_static_sized_alloc(); @@ -430,6 +442,32 @@ void test_static_sized_allocs() // test_static_sized_alloc(); } +void test_remaining_bytes() +{ + auto& alloc = ThreadAlloc::get(); + for (size_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) + { + auto size = sizeclass_to_size(sc); + char* p = (char*)alloc.alloc(size); + for (size_t offset = 0; offset < size; offset++) + { + auto rem = alloc.remaining_bytes(p + offset); + if (rem != (size - offset)) + { + printf( + "Allocation size: %zu, Offset: %zu, Remaining bytes: %zu, " + "Expected: %zu\n", + size, + offset, + rem, + size - offset); + abort(); + } + } + alloc.dealloc(p); + } +} + int main(int argc, char** argv) { setup(); @@ -455,12 +493,12 @@ int main(int argc, char** argv) UNUSED(argc); UNUSED(argv); #endif - test_alloc_dealloc_64k(); test_random_allocation(); test_calloc(); test_double_alloc(); #ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features + test_remaining_bytes(); test_static_sized_allocs(); test_calloc_large_bug(); test_external_pointer_dealloc_bug(); diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index 305528f..72fe595 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -17,35 +17,37 @@ int main(int argc, char** argv) bool failed = false; - for (size_t size_class = 0; size_class < NUM_SIZECLASSES; size_class++) + for (size_t size_class = 0; size_class < NUM_SMALL_SIZECLASSES; size_class++) { size_t rsize = sizeclass_to_size((uint8_t)size_class); size_t max_offset = sizeclass_to_slab_size(size_class); + sizeclass_t sc = sizeclass_t::from_small_class(size_class); for (size_t offset = 0; offset < max_offset; offset++) { - size_t rounded = (offset / rsize) * rsize; + size_t mod = offset % rsize; bool mod_0 = (offset % rsize) == 0; - size_t opt_rounded = round_by_sizeclass(size_class, offset); - if (rounded != opt_rounded) + size_t opt_mod = index_in_object(sc, offset); + if (mod != opt_mod) { std::cout << "rsize " << rsize << " offset " << offset << " opt " - << opt_rounded << " correct " << rounded << std::endl + << opt_mod << " correct " << mod << std::endl << std::flush; failed = true; } - bool opt_mod_0 = is_multiple_of_sizeclass(size_class, offset); + bool opt_mod_0 = divisible_by_sizeclass(size_class, offset); if (opt_mod_0 != mod_0) { - std::cout << "rsize " << rsize << " offset " << offset << " opt_mod " - << opt_mod_0 << " correct " << mod_0 << std::endl + std::cout << "rsize " << rsize << " offset " << offset + << " opt_mod0 " << opt_mod_0 << " correct " << mod_0 + << std::endl << std::flush; failed = true; } } + if (failed) + abort(); } - if (failed) - abort(); return 0; } diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 8b2faf3..5f3c4aa 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -3,7 +3,7 @@ #include NOINLINE -snmalloc::sizeclass_t size_to_sizeclass(size_t size) +snmalloc::smallsizeclass_t size_to_sizeclass(size_t size) { return snmalloc::size_to_sizeclass(size); } @@ -15,7 +15,7 @@ void test_align_size() SNMALLOC_CHECK(snmalloc::aligned_size(128, 160) == 256); for (size_t size = 1; - size < snmalloc::sizeclass_to_size(snmalloc::NUM_SIZECLASSES - 1); + size < snmalloc::sizeclass_to_size(snmalloc::NUM_SMALL_SIZECLASSES - 1); size++) { size_t rsize = snmalloc::round_size(size); @@ -39,7 +39,7 @@ void test_align_size() } for (size_t alignment_bits = 0; - alignment_bits < snmalloc::MAX_SIZECLASS_BITS; + alignment_bits < snmalloc::MAX_SMALL_SIZECLASS_BITS; alignment_bits++) { auto alignment = (size_t)1 << alignment_bits; @@ -78,10 +78,11 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; size_t slab_size = 0; - for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++) + for (snmalloc::smallsizeclass_t sz = 0; sz < snmalloc::NUM_SMALL_SIZECLASSES; + sz++) { if ( - sz < snmalloc::NUM_SIZECLASSES && + sz < snmalloc::NUM_SMALL_SIZECLASSES && slab_size != snmalloc::sizeclass_to_slab_size(sz)) { slab_size = snmalloc::sizeclass_to_slab_size(sz); diff --git a/src/test/func/teardown/teardown.cc b/src/test/func/teardown/teardown.cc index 7bfdbff..b3d2645 100644 --- a/src/test/func/teardown/teardown.cc +++ b/src/test/func/teardown/teardown.cc @@ -188,7 +188,7 @@ int main(int, char**) f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::MAX_SIZECLASS_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) { auto shifted = [exp](size_t v) { return v << exp; };