Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #62 from microsoft/opt

Browse Source 
Optimise the fast path for alloc and dealloc
This commit is contained in:
Matthew Parkinson
2019-07-02 17:06:01 +01:00
committed by GitHub
parent bb6b8e45c2 4cf19f3a4c
commit 38f0b62e39
21 changed files with 601 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@@ -105,7 +105,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY)
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi")
set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG")
else()
add_compile_options(-march=native -fno-exceptions -fno-rtti -g)
add_compile_options(-march=native -fno-exceptions -fno-rtti -g -ftls-model=initial-exec)
endif()
macro(subdirlist result curdir)
@@ -120,7 +120,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY)
endmacro()
macro(add_shim name)
add_library(${name} SHARED src/override/malloc.cc)
add_library(${name} SHARED src/override/malloc.cc src/override/new.cc)
target_link_libraries(${name} snmalloc_lib)
target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))")
set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden)

2
azure-pipelines.yml
View File

@@ -172,8 +172,8 @@ phases:
cmakeArgs: '..'
- script: |
if [ "$(git diff $(Build.SourceVersion))" != "" ]; then exit 1; fi
make clangformat
if [ "$(git diff $(Build.SourceVersion))" != "" ]; then exit 1; fi
workingDirectory: build
displayName: 'Clang-Format'

19
difference.md
View File

@@ -18,3 +18,22 @@ This document outlines the changes that have diverged from
The value 1, is never a valid bump allocation value, as we initially
allocate the first entry as the link, so we can use 1 as the no more bump
space value.
2. Separate Bump/Free list. We have separate bump ptr and free list. This
is required to have a "fast free list" in each allocator for each
sizeclass. We bump allocate a whole os page (4KiB) worth of allocations
in one go, so that the CPU predicts the free list path for the fast
path.
3. Per allocator per sizeclass fast free list. Each allocator has an array
for each small size class that contains a free list of some elements for
that sizeclass. This enables a very compressed path for the common
allocation case.
4. We now store a direct pointer to the next element in each slabs free list
rather than a relative offset into the slab. This enables list
calculation on the fast path.
[2-4] Are changes that are directly inspired by
(mimalloc)[http://github.com/microsoft/mimalloc].

31
src/ds/bits.h
View File

@@ -11,6 +11,9 @@
# define HEADER_GLOBAL __declspec(selectany)
# define likely(x) !!(x)
# define unlikely(x) !!(x)
# define SNMALLOC_SLOW_PATH NOINLINE
# define SNMALLOC_FAST_PATH ALWAYSINLINE
# define SNMALLOC_PURE
#else
# define likely(x) __builtin_expect(!!(x), 1)
# define unlikely(x) __builtin_expect(!!(x), 0)
@@ -18,6 +21,9 @@
# include <emmintrin.h>
# define ALWAYSINLINE __attribute__((always_inline))
# define NOINLINE __attribute__((noinline))
# define SNMALLOC_SLOW_PATH NOINLINE
# define SNMALLOC_FAST_PATH inline ALWAYSINLINE
# define SNMALLOC_PURE __attribute__((const))
# ifdef __clang__
# define HEADER_GLOBAL __attribute__((selectany))
# else
@@ -52,13 +58,17 @@
#define UNUSED(x) ((void)(x))
#if __has_builtin(__builtin_assume)
# define SNMALLOC_ASSUME(x) __builtin_assume(x)
#ifndef NDEBUG
# define SNMALLOC_ASSUME(x) assert(x)
#else
# define SNMALLOC_ASSUME(x) \
do \
{ \
} while (0)
# if __has_builtin(__builtin_assume)
# define SNMALLOC_ASSUME(x) __builtin_assume((x))
# else
# define SNMALLOC_ASSUME(x) \
do \
{ \
} while (0)
# endif
#endif
// #define USE_LZCNT
@@ -120,6 +130,15 @@ namespace snmalloc
#endif
}
inline void prefetch(void* ptr)
{
#if defined(PLATFORM_IS_X86)
_mm_prefetch(reinterpret_cast<const char*>(ptr), _MM_HINT_T0);
#else
# warning "Missing prefetch intrinsic"
#endif
}
inline uint64_t tick()
{
#if defined(PLATFORM_IS_X86)

2
src/ds/helpers.h
View File

@@ -27,7 +27,7 @@ namespace snmalloc
// If defined should be initially false;
assert(first == nullptr || *first == false);
if (!initialised.load(std::memory_order_acquire))
if (unlikely(!initialised.load(std::memory_order_acquire)))
{
FlagLock lock(flag);
if (!initialised)

13
src/ds/mpscq.h
View File

@@ -3,6 +3,7 @@
#include "bits.h"
#include "helpers.h"
#include <utility>
namespace snmalloc
{
template<class T>
@@ -13,8 +14,8 @@ namespace snmalloc
std::is_same<decltype(T::next), std::atomic<T*>>::value,
"T->next must be a std::atomic<T*>");
std::atomic<T*> back;
T* front;
std::atomic<T*> back = nullptr;
T* front = nullptr;
public:
void invariant()
@@ -59,7 +60,7 @@ namespace snmalloc
prev->next.store(first, std::memory_order_relaxed);
}
T* dequeue()
std::pair<T*, bool> dequeue()
{
// Returns the front message, or null if not possible to return a message.
invariant();
@@ -69,14 +70,14 @@ namespace snmalloc
if (next != nullptr)
{
front = next;
bits::prefetch(&(next->next));
assert(front);
std::atomic_thread_fence(std::memory_order_acquire);
invariant();
return first;
return std::pair(first, true);
}
return nullptr;
return std::pair(nullptr, false);
}
};
} // namespace snmalloc

283
src/mem/alloc.h
View File

@@ -1,10 +1,5 @@
#pragma once
#if !defined(NDEBUG) && !defined(OPEN_ENCLAVE) && !defined(FreeBSD_KERNEL) && \
!defined(USE_SNMALLOC_STATS)
# define USE_SNMALLOC_STATS
#endif
#ifdef _MSC_VER
# define ALLOCATOR __declspec(allocator)
#else
@@ -226,6 +221,19 @@ namespace snmalloc
# define SNMALLOC_DEFAULT_PAGEMAP snmalloc::SuperslabMap<>
#endif
// This class is just used so that the free lists are the first entry
// in the allocator and hence has better code gen.
// It contains a free list per small size class. These are used for
// allocation on the fast path. This part of the code is inspired by mimalloc.
class FastFreeLists
{
protected:
FreeListHead small_fast_free_lists[NUM_SMALL_CLASSES];
public:
FastFreeLists() : small_fast_free_lists() {}
};
/**
* Allocator. This class is parameterised on three template parameters. The
* `MemoryProvider` defines the source of memory for this allocator.
@@ -247,7 +255,8 @@ namespace snmalloc
class PageMap = SNMALLOC_DEFAULT_PAGEMAP,
bool IsQueueInline = true>
class Allocator
: public Pooled<Allocator<MemoryProvider, PageMap, IsQueueInline>>
: public FastFreeLists,
public Pooled<Allocator<MemoryProvider, PageMap, IsQueueInline>>
{
LargeAlloc<MemoryProvider> large_allocator;
PageMap page_map;
@@ -277,32 +286,31 @@ namespace snmalloc
else
return calloc(1, size);
#else
constexpr uint8_t sizeclass = size_to_sizeclass_const(size);
constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size);
stats().alloc_request(size);
handle_message_queue();
// Allocate memory of a statically known size.
if constexpr (sizeclass < NUM_SMALL_CLASSES)
{
constexpr size_t rsize = sizeclass_to_size(sizeclass);
return small_alloc<zero_mem, allow_reserve>(sizeclass, rsize);
return small_alloc<zero_mem, allow_reserve>(size);
}
else if constexpr (sizeclass < NUM_SIZECLASSES)
{
handle_message_queue();
constexpr size_t rsize = sizeclass_to_size(sizeclass);
return medium_alloc<zero_mem, allow_reserve>(sizeclass, rsize, size);
}
else
{
handle_message_queue();
return large_alloc<zero_mem, allow_reserve>(size);
}
#endif
}
template<ZeroMem zero_mem = NoZero, AllowReserve allow_reserve = YesReserve>
ALLOCATOR void* alloc(size_t size)
inline ALLOCATOR void* alloc(size_t size)
{
#ifdef USE_MALLOC
static_assert(
@@ -315,18 +323,30 @@ namespace snmalloc
#else
stats().alloc_request(size);
handle_message_queue();
uint8_t sizeclass = size_to_sizeclass(size);
// Allocate memory of a dynamically known size.
if (sizeclass < NUM_SMALL_CLASSES)
// Perform the - 1 on size, so that zero wraps around and ends up on
// slow path.
if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1)))
{
// Allocations smaller than the slab size are more likely. Improve
// branch prediction by placing this case first.
size_t rsize = sizeclass_to_size(sizeclass);
return small_alloc<zero_mem, allow_reserve>(sizeclass, rsize);
return small_alloc<zero_mem, allow_reserve>(size);
}
return alloc_not_small<zero_mem, allow_reserve>(size);
}
template<ZeroMem zero_mem = NoZero, AllowReserve allow_reserve = YesReserve>
SNMALLOC_SLOW_PATH ALLOCATOR void* alloc_not_small(size_t size)
{
handle_message_queue();
if (size == 0)
{
return small_alloc<zero_mem, allow_reserve>(1);
}
sizeclass_t sizeclass = size_to_sizeclass(size);
if (sizeclass < NUM_SIZECLASSES)
{
size_t rsize = sizeclass_to_size(sizeclass);
@@ -346,7 +366,7 @@ namespace snmalloc
return free(p);
#else
constexpr uint8_t sizeclass = size_to_sizeclass_const(size);
constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size);
handle_message_queue();
@@ -389,7 +409,7 @@ namespace snmalloc
// Free memory of a dynamically known size. Must be called with an
// external pointer.
uint8_t sizeclass = size_to_sizeclass(size);
sizeclass_t sizeclass = size_to_sizeclass(size);
if (sizeclass < NUM_SMALL_CLASSES)
{
@@ -418,25 +438,19 @@ namespace snmalloc
#endif
}
void dealloc(void* p)
SNMALLOC_FAST_PATH void dealloc(void* p)
{
#ifdef USE_MALLOC
return free(p);
#else
handle_message_queue();
// Free memory of an unknown size. Must be called with an external
// pointer.
uint8_t size = pagemap().get(address_cast(p));
if (size == 0)
{
error("Not allocated by this allocator");
}
Superslab* super = Superslab::get(p);
if (size == PMSuperslab)
if (likely(size == PMSuperslab))
{
RemoteAllocator* target = super->get_allocator();
Slab* slab = Slab::get(p);
@@ -445,14 +459,24 @@ namespace snmalloc
// Reading a remote sizeclass won't fail, since the other allocator
// can't reuse the slab, as we have not yet deallocated this
// pointer.
uint8_t sizeclass = meta.sizeclass;
sizeclass_t sizeclass = meta.sizeclass;
if (super->get_allocator() == public_state())
if (likely(super->get_allocator() == public_state()))
small_dealloc(super, p, sizeclass);
else
remote_dealloc(target, p, sizeclass);
return;
}
dealloc_not_small(p, size);
}
SNMALLOC_SLOW_PATH void dealloc_not_small(void* p, uint8_t size)
{
handle_message_queue();
if (p == nullptr)
return;
if (size == PMMediumslab)
{
Mediumslab* slab = Mediumslab::get(p);
@@ -460,7 +484,7 @@ namespace snmalloc
// Reading a remote sizeclass won't fail, since the other allocator
// can't reuse the slab, as we have no yet deallocated this pointer.
uint8_t sizeclass = slab->get_sizeclass();
sizeclass_t sizeclass = slab->get_sizeclass();
if (target == public_state())
medium_dealloc(slab, p, sizeclass);
@@ -469,7 +493,13 @@ namespace snmalloc
return;
}
if (size == 0)
{
error("Not allocated by this allocator");
}
# ifdef CHECK_CLIENT
Superslab* super = Superslab::get(p);
if (size > 64 || address_cast(super) != address_cast(p))
{
error("Not deallocating start of an object");
@@ -494,7 +524,7 @@ namespace snmalloc
Slab* slab = Slab::get(p);
Metaslab& meta = super->get_meta(slab);
uint8_t sc = meta.sizeclass;
sizeclass_t sc = meta.sizeclass;
size_t slab_end = static_cast<size_t>(address_cast(slab) + SLAB_SIZE);
return external_pointer<location>(p, sc, slab_end);
@@ -503,7 +533,7 @@ namespace snmalloc
{
Mediumslab* slab = Mediumslab::get(p);
uint8_t sc = slab->get_sizeclass();
sizeclass_t sc = slab->get_sizeclass();
size_t slab_end =
static_cast<size_t>(address_cast(slab) + SUPERSLAB_SIZE);
@@ -622,7 +652,8 @@ namespace snmalloc
return (id >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK;
}
void dealloc(alloc_id_t target_id, void* p, uint8_t sizeclass)
SNMALLOC_FAST_PATH void
dealloc(alloc_id_t target_id, void* p, sizeclass_t sizeclass)
{
this->size += sizeclass_to_size(sizeclass);
@@ -698,7 +729,6 @@ namespace snmalloc
DLList<Superslab> super_only_short_available;
RemoteCache remote;
Remote stub;
std::conditional_t<IsQueueInline, RemoteAllocator, RemoteAllocator*>
remote_alloc;
@@ -706,7 +736,7 @@ namespace snmalloc
#ifdef CACHE_FRIENDLY_OFFSET
size_t remote_offset = 0;
void* apply_cache_friendly_offset(void* p, uint8_t sizeclass)
void* apply_cache_friendly_offset(void* p, sizeclass_t sizeclass)
{
size_t mask = sizeclass_to_cache_friendly_mask(sizeclass);
@@ -716,7 +746,7 @@ namespace snmalloc
return (void*)((uintptr_t)p + offset);
}
#else
void* apply_cache_friendly_offset(void* p, uint8_t sizeclass)
void* apply_cache_friendly_offset(void* p, sizeclass_t sizeclass)
{
UNUSED(sizeclass);
return p;
@@ -770,11 +800,11 @@ namespace snmalloc
message_queue().invariant();
#ifndef NDEBUG
for (uint8_t i = 0; i < NUM_SIZECLASSES; i++)
for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++)
{
size_t size = sizeclass_to_size(i);
uint8_t sc1 = size_to_sizeclass(size);
uint8_t sc2 = size_to_sizeclass_const(size);
sizeclass_t sc1 = size_to_sizeclass(size);
sizeclass_t sc2 = size_to_sizeclass_const(size);
size_t size1 = sizeclass_to_size(sc1);
size_t size2 = sizeclass_to_size(sc2);
@@ -794,7 +824,7 @@ namespace snmalloc
template<Boundary location>
static uintptr_t
external_pointer(void* p, uint8_t sizeclass, size_t end_point)
external_pointer(void* p, sizeclass_t sizeclass, size_t end_point)
{
size_t rsize = sizeclass_to_size(sizeclass);
size_t end_point_correction = location == End ?
@@ -808,72 +838,83 @@ namespace snmalloc
void init_message_queue()
{
message_queue().init(&stub);
// Manufacture an allocation to prime the queue
// Using an actual allocation removes a conditional of a critical path.
Remote* dummy = reinterpret_cast<Remote*>(alloc<YesZero>(MIN_ALLOC_SIZE));
dummy->set_target_id(id());
message_queue().init(dummy);
}
void handle_dealloc_remote(Remote* p)
SNMALLOC_FAST_PATH void handle_dealloc_remote(Remote* p)
{
if (p != &stub)
{
Superslab* super = Superslab::get(p);
Superslab* super = Superslab::get(p);
#ifdef CHECK_CLIENT
if (p->target_id() != super->get_allocator()->id())
error("Detected memory corruption. Potential use-after-free");
if (p->target_id() != super->get_allocator()->id())
error("Detected memory corruption. Potential use-after-free");
#endif
if (super->get_kind() == Super)
if (likely(super->get_kind() == Super))
{
Slab* slab = Slab::get(p);
Metaslab& meta = super->get_meta(slab);
if (likely(p->target_id() == id()))
{
Slab* slab = Slab::get(p);
Metaslab& meta = super->get_meta(slab);
if (p->target_id() == id())
{
small_dealloc_offseted(super, p, meta.sizeclass);
}
else
{
// Queue for remote dealloc elsewhere.
remote.dealloc(p->target_id(), p, meta.sizeclass);
}
small_dealloc_offseted(super, p, meta.sizeclass);
return;
}
}
handle_dealloc_remote_slow(p);
}
SNMALLOC_SLOW_PATH void handle_dealloc_remote_slow(Remote* p)
{
Superslab* super = Superslab::get(p);
if (likely(super->get_kind() == Medium))
{
Mediumslab* slab = Mediumslab::get(p);
if (p->target_id() == id())
{
sizeclass_t sizeclass = slab->get_sizeclass();
void* start = remove_cache_friendly_offset(p, sizeclass);
medium_dealloc(slab, start, sizeclass);
}
else
{
Mediumslab* slab = Mediumslab::get(p);
if (p->target_id() == id())
{
uint8_t sizeclass = slab->get_sizeclass();
void* start = remove_cache_friendly_offset(p, sizeclass);
medium_dealloc(slab, start, sizeclass);
}
else
{
// Queue for remote dealloc elsewhere.
remote.dealloc(p->target_id(), p, slab->get_sizeclass());
}
// Queue for remote dealloc elsewhere.
remote.dealloc(p->target_id(), p, slab->get_sizeclass());
}
}
else
{
assert(likely(p->target_id() != id()));
Slab* slab = Slab::get(p);
Metaslab& meta = super->get_meta(slab);
// Queue for remote dealloc elsewhere.
remote.dealloc(p->target_id(), p, meta.sizeclass);
}
}
NOINLINE void handle_message_queue_inner()
SNMALLOC_SLOW_PATH void handle_message_queue_inner()
{
for (size_t i = 0; i < REMOTE_BATCH; i++)
{
Remote* r = message_queue().dequeue();
auto r = message_queue().dequeue();
if (r == nullptr)
if (unlikely(!r.second))
break;
handle_dealloc_remote(r);
handle_dealloc_remote(r.first);
}
// Our remote queues may be larger due to forwarding remote frees.
if (remote.size < REMOTE_CACHE)
if (likely(remote.size < REMOTE_CACHE))
return;
stats().remote_post();
remote.post(id());
}
ALWAYSINLINE void handle_message_queue()
SNMALLOC_FAST_PATH void handle_message_queue()
{
// Inline the empty check, but not necessarily the full queue handling.
if (likely(message_queue().is_empty()))
@@ -938,7 +979,7 @@ namespace snmalloc
}
template<AllowReserve allow_reserve>
Slab* alloc_slab(uint8_t sizeclass)
Slab* alloc_slab(sizeclass_t sizeclass)
{
stats().sizeclass_alloc_slab(sizeclass);
if (Superslab::is_short_sizeclass(sizeclass))
@@ -978,7 +1019,7 @@ namespace snmalloc
}
template<ZeroMem zero_mem, AllowReserve allow_reserve>
void* small_alloc(uint8_t sizeclass, size_t rsize)
inline void* small_alloc(size_t size)
{
MEASURE_TIME_MARKERS(
small_alloc,
@@ -988,14 +1029,41 @@ namespace snmalloc
zero_mem == YesZero ? "zeromem" : "nozeromem",
allow_reserve == NoReserve ? "noreserve" : "reserve"));
SNMALLOC_ASSUME(size <= SLAB_SIZE);
sizeclass_t sizeclass = size_to_sizeclass(size);
stats().sizeclass_alloc(sizeclass);
SlabList* sc = &small_classes[sizeclass];
assert(sizeclass < NUM_SMALL_CLASSES);
auto& fl = small_fast_free_lists[sizeclass];
auto head = fl.value;
if (likely((reinterpret_cast<size_t>(head) & 1) == 0))
{
void* p = head;
// Read the next slot from the memory that's about to be allocated.
fl.value = Metaslab::follow_next(p);
if constexpr (zero_mem == YesZero)
{
large_allocator.memory_provider.zero(p, size);
}
return p;
}
return small_alloc_slow<zero_mem, allow_reserve>(sizeclass);
}
template<ZeroMem zero_mem, AllowReserve allow_reserve>
SNMALLOC_SLOW_PATH void* small_alloc_slow(sizeclass_t sizeclass)
{
handle_message_queue();
size_t rsize = sizeclass_to_size(sizeclass);
auto& sl = small_classes[sizeclass];
Slab* slab;
if (!sc->is_empty())
if (!sl.is_empty())
{
SlabLink* link = sc->get_head();
SlabLink* link = sl.get_head();
slab = link->get_slab();
}
else
@@ -1005,13 +1073,15 @@ namespace snmalloc
if ((allow_reserve == NoReserve) && (slab == nullptr))
return nullptr;
sc->insert(slab->get_link());
sl.insert(slab->get_link());
}
return slab->alloc<zero_mem>(sc, rsize, large_allocator.memory_provider);
auto& ffl = small_fast_free_lists[sizeclass];
return slab->alloc<zero_mem>(
sl, ffl, rsize, large_allocator.memory_provider);
}
void small_dealloc(Superslab* super, void* p, uint8_t sizeclass)
SNMALLOC_FAST_PATH void
small_dealloc(Superslab* super, void* p, sizeclass_t sizeclass)
{
#ifdef CHECK_CLIENT
Slab* slab = Slab::get(p);
@@ -1025,19 +1095,29 @@ namespace snmalloc
small_dealloc_offseted(super, offseted, sizeclass);
}
void small_dealloc_offseted(Superslab* super, void* p, uint8_t sizeclass)
SNMALLOC_FAST_PATH void
small_dealloc_offseted(Superslab* super, void* p, sizeclass_t sizeclass)
{
MEASURE_TIME(small_dealloc, 4, 16);
stats().sizeclass_dealloc(sizeclass);
bool was_full = super->is_full();
SlabList* sc = &small_classes[sizeclass];
Slab* slab = Slab::get(p);
Superslab::Action a =
slab->dealloc(sc, super, p, large_allocator.memory_provider);
if (a == Superslab::NoSlabReturn)
if (likely(slab->dealloc_fast(super, p)))
return;
small_dealloc_offseted_slow(super, p, sizeclass);
}
SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow(
Superslab* super, void* p, sizeclass_t sizeclass)
{
bool was_full = super->is_full();
SlabList* sl = &small_classes[sizeclass];
Slab* slab = Slab::get(p);
Superslab::Action a =
slab->dealloc_slow(sl, super, p, large_allocator.memory_provider);
if (likely(a == Superslab::NoSlabReturn))
return;
stats().sizeclass_dealloc_slab(sizeclass);
if (a == Superslab::NoStatusChange)
@@ -1100,7 +1180,7 @@ namespace snmalloc
}
template<ZeroMem zero_mem, AllowReserve allow_reserve>
void* medium_alloc(uint8_t sizeclass, size_t rsize, size_t size)
void* medium_alloc(sizeclass_t sizeclass, size_t rsize, size_t size)
{
MEASURE_TIME_MARKERS(
medium_alloc,
@@ -1110,7 +1190,7 @@ namespace snmalloc
zero_mem == YesZero ? "zeromem" : "nozeromem",
allow_reserve == NoReserve ? "noreserve" : "reserve"));
uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES;
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
DLList<Mediumslab>* sc = &medium_classes[medium_class];
Mediumslab* slab = sc->get_head();
@@ -1144,7 +1224,7 @@ namespace snmalloc
return p;
}
void medium_dealloc(Mediumslab* slab, void* p, uint8_t sizeclass)
void medium_dealloc(Mediumslab* slab, void* p, sizeclass_t sizeclass)
{
MEASURE_TIME(medium_dealloc, 4, 16);
stats().sizeclass_dealloc(sizeclass);
@@ -1163,7 +1243,7 @@ namespace snmalloc
{
if (!was_full)
{
uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES;
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
DLList<Mediumslab>* sc = &medium_classes[medium_class];
sc->remove(slab);
}
@@ -1180,7 +1260,7 @@ namespace snmalloc
}
else if (was_full)
{
uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES;
sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES;
DLList<Mediumslab>* sc = &medium_classes[medium_class];
sc->insert(slab);
}
@@ -1233,10 +1313,13 @@ namespace snmalloc
large_allocator.dealloc(slab, large_class);
}
void remote_dealloc(RemoteAllocator* target, void* p, uint8_t sizeclass)
SNMALLOC_FAST_PATH void
remote_dealloc(RemoteAllocator* target, void* p, sizeclass_t sizeclass)
{
MEASURE_TIME(remote_dealloc, 4, 16);
handle_message_queue();
void* offseted = apply_cache_friendly_offset(p, sizeclass);
stats().remote_free(sizeclass);

15
src/mem/allocstats.h
View File

@@ -1,6 +1,7 @@
#pragma once
#include "../ds/bits.h"
#include "../mem/sizeclass.h"
#include <cstdint>
@@ -165,7 +166,7 @@ namespace snmalloc
#endif
}
void sizeclass_alloc(uint8_t sc)
void sizeclass_alloc(sizeclass_t sc)
{
UNUSED(sc);
@@ -175,7 +176,7 @@ namespace snmalloc
#endif
}
void sizeclass_dealloc(uint8_t sc)
void sizeclass_dealloc(sizeclass_t sc)
{
UNUSED(sc);
@@ -194,7 +195,7 @@ namespace snmalloc
#endif
}
void sizeclass_alloc_slab(uint8_t sc)
void sizeclass_alloc_slab(sizeclass_t sc)
{
UNUSED(sc);
@@ -204,7 +205,7 @@ namespace snmalloc
#endif
}
void sizeclass_dealloc_slab(uint8_t sc)
void sizeclass_dealloc_slab(sizeclass_t sc)
{
UNUSED(sc);
@@ -251,7 +252,7 @@ namespace snmalloc
#endif
}
void remote_free(uint8_t sc)
void remote_free(sizeclass_t sc)
{
UNUSED(sc);
@@ -267,7 +268,7 @@ namespace snmalloc
#endif
}
void remote_receive(uint8_t sc)
void remote_receive(sizeclass_t sc)
{
UNUSED(sc);
@@ -348,7 +349,7 @@ namespace snmalloc
<< "Count" << csv.endl;
}
for (uint8_t i = 0; i < N; i++)
for (sizeclass_t i = 0; i < N; i++)
{
if (sizeclass[i].count.is_unused())
continue;

4
src/mem/largealloc.h
View File

@@ -189,7 +189,7 @@ namespace snmalloc
* (over the lifetime of this process). If the underlying system does not
* support low memory notifications, this will return 0.
*/
ALWAYSINLINE
SNMALLOC_FAST_PATH
uint64_t low_memory_epoch()
{
if constexpr (pal_supports<LowMemoryNotification>())
@@ -235,7 +235,7 @@ namespace snmalloc
}
}
ALWAYSINLINE void lazy_decommit_if_needed()
SNMALLOC_FAST_PATH void lazy_decommit_if_needed()
{
#ifdef TEST_LAZY_DECOMMIT
static_assert(

4
src/mem/mediumslab.h
View File

@@ -44,7 +44,7 @@ namespace snmalloc
return pointer_cast<Mediumslab>(address_cast(p) & SUPERSLAB_MASK);
}
void init(RemoteAllocator* alloc, uint8_t sc, size_t rsize)
void init(RemoteAllocator* alloc, sizeclass_t sc, size_t rsize)
{
assert(sc >= NUM_SMALL_CLASSES);
assert((sc - NUM_SMALL_CLASSES) < NUM_MEDIUM_CLASSES);
@@ -56,7 +56,7 @@ namespace snmalloc
// initialise the allocation stack.
if ((kind != Medium) || (sizeclass != sc))
{
sizeclass = sc;
sizeclass = static_cast<uint8_t>(sc);
uint16_t ssize = static_cast<uint16_t>(rsize >> 8);
kind = Medium;
free = medium_slab_free(sc);

106
src/mem/metaslab.h
View File

@@ -29,22 +29,24 @@ namespace snmalloc
// This can be either a short or a standard slab.
class Metaslab
{
private:
// How many entries are used in this slab.
public:
// How many entries are not in the free list of slab.
uint16_t used = 0;
public:
// Bump free list of unused entries in this sizeclass.
// If the bottom bit is 1, then this represents a bump_ptr
// of where we have allocated up to in this slab. Otherwise,
// it represents the location of the first block in the free
// list. The free list is chained through deallocated blocks.
// It is terminated with a bump ptr.
//
// Note that, the first entry in a slab is never bump allocated
// but is used for the link. This means that 1 represents the fully
// bump allocated slab.
// How many entries have been allocated from this slab.
uint16_t allocated;
// Index of first entry in this slab that forms the free
// list. The list entries are stored as the first pointer
// in each unused object. The terminator is a pointer or
// offset into the block with the bottom bit set. This means
// I.e.
// * an empty list has a head of 1.
// * a one element list has an head contains an offset to this
// this block, and then contains a pointer with the bottom
// bit set.
Mod<SLAB_SIZE, uint16_t> head;
// When a slab has free space it will be on the has space list for
// that size class. We use an empty block in this slab to be the
// doubly linked node into that size class's free list.
@@ -93,35 +95,37 @@ namespace snmalloc
/// Value used to check for corruptions in a block
static constexpr size_t POISON =
static_cast<size_t>(bits::is64() ? 0xDEADBEEFDEAD0000 : 0xDEAD0000);
static_cast<size_t>(bits::is64() ? 0xDEADBEEFDEADBEEF : 0xDEADBEEF);
/// Store next pointer in a block. In Debug using magic value to detect some
/// simple corruptions.
static void store_next(void* p, uint16_t head)
static SNMALLOC_FAST_PATH void store_next(void* p, void* head)
{
#ifndef CHECK_CLIENT
*static_cast<size_t*>(p) = head;
*static_cast<void**>(p) = head;
#else
*static_cast<size_t*>(p) =
head ^ POISON ^ (static_cast<size_t>(head) << (bits::BITS - 16));
*static_cast<void**>(p) = head;
*(static_cast<uintptr_t*>(p) + 1) = address_cast(head) ^ POISON;
#endif
}
/// Accessor function for the next pointer in a block.
/// In Debug checks for simple corruptions.
static uint16_t follow_next(void* node)
static SNMALLOC_FAST_PATH void* follow_next(void* node)
{
size_t next = *static_cast<size_t*>(node);
#ifdef CHECK_CLIENT
if (((next ^ POISON) ^ (next << (bits::BITS - 16))) > 0xFFFF)
uintptr_t next = *static_cast<uintptr_t*>(node);
uintptr_t chk = *(static_cast<uintptr_t*>(node) + 1);
if ((next ^ chk) != POISON)
error("Detected memory corruption. Use-after-free.");
#endif
return static_cast<uint16_t>(next);
return *static_cast<void**>(node);
}
bool valid_head(bool is_short)
{
size_t size = sizeclass_to_size(sizeclass);
size_t slab_start = get_initial_link(sizeclass, is_short);
size_t slab_start = get_initial_offset(sizeclass, is_short);
size_t all_high_bits = ~static_cast<size_t>(1);
size_t head_start =
@@ -138,18 +142,19 @@ namespace snmalloc
* We don't expect a cycle, so worst case is only followed by a crash, so
* slow doesn't mater.
**/
void debug_slab_acyclic_free_list(Slab* slab)
size_t debug_slab_acyclic_free_list(Slab* slab)
{
#ifndef NDEBUG
uint16_t curr = head;
uint16_t curr_slow = head;
size_t length = 0;
void* curr = pointer_offset(slab, head);
void* curr_slow = pointer_offset(slab, head);
bool both = false;
while ((curr & 1) != 1)
while ((reinterpret_cast<size_t>(curr) & 1) == 0)
{
curr = follow_next(pointer_offset(slab, curr));
curr = follow_next(curr);
if (both)
{
curr_slow = follow_next(pointer_offset(slab, curr_slow));
curr_slow = follow_next(curr_slow);
}
if (curr == curr_slow)
@@ -158,9 +163,12 @@ namespace snmalloc
}
both = !both;
length++;
}
return length;
#else
UNUSED(slab);
return 0;
#endif
}
@@ -168,7 +176,10 @@ namespace snmalloc
{
#if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS)
size_t size = sizeclass_to_size(sizeclass);
size_t offset = get_initial_link(sizeclass, is_short);
size_t offset = get_initial_offset(sizeclass, is_short);
if (is_unused())
return;
size_t accounted_for = used * size + offset;
@@ -184,38 +195,41 @@ namespace snmalloc
// Block is not full
assert(SLAB_SIZE > accounted_for);
debug_slab_acyclic_free_list(slab);
// Keep variable so it appears in debugger.
size_t length = debug_slab_acyclic_free_list(slab);
UNUSED(length);
// Walk bump-free-list-segment accounting for unused space
uint16_t curr = head;
while ((curr & 1) != 1)
void* curr = pointer_offset(slab, head);
while ((address_cast(curr) & 1) == 0)
{
// Check we are looking at a correctly aligned block
uint16_t start = remove_cache_friendly_offset(curr, sizeclass);
assert((start - offset) % size == 0);
void* start = curr;
assert(
((address_cast(start) - address_cast(slab) - offset) % size) == 0);
// Account for free elements in free list
accounted_for += size;
assert(SLAB_SIZE >= accounted_for);
// We should never reach the link node in the free list.
assert(curr != link);
assert(curr != pointer_offset(slab, link));
// Iterate bump/free list segment
curr = follow_next(pointer_offset(slab, curr));
curr = follow_next(curr);
}
if (curr != 1)
auto bumpptr = (allocated * size) + offset;
// Check we haven't allocaated more than gits in a slab
assert(bumpptr <= SLAB_SIZE);
// Account for to be bump allocated space
accounted_for += SLAB_SIZE - bumpptr;
if (bumpptr != SLAB_SIZE)
{
// Check we terminated traversal on a correctly aligned block
uint16_t start = remove_cache_friendly_offset(curr & ~1, sizeclass);
assert((start - offset) % size == 0);
// Account for to be bump allocated space
accounted_for += SLAB_SIZE - (curr - 1);
// The link should be the first allocation as we
// haven't completely filled this block at any point.
assert(link == get_initial_link(sizeclass, is_short));
assert(link == get_initial_offset(sizeclass, is_short));
}
assert(!is_full());

71
src/mem/pagemap.h
View File

@@ -105,7 +105,33 @@ namespace snmalloc
std::atomic<PagemapEntry*> top[TOPLEVEL_ENTRIES]; // = {nullptr};
template<bool create_addr>
inline PagemapEntry* get_node(std::atomic<PagemapEntry*>* e, bool& result)
SNMALLOC_FAST_PATH PagemapEntry*
get_node(std::atomic<PagemapEntry*>* e, bool& result)
{
// The page map nodes are all allocated directly from the OS zero
// initialised with a system call. We don't need any ordered to guarantee
// to see that correctly. The only transistions are monotone and handled
// by the slow path.
PagemapEntry* value = e->load(std::memory_order_relaxed);
if (likely(value > LOCKED_ENTRY))
{
result = true;
return value;
}
if constexpr (create_addr)
{
return get_node_slow(e, result);
}
else
{
result = false;
return nullptr;
}
}
SNMALLOC_SLOW_PATH PagemapEntry*
get_node_slow(std::atomic<PagemapEntry*>* e, bool& result)
{
// The page map nodes are all allocated directly from the OS zero
// initialised with a system call. We don't need any ordered to guarantee
@@ -114,31 +140,23 @@ namespace snmalloc
if ((value == nullptr) || (value == LOCKED_ENTRY))
{
if constexpr (create_addr)
{
value = nullptr;
value = nullptr;
if (e->compare_exchange_strong(
value, LOCKED_ENTRY, std::memory_order_relaxed))
{
auto& v = default_memory_provider;
value = v.alloc_chunk<PagemapEntry, OS_PAGE_SIZE>();
e->store(value, std::memory_order_release);
}
else
{
while (address_cast(e->load(std::memory_order_relaxed)) ==
LOCKED_ENTRY)
{
bits::pause();
}
value = e->load(std::memory_order_acquire);
}
if (e->compare_exchange_strong(
value, LOCKED_ENTRY, std::memory_order_relaxed))
{
auto& v = default_memory_provider;
value = v.alloc_chunk<PagemapEntry, OS_PAGE_SIZE>();
e->store(value, std::memory_order_release);
}
else
{
result = false;
return nullptr;
while (address_cast(e->load(std::memory_order_relaxed)) ==
LOCKED_ENTRY)
{
bits::pause();
}
value = e->load(std::memory_order_acquire);
}
}
result = true;
@@ -146,7 +164,8 @@ namespace snmalloc
}
template<bool create_addr>
inline std::pair<Leaf*, size_t> get_leaf_index(uintptr_t addr, bool& result)
SNMALLOC_FAST_PATH std::pair<Leaf*, size_t>
get_leaf_index(uintptr_t addr, bool& result)
{
#ifdef FreeBSD_KERNEL
// Zero the top 16 bits - kernel addresses all have them set, but the
@@ -160,7 +179,7 @@ namespace snmalloc
for (size_t i = 0; i < INDEX_LEVELS; i++)
{
PagemapEntry* value = get_node<create_addr>(e, result);
if (!result)
if (unlikely(!result))
return std::pair(nullptr, 0);
shift -= BITS_PER_INDEX_LEVEL;
@@ -180,7 +199,7 @@ namespace snmalloc
Leaf* leaf = reinterpret_cast<Leaf*>(get_node<create_addr>(e, result));
if (!result)
if (unlikely(!result))
return std::pair(nullptr, 0);
shift -= BITS_FOR_LEAF;
@@ -189,7 +208,7 @@ namespace snmalloc
}
template<bool create_addr>
inline std::atomic<T>* get_addr(uintptr_t p, bool& success)
SNMALLOC_FAST_PATH std::atomic<T>* get_addr(uintptr_t p, bool& success)
{
auto leaf_ix = get_leaf_index<create_addr>(p, success);
return &(leaf_ix.first->values[leaf_ix.second]);

51
src/mem/sizeclass.h
View File

@@ -4,29 +4,31 @@
namespace snmalloc
{
constexpr static uint16_t get_initial_bumpptr(uint8_t sc, bool is_short);
constexpr static uint16_t get_initial_link(uint8_t sc, bool is_short);
constexpr static size_t sizeclass_to_size(uint8_t sizeclass);
constexpr static size_t sizeclass_to_cache_friendly_mask(uint8_t sizeclass);
constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(uint8_t sc);
constexpr static uint16_t medium_slab_free(uint8_t sizeclass);
// Both usings should compile
// We use size_t as it generates better code.
using sizeclass_t = size_t;
// using sizeclass_t = uint8_t;
static inline uint8_t size_to_sizeclass(size_t size)
constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short);
constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass);
constexpr static size_t
sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass);
constexpr static size_t
sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc);
constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass);
static sizeclass_t size_to_sizeclass(size_t size);
constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size)
{
// Don't use sizeclasses that are not a multiple of the alignment.
// For example, 24 byte allocations can be
// problematic for some data due to alignment issues.
return static_cast<uint8_t>(
bits::to_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(size));
}
constexpr static inline uint8_t size_to_sizeclass_const(size_t size)
{
// Don't use sizeclasses that are not a multiple of the alignment.
// For example, 24 byte allocations can be
// problematic for some data due to alignment issues.
return static_cast<uint8_t>(
auto sc = static_cast<sizeclass_t>(
bits::to_exp_mant_const<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(size));
assert(sc == static_cast<uint8_t>(sc));
return sc;
}
constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class)
@@ -143,26 +145,29 @@ namespace snmalloc
}
#ifdef CACHE_FRIENDLY_OFFSET
inline static void* remove_cache_friendly_offset(void* p, uint8_t sizeclass)
SNMALLOC_FAST_PATH static void*
remove_cache_friendly_offset(void* p, sizeclass_t sizeclass)
{
size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass);
return p = (void*)((uintptr_t)p & mask);
}
inline static uint16_t
remove_cache_friendly_offset(uint16_t relative, uint8_t sizeclass)
SNMALLOC_FAST_PATH static uint16_t
remove_cache_friendly_offset(uint16_t relative, sizeclass_t sizeclass)
{
size_t mask = sizeclass_to_inverse_cache_friendly_mask(sizeclass);
return relative & mask;
}
#else
inline static void* remove_cache_friendly_offset(void* p, uint8_t sizeclass)
SNMALLOC_FAST_PATH static void*
remove_cache_friendly_offset(void* p, sizeclass_t sizeclass)
{
UNUSED(sizeclass);
return p;
}
inline static uint16_t
remove_cache_friendly_offset(uint16_t relative, uint8_t sizeclass)
SNMALLOC_FAST_PATH static uint16_t
remove_cache_friendly_offset(uint16_t relative, sizeclass_t sizeclass)
{
UNUSED(sizeclass);
return relative;

97
src/mem/sizeclasstable.h
View File

@@ -5,31 +5,52 @@
namespace snmalloc
{
constexpr size_t PTR_BITS = bits::next_pow2_bits_const(sizeof(void*));
constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s)
{
// We subtract and shirt to reduce the size of the table, i.e. we don't have
// to store a value for every size class.
// We could shift by MIN_ALLOC_BITS, as this would give us the most
// compressed table, but by shifting by PTR_BITS the code-gen is better
// as the most important path using this subsequently shifts left by
// PTR_BITS, hence they can be fused into a single mask.
return (s - 1) >> PTR_BITS;
}
constexpr static size_t sizeclass_lookup_size =
sizeclass_lookup_index(SLAB_SIZE + 1);
struct SizeClassTable
{
sizeclass_t sizeclass_lookup[sizeclass_lookup_size] = {{}};
ModArray<NUM_SIZECLASSES, size_t> size;
ModArray<NUM_SIZECLASSES, size_t> cache_friendly_mask;
ModArray<NUM_SIZECLASSES, size_t> inverse_cache_friendly_mask;
ModArray<NUM_SMALL_CLASSES, uint16_t> bump_ptr_start;
ModArray<NUM_SMALL_CLASSES, uint16_t> short_bump_ptr_start;
ModArray<NUM_SMALL_CLASSES, uint16_t> initial_link_ptr;
ModArray<NUM_SMALL_CLASSES, uint16_t> short_initial_link_ptr;
ModArray<NUM_SMALL_CLASSES, uint16_t> initial_offset_ptr;
ModArray<NUM_SMALL_CLASSES, uint16_t> short_initial_offset_ptr;
ModArray<NUM_MEDIUM_CLASSES, uint16_t> medium_slab_slots;
constexpr SizeClassTable()
: size(),
cache_friendly_mask(),
inverse_cache_friendly_mask(),
bump_ptr_start(),
short_bump_ptr_start(),
initial_link_ptr(),
short_initial_link_ptr(),
initial_offset_ptr(),
short_initial_offset_ptr(),
medium_slab_slots()
{
for (uint8_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++)
size_t curr = 1;
for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++)
{
size[sizeclass] =
bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(sizeclass);
if (sizeclass < NUM_SMALL_CLASSES)
{
for (; curr <= size[sizeclass]; curr += 1 << PTR_BITS)
{
sizeclass_lookup[sizeclass_lookup_index(curr)] = sizeclass;
}
}
size_t alignment = bits::min(
bits::one_at_bit(bits::ctz_const(size[sizeclass])), OS_PAGE_SIZE);
@@ -40,7 +61,7 @@ namespace snmalloc
size_t header_size = sizeof(Superslab);
size_t short_slab_size = SLAB_SIZE - header_size;
for (uint8_t i = 0; i < NUM_SMALL_CLASSES; i++)
for (sizeclass_t i = 0; i < NUM_SMALL_CLASSES; i++)
{
// We align to the end of the block to remove special cases for the
// short block. Calculate remainders
@@ -48,22 +69,12 @@ namespace snmalloc
size_t correction = SLAB_SIZE % size[i];
// First element in the block is the link
initial_link_ptr[i] = static_cast<uint16_t>(correction);
short_initial_link_ptr[i] =
initial_offset_ptr[i] = static_cast<uint16_t>(correction);
short_initial_offset_ptr[i] =
static_cast<uint16_t>(header_size + short_correction);
// Move to object after link.
auto short_after_link = short_initial_link_ptr[i] + size[i];
size_t after_link = initial_link_ptr[i] + size[i];
// Bump ptr has bottom bit set.
// In case we only have one object on this slab check for wrap around.
short_bump_ptr_start[i] =
static_cast<uint16_t>((short_after_link + 1) % SLAB_SIZE);
bump_ptr_start[i] = static_cast<uint16_t>((after_link + 1) % SLAB_SIZE);
}
for (uint8_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++)
for (sizeclass_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++)
{
medium_slab_slots[i - NUM_SMALL_CLASSES] = static_cast<uint16_t>(
(SUPERSLAB_SIZE - Mediumslab::header_size()) / size[i]);
@@ -74,40 +85,48 @@ namespace snmalloc
static constexpr SizeClassTable sizeclass_metadata = SizeClassTable();
static inline constexpr uint16_t
get_initial_bumpptr(uint8_t sc, bool is_short)
get_initial_offset(sizeclass_t sc, bool is_short)
{
if (is_short)
return sizeclass_metadata.short_bump_ptr_start[sc];
return sizeclass_metadata.short_initial_offset_ptr[sc];
return sizeclass_metadata.bump_ptr_start[sc];
return sizeclass_metadata.initial_offset_ptr[sc];
}
static inline constexpr uint16_t get_initial_link(uint8_t sc, bool is_short)
{
if (is_short)
return sizeclass_metadata.short_initial_link_ptr[sc];
return sizeclass_metadata.initial_link_ptr[sc];
}
constexpr static inline size_t sizeclass_to_size(uint8_t sizeclass)
constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass)
{
return sizeclass_metadata.size[sizeclass];
}
constexpr static inline size_t
sizeclass_to_cache_friendly_mask(uint8_t sizeclass)
sizeclass_to_cache_friendly_mask(sizeclass_t sizeclass)
{
return sizeclass_metadata.cache_friendly_mask[sizeclass];
}
constexpr static inline size_t
sizeclass_to_inverse_cache_friendly_mask(uint8_t sizeclass)
constexpr static SNMALLOC_FAST_PATH size_t
sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sizeclass)
{
return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass];
}
constexpr static inline uint16_t medium_slab_free(uint8_t sizeclass)
static inline sizeclass_t size_to_sizeclass(size_t size)
{
if ((size - 1) <= (SLAB_SIZE - 1))
{
auto index = sizeclass_lookup_index(size);
SNMALLOC_ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE));
return sizeclass_metadata.sizeclass_lookup[index];
}
// Don't use sizeclasses that are not a multiple of the alignment.
// For example, 24 byte allocations can be
// problematic for some data due to alignment issues.
return static_cast<sizeclass_t>(
bits::to_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(size));
}
constexpr static inline uint16_t medium_slab_free(sizeclass_t sizeclass)
{
return sizeclass_metadata
.medium_slab_slots[(sizeclass - NUM_SMALL_CLASSES)];

190
src/mem/slab.h
View File

@@ -4,6 +4,12 @@
namespace snmalloc
{
struct FreeListHead
{
// Use a value with bottom bit set for empty list.
void* value = pointer_offset<void*>(nullptr, 1);
};
class Slab
{
private:
@@ -31,7 +37,11 @@ namespace snmalloc
}
template<ZeroMem zero_mem, typename MemoryProvider>
void* alloc(SlabList* sc, size_t rsize, MemoryProvider& memory_provider)
inline void* alloc(
SlabList& sl,
FreeListHead& fast_free_list,
size_t rsize,
MemoryProvider& memory_provider)
{
// Read the head from the metadata stored in the superslab.
Metaslab& meta = get_meta();
@@ -39,38 +49,83 @@ namespace snmalloc
assert(rsize == sizeclass_to_size(meta.sizeclass));
meta.debug_slab_invariant(is_short(), this);
assert(sc->get_head() == (SlabLink*)((size_t)this + meta.link));
assert(sl.get_head() == (SlabLink*)((size_t)this + meta.link));
assert(!meta.is_full());
meta.add_use();
void* p = nullptr;
bool p_has_value = false;
void* p;
if ((head & 1) == 0)
if (head == 1)
{
void* node = pointer_offset(this, head);
// Read the next slot from the memory that's about to be allocated.
meta.head = Metaslab::follow_next(node);
p = remove_cache_friendly_offset(node, meta.sizeclass);
}
else
{
if (meta.head == 1)
size_t bumpptr = get_initial_offset(meta.sizeclass, is_short());
bumpptr += meta.allocated * rsize;
if (bumpptr == SLAB_SIZE)
{
meta.add_use();
assert(meta.used == meta.allocated);
p = pointer_offset(this, meta.link);
sc->pop();
meta.set_full();
sl.pop();
p_has_value = true;
}
else
{
// This slab is being bump allocated.
p = pointer_offset(this, head - 1);
meta.head = (head + static_cast<uint16_t>(rsize)) & (SLAB_SIZE - 1);
void* curr = nullptr;
bool commit = false;
while (true)
{
size_t newbumpptr = bumpptr + rsize;
auto alignedbumpptr = bits::align_up(bumpptr - 1, OS_PAGE_SIZE);
auto alignednewbumpptr = bits::align_up(newbumpptr, OS_PAGE_SIZE);
if (alignedbumpptr != alignednewbumpptr)
{
// We have committed once already.
if (commit)
break;
memory_provider.template notify_using<NoZero>(
pointer_offset(this, alignedbumpptr),
alignednewbumpptr - alignedbumpptr);
commit = true;
}
if (curr == nullptr)
{
meta.head = static_cast<uint16_t>(bumpptr);
}
else
{
Metaslab::store_next(curr, pointer_offset(this, bumpptr));
}
curr = pointer_offset(this, bumpptr);
bumpptr = newbumpptr;
meta.allocated = meta.allocated + 1;
}
assert(curr != nullptr);
Metaslab::store_next(curr, pointer_offset<void*>(nullptr, 1));
}
}
if (!p_has_value)
{
p = pointer_offset(this, meta.head);
// Read the next slot from the memory that's about to be allocated.
void* next = Metaslab::follow_next(p);
// Put everything in allocators small_class free list.
meta.head = 1;
fast_free_list.value = next;
// Treat stealing the free list as allocating it all.
// Link is not in use, i.e. - 1 is required.
meta.used = meta.allocated - 1;
}
assert(is_start_of_object(Superslab::get(p), p));
meta.debug_slab_invariant(is_short(), this);
if constexpr (zero_mem == YesZero)
@@ -92,38 +147,58 @@ namespace snmalloc
address_cast(this) + SLAB_SIZE - address_cast(p));
}
// Returns true, if it alters get_status.
template<typename MemoryProvider>
inline typename Superslab::Action dealloc(
SlabList* sc, Superslab* super, void* p, MemoryProvider& memory_provider)
// Returns true, if it deallocation can proceed without changing any status
// bits. Note that this does remove the use from the meta slab, so it
// doesn't need doing on the slow path.
SNMALLOC_FAST_PATH bool dealloc_fast(Superslab* super, void* p)
{
Metaslab& meta = super->get_meta(this);
bool was_full = meta.is_full();
#ifdef CHECK_CLIENT
if (meta.is_unused())
error("Detected potential double free.");
#endif
meta.debug_slab_invariant(is_short(), this);
meta.sub_use();
bool was_full = meta.is_full();
if (unlikely(was_full))
return false;
bool is_unused = meta.is_unused();
if (unlikely(is_unused))
return false;
// Update the head and the next pointer in the free list.
uint16_t head = meta.head;
uint16_t current = pointer_to_index(p);
// Set the head to the memory being deallocated.
meta.head = current;
assert(meta.valid_head(is_short()));
// Set the next pointer to the previous head.
Metaslab::store_next(p, pointer_offset(this, head));
meta.debug_slab_invariant(is_short(), this);
return true;
}
// If dealloc fast returns false, then call this.
// This does not need to remove the "use" as done by the fast path.
// Returns a complex return code for managing the superslab meta data.
// i.e. This deallocation could make an entire superslab free.
template<typename MemoryProvider>
SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow(
SlabList* sl, Superslab* super, void* p, MemoryProvider& memory_provider)
{
Metaslab& meta = super->get_meta(this);
bool was_full = meta.is_full();
bool is_unused = meta.is_unused();
if (was_full)
{
// We are not on the sizeclass list.
if (!meta.is_unused())
{
// Update the head and the sizeclass link.
uint16_t index = pointer_to_index(p);
assert(meta.head == 1);
meta.link = index;
// Push on the list of slabs for this sizeclass.
sc->insert(meta.get_link(this));
meta.debug_slab_invariant(is_short(), this);
}
else
if (is_unused)
{
// Dealloc on the superslab.
if (is_short())
@@ -131,37 +206,30 @@ namespace snmalloc
return super->dealloc_slab(this, memory_provider);
}
// Update the head and the sizeclass link.
uint16_t index = pointer_to_index(p);
assert(meta.head == 1);
// assert(meta.fully_allocated(is_short()));
meta.link = index;
// Push on the list of slabs for this sizeclass.
sl->insert(meta.get_link(this));
meta.debug_slab_invariant(is_short(), this);
return Superslab::NoSlabReturn;
}
else if (meta.is_unused())
if (is_unused)
{
// Remove from the sizeclass list and dealloc on the superslab.
sc->remove(meta.get_link(this));
sl->remove(meta.get_link(this));
if (is_short())
return super->dealloc_short_slab(memory_provider);
return super->dealloc_slab(this, memory_provider);
}
else
{
#ifndef NDEBUG
sc->debug_check_contains(meta.get_link(this));
#endif
// Update the head and the next pointer in the free list.
uint16_t head = meta.head;
uint16_t current = pointer_to_index(p);
// Set the head to the memory being deallocated.
meta.head = current;
assert(meta.valid_head(is_short()));
// Set the next pointer to the previous head.
Metaslab::store_next(p, head);
meta.debug_slab_invariant(is_short(), this);
}
return Superslab::NoSlabReturn;
abort();
}
bool is_short()

24
src/mem/superslab.h
View File

@@ -70,9 +70,9 @@ namespace snmalloc
return pointer_cast<Superslab>(address_cast(p) & SUPERSLAB_MASK);
}
static bool is_short_sizeclass(uint8_t sizeclass)
static bool is_short_sizeclass(sizeclass_t sizeclass)
{
constexpr uint8_t h = size_to_sizeclass_const(sizeof(Superslab));
constexpr sizeclass_t h = size_to_sizeclass_const(sizeof(Superslab));
return sizeclass <= h;
}
@@ -154,16 +154,17 @@ namespace snmalloc
}
template<typename MemoryProvider>
Slab* alloc_short_slab(uint8_t sizeclass, MemoryProvider& memory_provider)
Slab*
alloc_short_slab(sizeclass_t sizeclass, MemoryProvider& memory_provider)
{
if ((used & 1) == 1)
return alloc_slab(sizeclass, memory_provider);
meta[0].head = get_initial_bumpptr(sizeclass, true);
meta[0].sizeclass = sizeclass;
meta[0].link = get_initial_link(sizeclass, true);
meta[0].allocated = 1;
meta[0].head = 1;
meta[0].sizeclass = static_cast<uint8_t>(sizeclass);
meta[0].link = get_initial_offset(sizeclass, true);
if constexpr (decommit_strategy == DecommitAll)
{
memory_provider.template notify_using<NoZero>(
pointer_offset(this, OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE);
@@ -174,7 +175,7 @@ namespace snmalloc
}
template<typename MemoryProvider>
Slab* alloc_slab(uint8_t sizeclass, MemoryProvider& memory_provider)
Slab* alloc_slab(sizeclass_t sizeclass, MemoryProvider& memory_provider)
{
uint8_t h = head;
Slab* slab = pointer_cast<Slab>(
@@ -182,9 +183,10 @@ namespace snmalloc
uint8_t n = meta[h].next;
meta[h].head = get_initial_bumpptr(sizeclass, false);
meta[h].sizeclass = sizeclass;
meta[h].link = get_initial_link(sizeclass, false);
meta[h].head = 1;
meta[h].allocated = 1;
meta[h].sizeclass = static_cast<uint8_t>(sizeclass);
meta[h].link = get_initial_offset(sizeclass, false);
head = h + n + 1;
used += 2;

12
src/mem/threadalloc.h
View File

@@ -28,7 +28,7 @@ namespace snmalloc
class ThreadAllocUntypedWrapper
{
public:
static inline Alloc*& get()
static SNMALLOC_FAST_PATH Alloc*& get()
{
return (Alloc*&)ThreadAllocUntyped::get();
}
@@ -76,7 +76,7 @@ namespace snmalloc
* The non-create case exists so that the `per_thread` variable can be a
* local static and not a global, allowing ODR to deduplicate it.
*/
static inline Alloc*& get(bool create = true)
static SNMALLOC_FAST_PATH Alloc*& get(bool create = true)
{
static thread_local Alloc* per_thread;
if (!per_thread && create)
@@ -224,7 +224,7 @@ namespace snmalloc
* Private accessor to the per thread allocator
* Provides no checking for initialization
*/
static ALWAYSINLINE Alloc*& inner_get()
static SNMALLOC_FAST_PATH Alloc*& inner_get()
{
static thread_local Alloc* per_thread;
return per_thread;
@@ -242,7 +242,7 @@ namespace snmalloc
/**
* Private initialiser for the per thread allocator
*/
static NOINLINE Alloc*& inner_init()
static SNMALLOC_SLOW_PATH Alloc*& inner_init()
{
Alloc*& per_thread = inner_get();
@@ -280,11 +280,11 @@ namespace snmalloc
* Public interface, returns the allocator for the current thread,
* constructing it if necessary.
*/
static ALWAYSINLINE Alloc*& get()
static SNMALLOC_FAST_PATH Alloc*& get()
{
Alloc*& per_thread = inner_get();
if (per_thread != nullptr)
if (likely(per_thread != nullptr))
return per_thread;
// Slow path that performs initialization

10
src/override/malloc.cc
View File

@@ -23,17 +23,11 @@ extern "C"
SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size)
{
// Include size 0 in the first sizeclass.
size = ((size - 1) >> (bits::BITS - 1)) + size;
return ThreadAlloc::get()->alloc(size);
}
SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free)(void* ptr)
{
if (ptr == nullptr)
return;
ThreadAlloc::get()->dealloc(ptr);
}
@@ -46,8 +40,6 @@ extern "C"
errno = ENOMEM;
return nullptr;
}
// Include size 0 in the first sizeclass.
sz = ((sz - 1) >> (bits::BITS - 1)) + sz;
return ThreadAlloc::get()->alloc<ZeroMem::YesZero>(sz);
}
@@ -137,7 +129,7 @@ extern "C"
}
size = bits::max(size, alignment);
uint8_t sc = size_to_sizeclass(size);
snmalloc::sizeclass_t sc = size_to_sizeclass(size);
if (sc >= NUM_SIZECLASSES)
{
// large allocs are 16M aligned.

7
src/pal/pal_freebsd.h
View File

@@ -38,7 +38,14 @@ namespace snmalloc
assert(
bits::is_aligned_block<OS_PAGE_SIZE>(p, size) || (zero_mem == NoZero));
if constexpr (zero_mem == YesZero)
{
zero(p, size);
}
else
{
UNUSED(size);
UNUSED(p);
}
}
/// OS specific function for zeroing memory

4
src/test/func/malloc/malloc.cc
View File

@@ -77,7 +77,7 @@ int main(int argc, char** argv)
test_calloc(0, 0, SUCCESS, false);
for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++)
for (snmalloc::sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++)
{
const size_t size = sizeclass_to_size(sc);
@@ -103,7 +103,7 @@ int main(int argc, char** argv)
for (size_t align = sizeof(size_t); align <= SUPERSLAB_SIZE; align <<= 1)
{
for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++)
for (snmalloc::sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++)
{
const size_t size = sizeclass_to_size(sc);
test_posix_memalign(size, align, SUCCESS, false);

4
src/test/func/sizeclass/sizeclass.cc
View File

@@ -2,7 +2,7 @@
#include <snmalloc.h>
NOINLINE
uint8_t size_to_sizeclass(size_t size)
snmalloc::sizeclass_t size_to_sizeclass(size_t size)
{
return snmalloc::size_to_sizeclass(size);
}
@@ -17,7 +17,7 @@ int main(int, char**)
std::cout << "sizeclass |-> [size_low, size_high] " << std::endl;
for (uint8_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++)
for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++)
{
// Separate printing for small and medium sizeclasses
if (sz == snmalloc::NUM_SMALL_CLASSES)