Files
snmalloc/src/ds/dllist.h
Matthew Parkinson d900e29424 Improve slow path performance for allocation (#143)
* Remote dealloc refactor.

* Improve remote dealloc

Change remote to count down to 0, so fast path does not need a constant.

Use signed value so that branch does not depend on addition.

* Inline remote_dealloc

The fast path of remote_dealloc is sufficiently compact that it can be
inlined.

* Improve fast path in Slab::alloc

Turn the internal structure into tail calls, to improve fast path.
Should be no algorithmic changes.

* Refactor initialisation to help fast path.

Break lazy initialisation into two functions, so it is easier to codegen
fast paths.

* Minor tidy to statically sized dealloc.

* Refactor semi-slow path for alloc

Make the backup path a bit faster.  Only algorithmic change is to delay
checking for first allocation. Otherwise, should be unchanged.

* Test initial operation of a thread

The first operation a new thread takes is special.  It results in
allocating an allocator, and swinging it into the TLS.  This makes
this a very special path, that is rarely tested.  This test generates
a lot of threads to cover the first alloc and dealloc operations.

* Correctly handle reusing get_noncachable

* Fix large alloc stats

Large alloc stats aren't necessarily balanced on a thread, this changes
to tracking individual pushs and pops, rather than the net effect
(with an unsigned value).

* Fix TLS init on large alloc path

* Add Bump ptrs to allocator

Each allocator has a bump ptr for each size class.  This is no longer
slab local.

Slabs that haven't been fully allocated no longer need to be in the DLL
for this sizeclass.

* Change to a cycle non-empty list

This change reduces the branching in the case of finding a new free
list. Using a non-empty cyclic list enables branch free add, and a
single branch in remove to detect the empty case.

* Update differences

* Rename first allocation

Use needs initialisation as makes more sense for other scenarios.

* Use a ptrdiff to help with zero init.

* Make GlobalPlaceholder zero init

The GlobalPlaceholder allocator is now a zero init block of memory.
This removes various issues for when things are initialised. It is made read-only
to we detect write to it on some platforms.
2020-03-31 09:17:53 +01:00

254 lines
4.7 KiB
C++

#pragma once
#include <cstdint>
#include <type_traits>
namespace snmalloc
{
/**
* Invalid pointer class. This is similar to `std::nullptr_t`, but allows
* other values.
*/
template<address_t Sentinel>
struct InvalidPointer
{
/**
* Equality comparison. Two invalid pointer values with the same sentinel
* are always the same, invalid pointer values with different sentinels are
* always different.
*/
template<uintptr_t OtherSentinel>
constexpr bool operator==(const InvalidPointer<OtherSentinel>&)
{
return Sentinel == OtherSentinel;
}
/**
* Equality comparison. Two invalid pointer values with the same sentinel
* are always the same, invalid pointer values with different sentinels are
* always different.
*/
template<uintptr_t OtherSentinel>
constexpr bool operator!=(const InvalidPointer<OtherSentinel>&)
{
return Sentinel != OtherSentinel;
}
/**
* Implicit conversion, creates a pointer with the value of the sentinel.
* On CHERI and other provenance-tracking systems, this is a
* provenance-free integer and so will trap if dereferenced, on other
* systems the sentinel should be a value in unmapped memory.
*/
template<typename T>
operator T*() const
{
return reinterpret_cast<T*>(Sentinel);
}
/**
* Implicit conversion to an address, returns the sentinel value.
*/
operator address_t() const
{
return Sentinel;
}
};
template<
class T,
class Terminator = std::nullptr_t,
bool delete_on_clear = false>
class DLList final
{
private:
static_assert(
std::is_same<decltype(T::prev), T*>::value, "T->prev must be a T*");
static_assert(
std::is_same<decltype(T::next), T*>::value, "T->next must be a T*");
T* head = Terminator();
T* tail = Terminator();
public:
~DLList()
{
clear();
}
DLList() = default;
DLList(DLList&& o) noexcept
{
head = o.head;
tail = o.tail;
o.head = nullptr;
o.tail = nullptr;
}
DLList& operator=(DLList&& o) noexcept
{
head = o.head;
tail = o.tail;
o.head = nullptr;
o.tail = nullptr;
return *this;
}
SNMALLOC_FAST_PATH bool is_empty()
{
return head == Terminator();
}
SNMALLOC_FAST_PATH T* get_head()
{
return head;
}
T* get_tail()
{
return tail;
}
SNMALLOC_FAST_PATH T* pop()
{
T* item = head;
if (item != Terminator())
remove(item);
return item;
}
T* pop_tail()
{
T* item = tail;
if (item != Terminator())
remove(item);
return item;
}
void insert(T* item)
{
#ifndef NDEBUG
debug_check_not_contains(item);
#endif
item->next = head;
item->prev = Terminator();
if (head != Terminator())
head->prev = item;
else
tail = item;
head = item;
#ifndef NDEBUG
debug_check();
#endif
}
void insert_back(T* item)
{
#ifndef NDEBUG
debug_check_not_contains(item);
#endif
item->prev = tail;
item->next = Terminator();
if (tail != Terminator())
tail->next = item;
else
head = item;
tail = item;
#ifndef NDEBUG
debug_check();
#endif
}
SNMALLOC_FAST_PATH void remove(T* item)
{
#ifndef NDEBUG
debug_check_contains(item);
#endif
if (item->next != Terminator())
item->next->prev = item->prev;
else
tail = item->prev;
if (item->prev != Terminator())
item->prev->next = item->next;
else
head = item->next;
#ifndef NDEBUG
debug_check();
#endif
}
void clear()
{
while (head != nullptr)
{
auto c = head;
remove(c);
if (delete_on_clear)
{
delete c;
}
}
}
void debug_check_contains(T* item)
{
#ifndef NDEBUG
debug_check();
T* curr = head;
while (curr != item)
{
SNMALLOC_ASSERT(curr != Terminator());
curr = curr->next;
}
#else
UNUSED(item);
#endif
}
void debug_check_not_contains(T* item)
{
#ifndef NDEBUG
debug_check();
T* curr = head;
while (curr != Terminator())
{
SNMALLOC_ASSERT(curr != item);
curr = curr->next;
}
#else
UNUSED(item);
#endif
}
void debug_check()
{
#ifndef NDEBUG
T* item = head;
T* prev = Terminator();
while (item != Terminator())
{
SNMALLOC_ASSERT(item->prev == prev);
prev = item;
item = item->next;
}
#endif
}
};
} // namespace snmalloc