Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[proxy](1/n) start self-vendored STL with atomic (#715)

Browse Source 
* [proxy] start self-vendored STL

* address CR
This commit is contained in:
Schrodinger ZHU Yifan
2025-01-03 15:49:32 +08:00
committed by GitHub
parent fb29a5e085
commit feff2e8151
34 changed files with 469 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/snmalloc/backend/globalconfig.h
View File

@@ -84,7 +84,7 @@ namespace snmalloc
* Specifies if the Configuration has been initialised.
*/
SNMALLOC_REQUIRE_CONSTINIT
inline static std::atomic<bool> initialised{false};
inline static stl::Atomic<bool> initialised{false};
/**
* Used to prevent two threads attempting to initialise the configuration
@@ -126,7 +126,7 @@ namespace snmalloc
Authmap::init();
}
initialised.store(true, std::memory_order_release);
initialised.store(true, stl::memory_order_release);
});
}
@@ -146,7 +146,7 @@ namespace snmalloc
// and concurrency safe.
SNMALLOC_FAST_PATH static void ensure_init()
{
if (SNMALLOC_LIKELY(initialised.load(std::memory_order_acquire)))
if (SNMALLOC_LIKELY(initialised.load(stl::memory_order_acquire)))
return;
ensure_init_slow();

2
src/snmalloc/backend_helpers/pagemap.h
View File

@@ -2,8 +2,8 @@
#include "../ds/ds.h"
#include "../mem/mem.h"
#include "snmalloc/stl/atomic.h"
#include <atomic>
#include <utility>
namespace snmalloc

7
src/snmalloc/backend_helpers/statsrange.h
View File

@@ -2,8 +2,7 @@
#include "empty_range.h"
#include "range_helpers.h"
#include <atomic>
#include "snmalloc/stl/atomic.h"
namespace snmalloc
{
@@ -17,8 +16,8 @@ namespace snmalloc
{
using ContainsParent<ParentRange>::parent;
static inline std::atomic<size_t> current_usage{};
static inline std::atomic<size_t> peak_usage{};
static inline stl::Atomic<size_t> current_usage{};
static inline stl::Atomic<size_t> peak_usage{};
public:
static constexpr bool Aligned = ParentRange::Aligned;

32
src/snmalloc/ds/aba.h
View File

@@ -35,8 +35,8 @@ namespace snmalloc
struct Independent
{
std::atomic<T*> ptr{nullptr};
std::atomic<uintptr_t> aba{0};
stl::Atomic<T*> ptr{nullptr};
stl::Atomic<uintptr_t> aba{0};
};
static_assert(
@@ -49,7 +49,7 @@ namespace snmalloc
private:
union
{
alignas(2 * sizeof(std::size_t)) std::atomic<Linked> linked;
alignas(2 * sizeof(std::size_t)) stl::Atomic<Linked> linked;
Independent independent;
};
@@ -58,8 +58,8 @@ namespace snmalloc
void init(T* x)
{
independent.ptr.store(x, std::memory_order_relaxed);
independent.aba.store(0, std::memory_order_relaxed);
independent.ptr.store(x, stl::memory_order_relaxed);
independent.aba.store(0, stl::memory_order_relaxed);
}
struct Cmp;
@@ -72,8 +72,8 @@ namespace snmalloc
operation_in_flight = true;
# endif
return Cmp{
{independent.ptr.load(std::memory_order_relaxed),
independent.aba.load(std::memory_order_relaxed)},
{independent.ptr.load(stl::memory_order_relaxed),
independent.aba.load(stl::memory_order_relaxed)},
this};
}
@@ -109,10 +109,10 @@ namespace snmalloc
# endif
Linked xchg{value, old.aba + 1};
std::atomic<Linked>& addr = parent->linked;
stl::Atomic<Linked>& addr = parent->linked;
auto result = addr.compare_exchange_weak(
old, xchg, std::memory_order_acq_rel, std::memory_order_relaxed);
old, xchg, stl::memory_order_acq_rel, stl::memory_order_relaxed);
# endif
return result;
}
@@ -131,7 +131,7 @@ namespace snmalloc
// This method is used in Verona
T* peek()
{
return independent.ptr.load(std::memory_order_relaxed);
return independent.ptr.load(stl::memory_order_relaxed);
}
};
#else
@@ -141,21 +141,21 @@ namespace snmalloc
template<typename T, Construction c = RequiresInit>
class ABA
{
std::atomic<T*> ptr = nullptr;
std::atomic<bool> lock{false};
stl::Atomic<T*> ptr = nullptr;
stl::Atomic<bool> lock{false};
public:
// This method is used in Verona
void init(T* x)
{
ptr.store(x, std::memory_order_relaxed);
ptr.store(x, stl::memory_order_relaxed);
}
struct Cmp;
Cmp read()
{
while (lock.exchange(true, std::memory_order_acquire))
while (lock.exchange(true, stl::memory_order_acquire))
Aal::pause();
# if !defined(NDEBUG) && !defined(SNMALLOC_DISABLE_ABA_VERIFY)
@@ -185,7 +185,7 @@ namespace snmalloc
~Cmp()
{
parent->lock.store(false, std::memory_order_release);
parent->lock.store(false, stl::memory_order_release);
# if !defined(NDEBUG) && !defined(SNMALLOC_DISABLE_ABA_VERIFY)
operation_in_flight = false;
# endif
@@ -195,7 +195,7 @@ namespace snmalloc
// This method is used in Verona
T* peek()
{
return ptr.load(std::memory_order_relaxed);
return ptr.load(stl::memory_order_relaxed);
}
};
#endif

49
src/snmalloc/ds/combininglock.h
View File

@@ -2,8 +2,7 @@
#include "../aal/aal.h"
#include "../pal/pal.h"
#include <atomic>
#include "snmalloc/stl/atomic.h"
namespace snmalloc
{
@@ -12,14 +11,14 @@ namespace snmalloc
struct CombiningLock
{
// Fast path lock incase there is no contention.
std::atomic<bool> flag{false};
stl::Atomic<bool> flag{false};
// MCS queue of work items
std::atomic<CombiningLockNode*> last{nullptr};
stl::Atomic<CombiningLockNode*> last{nullptr};
void release()
{
flag.store(false, std::memory_order_release);
flag.store(false, stl::memory_order_release);
}
};
@@ -86,10 +85,10 @@ namespace snmalloc
// Status of the queue, set by the thread at the head of the queue,
// When it makes the thread for this node either the head of the queue
// or completes its work.
std::atomic<LockStatus> status{LockStatus::WAITING};
stl::Atomic<LockStatus> status{LockStatus::WAITING};
// Used to store the queue
std::atomic<CombiningLockNode*> next{nullptr};
stl::Atomic<CombiningLockNode*> next{nullptr};
// Stores the C++ lambda associated with this node in the queue.
void (*f_raw)(CombiningLockNode*);
@@ -98,7 +97,7 @@ namespace snmalloc
void set_status(LockStatus s)
{
status.store(s, std::memory_order_release);
status.store(s, stl::memory_order_release);
}
template<typename Pal = DefaultPal>
@@ -111,7 +110,7 @@ namespace snmalloc
else
{
if (
node->status.exchange(message, std::memory_order_acq_rel) ==
node->status.exchange(message, stl::memory_order_acq_rel) ==
LockStatus::SLEEPING)
{
Pal::notify_one_on_address(node->status);
@@ -124,7 +123,7 @@ namespace snmalloc
{
if constexpr (!use_wait_on_address<Pal>)
{
while (status.load(std::memory_order_acquire) == LockStatus::WAITING)
while (status.load(stl::memory_order_acquire) == LockStatus::WAITING)
Aal::pause();
}
else
@@ -132,14 +131,14 @@ namespace snmalloc
int remaining = 100;
while (remaining > 0)
{
if (status.load(std::memory_order_acquire) != LockStatus::WAITING)
if (status.load(stl::memory_order_acquire) != LockStatus::WAITING)
return;
Aal::pause();
remaining--;
}
LockStatus expected = LockStatus::WAITING;
if (status.compare_exchange_strong(
expected, LockStatus::SLEEPING, std::memory_order_acq_rel))
expected, LockStatus::SLEEPING, stl::memory_order_acq_rel))
{
Pal::wait_on_address(status, LockStatus::SLEEPING);
}
@@ -150,18 +149,18 @@ namespace snmalloc
{
// There is contention for the lock, we need to add our work to the
// queue of pending work
auto prev = lock.last.exchange(this, std::memory_order_acq_rel);
auto prev = lock.last.exchange(this, stl::memory_order_acq_rel);
if (prev != nullptr)
{
// If we aren't the head, link into predecessor
prev->next.store(this, std::memory_order_release);
prev->next.store(this, stl::memory_order_release);
// Wait to for predecessor to complete
wait();
// Determine if another thread completed our work.
if (status.load(std::memory_order_acquire) == LockStatus::DONE)
if (status.load(stl::memory_order_acquire) == LockStatus::DONE)
return;
}
else
@@ -170,9 +169,9 @@ namespace snmalloc
// lock. As we are in the queue future requests shouldn't try to
// acquire the fast path lock, but stale views of the queue being empty
// could still be concurrent with this thread.
while (lock.flag.exchange(true, std::memory_order_acquire))
while (lock.flag.exchange(true, stl::memory_order_acquire))
{
while (lock.flag.load(std::memory_order_relaxed))
while (lock.flag.load(stl::memory_order_relaxed))
{
Aal::pause();
}
@@ -190,14 +189,14 @@ namespace snmalloc
while (true)
{
// Start pulling in the next element of the queue
auto n = curr->next.load(std::memory_order_acquire);
auto n = curr->next.load(stl::memory_order_acquire);
Aal::prefetch(n);
// Perform work for head of the queue
curr->f_raw(curr);
// Determine if there are more elements.
n = curr->next.load(std::memory_order_acquire);
n = curr->next.load(stl::memory_order_acquire);
if (n == nullptr)
break;
// Signal this work was completed and move on to
@@ -212,8 +211,8 @@ namespace snmalloc
if (lock.last.compare_exchange_strong(
curr_c,
nullptr,
std::memory_order_release,
std::memory_order_relaxed))
stl::memory_order_release,
stl::memory_order_relaxed))
{
// Queue was successfully closed.
// Notify last element the work was completed.
@@ -224,10 +223,10 @@ namespace snmalloc
// Failed to close the queue wait for next thread to be
// added.
while (curr->next.load(std::memory_order_relaxed) == nullptr)
while (curr->next.load(stl::memory_order_relaxed) == nullptr)
Aal::pause();
auto n = curr->next.load(std::memory_order_acquire);
auto n = curr->next.load(stl::memory_order_acquire);
// As we had to wait, give the job to the next thread
// to carry on performing the work.
@@ -272,12 +271,12 @@ namespace snmalloc
inline void with(CombiningLock& lock, F&& f)
{
// Test if no one is waiting
if (SNMALLOC_LIKELY(lock.last.load(std::memory_order_relaxed) == nullptr))
if (SNMALLOC_LIKELY(lock.last.load(stl::memory_order_relaxed) == nullptr))
{
// No one was waiting so low contention. Attempt to acquire the flag
// lock.
if (SNMALLOC_LIKELY(
lock.flag.exchange(true, std::memory_order_acquire) == false))
lock.flag.exchange(true, stl::memory_order_acquire) == false))
{
// We grabbed the lock.
// Execute the thunk.

17
src/snmalloc/ds/flaglock.h
View File

@@ -2,14 +2,13 @@
#include "../aal/aal.h"
#include "../pal/pal.h"
#include <atomic>
#include "snmalloc/stl/atomic.h"
namespace snmalloc
{
/**
* @brief The DebugFlagWord struct
* Wrapper for std::atomic_flag so that we can examine
* Wrapper for stl::AtomicBool so that we can examine
* the re-entrancy problem at debug mode.
*/
struct DebugFlagWord
@@ -20,7 +19,7 @@ namespace snmalloc
* @brief flag
* The underlying atomic field.
*/
std::atomic_bool flag{false};
stl::AtomicBool flag{false};
constexpr DebugFlagWord() = default;
@@ -62,7 +61,7 @@ namespace snmalloc
* @brief owner
* We use the Pal to provide the ThreadIdentity.
*/
std::atomic<ThreadIdentity> owner = ThreadIdentity();
stl::Atomic<ThreadIdentity> owner = ThreadIdentity();
/**
* @brief get_thread_identity
@@ -82,7 +81,7 @@ namespace snmalloc
*/
struct ReleaseFlagWord
{
std::atomic_bool flag{false};
stl::AtomicBool flag{false};
constexpr ReleaseFlagWord() = default;
@@ -112,7 +111,7 @@ namespace snmalloc
public:
FlagLock(FlagWord& lock) : lock(lock)
{
while (lock.flag.exchange(true, std::memory_order_acquire))
while (lock.flag.exchange(true, stl::memory_order_acquire))
{
// assert_not_owned_by_current_thread is only called when the first
// acquiring is failed; which means the lock is already held somewhere
@@ -120,7 +119,7 @@ namespace snmalloc
lock.assert_not_owned_by_current_thread();
// This loop is better for spin-waiting because it won't issue
// expensive write operation (xchg for example).
while (lock.flag.load(std::memory_order_relaxed))
while (lock.flag.load(stl::memory_order_relaxed))
{
Aal::pause();
}
@@ -131,7 +130,7 @@ namespace snmalloc
~FlagLock()
{
lock.clear_owner();
lock.flag.store(false, std::memory_order_release);
lock.flag.store(false, stl::memory_order_release);
}
};

14
src/snmalloc/ds/mpmcstack.h
View File

@@ -16,17 +16,17 @@ namespace snmalloc
#ifdef SNMALLOC_THREAD_SANITIZER_ENABLED
__attribute__((no_sanitize("thread"))) static T*
racy_read(std::atomic<T*>& ptr)
racy_read(stl::Atomic<T*>& ptr)
{
// reinterpret_cast is required as TSAN still instruments
// std::atomic operations, even if you disable TSAN on
// stl::Atomic operations, even if you disable TSAN on
// the function.
return *reinterpret_cast<T**>(&ptr);
}
#else
static T* racy_read(std::atomic<T*>& ptr)
static T* racy_read(stl::Atomic<T*>& ptr)
{
return ptr.load(std::memory_order_relaxed);
return ptr.load(stl::memory_order_relaxed);
}
#endif
@@ -36,8 +36,8 @@ namespace snmalloc
void push(T* item)
{
static_assert(
std::is_same<decltype(T::next), std::atomic<T*>>::value,
"T->next must be an std::atomic<T*>");
std::is_same<decltype(T::next), stl::Atomic<T*>>::value,
"T->next must be an stl::Atomic<T*>");
return push(item, item);
}
@@ -50,7 +50,7 @@ namespace snmalloc
do
{
auto top = cmp.ptr();
last->next.store(top, std::memory_order_release);
last->next.store(top, stl::memory_order_release);
} while (!cmp.store_conditional(first));
}

8
src/snmalloc/ds/singleton.h
View File

@@ -2,8 +2,8 @@
#include "../ds_core/ds_core.h"
#include "flaglock.h"
#include "snmalloc/stl/atomic.h"
#include <atomic>
#include <type_traits>
namespace snmalloc
@@ -17,7 +17,7 @@ namespace snmalloc
class Singleton
{
inline static FlagWord flag;
inline static std::atomic<bool> initialised{false};
inline static stl::Atomic<bool> initialised{false};
inline static Object obj;
public:
@@ -31,13 +31,13 @@ namespace snmalloc
// If defined should be initially false;
SNMALLOC_ASSERT(first == nullptr || *first == false);
if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire)))
if (SNMALLOC_UNLIKELY(!initialised.load(stl::memory_order_acquire)))
{
with(flag, [&]() {
if (!initialised)
{
init(&obj);
initialised.store(true, std::memory_order_release);
initialised.store(true, stl::memory_order_release);
if (first != nullptr)
*first = true;
}

2
src/snmalloc/ds_core/bits.h
View File

@@ -6,8 +6,8 @@
// #define USE_LZCNT
#include "defines.h"
#include "snmalloc/stl/atomic.h"
#include <atomic>
#include <climits>
#include <cstdint>
#include <type_traits>

19
src/snmalloc/ds_core/ptrwrap.h
View File

@@ -2,8 +2,7 @@
#include "concept.h"
#include "defines.h"
#include <atomic>
#include "snmalloc/stl/atomic.h"
namespace snmalloc
{
@@ -41,7 +40,7 @@ namespace snmalloc
using Pointer = T*;
template<typename T>
using AtomicPointer = std::atomic<T*>;
using AtomicPointer = stl::Atomic<T*>;
/**
* Summaries of StrictProvenance metadata. We abstract away the particular
@@ -450,17 +449,17 @@ namespace snmalloc
/**
*
* Wrap a std::atomic<T*> with bounds annotation and speak in terms of
* Wrap a stl::Atomic<T*> with bounds annotation and speak in terms of
* bounds-annotated pointers at the interface.
*
* Note the membranous sleight of hand being pulled here: this class puts
* annotations around an un-annotated std::atomic<T*>, to appease C++, yet
* annotations around an un-annotated stl::Atomic<T*>, to appease C++, yet
* will expose or consume only CapPtr<T> with the same bounds annotation.
*/
template<typename T, SNMALLOC_CONCEPT(capptr::IsBound) bounds>
class AtomicCapPtr
{
std::atomic<T*> unsafe_capptr;
stl::Atomic<T*> unsafe_capptr;
public:
/**
@@ -487,7 +486,7 @@ namespace snmalloc
return CapPtr<T, bounds>(this->unsafe_capptr);
}
// Our copy-assignment operator follows std::atomic and returns a copy of
// Our copy-assignment operator follows stl::Atomic and returns a copy of
// the RHS. Clang finds this surprising; we suppress the warning.
// NOLINTNEXTLINE(misc-unconventional-assign-operator)
SNMALLOC_FAST_PATH CapPtr<T, bounds> operator=(CapPtr<T, bounds> p) noexcept
@@ -497,21 +496,21 @@ namespace snmalloc
}
SNMALLOC_FAST_PATH CapPtr<T, bounds>
load(std::memory_order order = std::memory_order_seq_cst) noexcept
load(stl::MemoryOrder order = stl::memory_order_seq_cst) noexcept
{
return CapPtr<T, bounds>::unsafe_from(this->unsafe_capptr.load(order));
}
SNMALLOC_FAST_PATH void store(
CapPtr<T, bounds> desired,
std::memory_order order = std::memory_order_seq_cst) noexcept
stl::MemoryOrder order = stl::memory_order_seq_cst) noexcept
{
this->unsafe_capptr.store(desired.unsafe_ptr(), order);
}
SNMALLOC_FAST_PATH CapPtr<T, bounds> exchange(
CapPtr<T, bounds> desired,
std::memory_order order = std::memory_order_seq_cst) noexcept
stl::MemoryOrder order = stl::memory_order_seq_cst) noexcept
{
return CapPtr<T, bounds>::unsafe_from(
this->unsafe_capptr.exchange(desired.unsafe_ptr(), order));

6
src/snmalloc/mem/freelist.h
View File

@@ -197,7 +197,7 @@ namespace snmalloc
{
auto n_wild = Object::decode_next(
address_cast(&this->next_object),
this->atomic_next_object.load(std::memory_order_acquire),
this->atomic_next_object.load(stl::memory_order_acquire),
key,
key_tweak);
auto n_tame = domesticate(n_wild);
@@ -472,7 +472,7 @@ namespace snmalloc
// so requires release semantics.
curr->atomic_next_object.store(
encode_next(address_cast(&curr->next_object), next, key, key_tweak),
std::memory_order_release);
stl::memory_order_release);
}
template<
@@ -491,7 +491,7 @@ namespace snmalloc
BQueuePtr<BQueue>(nullptr),
key,
key_tweak),
std::memory_order_relaxed);
stl::memory_order_relaxed);
}
};

11
src/snmalloc/mem/freelist_queue.h
View File

@@ -2,8 +2,7 @@
#include "../ds/ds.h"
#include "freelist.h"
#include <atomic>
#include "snmalloc/stl/atomic.h"
namespace snmalloc
{
@@ -61,14 +60,14 @@ namespace snmalloc
freelist::HeadPtr stub_ptr = freelist::HeadPtr::unsafe_from(&stub);
freelist::Object::atomic_store_null(stub_ptr, Key, Key_tweak);
front.store(freelist::QueuePtr::unsafe_from(&stub));
back.store(nullptr, std::memory_order_relaxed);
back.store(nullptr, stl::memory_order_relaxed);
invariant();
}
freelist::QueuePtr destroy()
{
freelist::QueuePtr fnt = front.load();
back.store(nullptr, std::memory_order_relaxed);
back.store(nullptr, stl::memory_order_relaxed);
if (address_cast(front.load()) == address_cast(&stub))
return nullptr;
return fnt;
@@ -116,7 +115,7 @@ namespace snmalloc
// * Needs to be acquire, so linking into the list does not race with
// the other threads nullptr init of the next field.
freelist::QueuePtr prev =
back.exchange(capptr_rewild(last), std::memory_order_acq_rel);
back.exchange(capptr_rewild(last), stl::memory_order_acq_rel);
if (SNMALLOC_LIKELY(prev != nullptr))
{
@@ -149,7 +148,7 @@ namespace snmalloc
SNMALLOC_ASSERT(front.load() != nullptr);
// Use back to bound, so we don't handle new entries.
auto b = back.load(std::memory_order_relaxed);
auto b = back.load(stl::memory_order_relaxed);
freelist::HeadPtr curr = domesticate_head(front.load());
while (address_cast(curr) != address_cast(b))

10
src/snmalloc/mem/pooled.h
View File

@@ -15,14 +15,14 @@ namespace snmalloc
template<class T>
class PoolState;
// clang-format off
#ifdef __cpp_concepts
template<typename C, typename T>
concept Constructable = requires() {
{
C::make()
} -> ConceptSame<capptr::Alloc<T>>;
};
{ C::make() } -> ConceptSame<capptr::Alloc<T>>;
};
#endif // __cpp_concepts
/**
* Required to be implemented by all types that are pooled.
@@ -46,7 +46,7 @@ namespace snmalloc
capptr::Alloc<T> next{nullptr};
/// Used by the pool to keep the list of all entries ever created.
capptr::Alloc<T> list_next;
std::atomic<bool> in_use{false};
stl::Atomic<bool> in_use{false};
public:
void set_in_use()

2
src/snmalloc/mem/remotecache.h
View File

@@ -6,9 +6,9 @@
#include "metadata.h"
#include "remoteallocator.h"
#include "sizeclasstable.h"
#include "snmalloc/stl/atomic.h"
#include <array>
#include <atomic>
namespace snmalloc
{

8
src/snmalloc/pal/pal_apple.h
View File

@@ -321,10 +321,10 @@ namespace snmalloc
# endif
template<class T>
static void wait_on_address(std::atomic<T>& addr, T expected)
static void wait_on_address(stl::Atomic<T>& addr, T expected)
{
[[maybe_unused]] int errno_backup = errno;
while (addr.load(std::memory_order_relaxed) == expected)
while (addr.load(stl::memory_order_relaxed) == expected)
{
# ifdef SNMALLOC_APPLE_HAS_OS_SYNC_WAIT_ON_ADDRESS
if (
@@ -349,7 +349,7 @@ namespace snmalloc
}
template<class T>
static void notify_one_on_address(std::atomic<T>& addr)
static void notify_one_on_address(stl::Atomic<T>& addr)
{
# ifdef SNMALLOC_APPLE_HAS_OS_SYNC_WAIT_ON_ADDRESS
os_sync_wake_by_address_any(&addr, sizeof(T), 0);
@@ -366,7 +366,7 @@ namespace snmalloc
}
template<class T>
static void notify_all_on_address(std::atomic<T>& addr)
static void notify_all_on_address(stl::Atomic<T>& addr)
{
# ifdef SNMALLOC_APPLE_HAS_OS_SYNC_WAIT_ON_ADDRESS
os_sync_wake_by_address_all(&addr, sizeof(T), 0);

3
src/snmalloc/pal/pal_consts.h
View File

@@ -1,8 +1,7 @@
#pragma once
#include "../ds_core/ds_core.h"
#include <atomic>
#include "snmalloc/stl/atomic.h"
namespace snmalloc
{

17
src/snmalloc/pal/pal_ds.h
View File

@@ -1,8 +1,7 @@
#pragma once
#include "../ds_core/ds_core.h"
#include <atomic>
#include "snmalloc/stl/atomic.h"
namespace snmalloc
{
@@ -12,10 +11,10 @@ namespace snmalloc
/**
* List of callbacks to notify
*/
std::atomic<T*> elements{nullptr};
stl::Atomic<T*> elements{nullptr};
static_assert(
std::is_same<decltype(T::pal_next), std::atomic<T*>>::value,
std::is_same<decltype(T::pal_next), stl::Atomic<T*>>::value,
"Required pal_next type.");
public:
@@ -57,7 +56,7 @@ namespace snmalloc
*/
struct PalNotificationObject
{
std::atomic<PalNotificationObject*> pal_next = nullptr;
stl::Atomic<PalNotificationObject*> pal_next = nullptr;
void (*pal_notify)(PalNotificationObject* self);
@@ -103,7 +102,7 @@ namespace snmalloc
template<typename T>
friend class PalList;
std::atomic<PalTimerObject*> pal_next;
stl::Atomic<PalTimerObject*> pal_next;
void (*pal_notify)(PalTimerObject* self);
@@ -140,10 +139,10 @@ namespace snmalloc
void check(uint64_t time_ms)
{
static std::atomic_bool lock{false};
static stl::AtomicBool lock{false};
// Deduplicate calls into here, and make single threaded.
if (lock.exchange(true, std::memory_order_acquire))
if (lock.exchange(true, stl::memory_order_acquire))
return;
timers.apply_all([time_ms](PalTimerObject* curr) {
@@ -155,7 +154,7 @@ namespace snmalloc
}
});
lock.store(false, std::memory_order_release);
lock.store(false, stl::memory_order_release);
}
};
} // namespace snmalloc

8
src/snmalloc/pal/pal_freebsd.h
View File

@@ -137,13 +137,13 @@ namespace snmalloc
using WaitingWord = unsigned int;
template<typename T>
static void wait_on_address(std::atomic<T>& addr, T expected)
static void wait_on_address(stl::Atomic<T>& addr, T expected)
{
static_assert(
sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord),
"T must be the same size and alignment as WaitingWord");
int backup = errno;
while (addr.load(std::memory_order_relaxed) == expected)
while (addr.load(stl::memory_order_relaxed) == expected)
{
int ret = _umtx_op(
&addr,
@@ -159,7 +159,7 @@ namespace snmalloc
}
template<typename T>
static void notify_one_on_address(std::atomic<T>& addr)
static void notify_one_on_address(stl::Atomic<T>& addr)
{
static_assert(
sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord),
@@ -168,7 +168,7 @@ namespace snmalloc
}
template<typename T>
static void notify_all_on_address(std::atomic<T>& addr)
static void notify_all_on_address(stl::Atomic<T>& addr)
{
static_assert(
sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord),

14
src/snmalloc/pal/pal_linux.h
View File

@@ -177,11 +177,11 @@ namespace snmalloc
// give a try to SYS_getrandom
# ifdef SYS_getrandom
static std::atomic_bool syscall_not_working = false;
static stl::AtomicBool syscall_not_working = false;
// Relaxed ordering should be fine here. This function will be called
// during early initialisation, which will examine the availability in a
// protected routine.
if (false == syscall_not_working.load(std::memory_order_relaxed))
if (false == syscall_not_working.load(stl::memory_order_relaxed))
{
auto current = std::begin(buffer);
auto target = std::end(buffer);
@@ -224,7 +224,7 @@ namespace snmalloc
// in this routine, the only possible situations should be ENOSYS
// or EPERM (forbidden by seccomp, for example).
SNMALLOC_ASSERT(errno == ENOSYS || errno == EPERM);
syscall_not_working.store(true, std::memory_order_relaxed);
syscall_not_working.store(true, stl::memory_order_relaxed);
}
else
{
@@ -246,13 +246,13 @@ namespace snmalloc
using WaitingWord = int;
template<class T>
static void wait_on_address(std::atomic<T>& addr, T expected)
static void wait_on_address(stl::Atomic<T>& addr, T expected)
{
int backup = errno;
static_assert(
sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord),
"T must be the same size and alignment as WaitingWord");
while (addr.load(std::memory_order_relaxed) == expected)
while (addr.load(stl::memory_order_relaxed) == expected)
{
long ret = syscall(
SYS_futex, &addr, FUTEX_WAIT_PRIVATE, expected, nullptr, nullptr, 0);
@@ -264,7 +264,7 @@ namespace snmalloc
}
template<class T>
static void notify_one_on_address(std::atomic<T>& addr)
static void notify_one_on_address(stl::Atomic<T>& addr)
{
static_assert(
sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord),
@@ -273,7 +273,7 @@ namespace snmalloc
}
template<class T>
static void notify_all_on_address(std::atomic<T>& addr)
static void notify_all_on_address(stl::Atomic<T>& addr)
{
static_assert(
sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord),

6
src/snmalloc/pal/pal_tid_default.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include <atomic>
#include "snmalloc/stl/atomic.h"
namespace snmalloc
{
@@ -18,7 +18,7 @@ namespace snmalloc
static inline ThreadIdentity get_tid() noexcept
{
static thread_local size_t tid{0};
static std::atomic<size_t> tid_source{0};
static stl::Atomic<size_t> tid_source{0};
if (tid == 0)
{
@@ -27,4 +27,4 @@ namespace snmalloc
return tid;
}
};
} // namespace snmalloc
} // namespace snmalloc

10
src/snmalloc/pal/pal_windows.h
View File

@@ -35,7 +35,7 @@ namespace snmalloc
* A flag indicating that we have tried to register for low-memory
* notifications.
*/
static inline std::atomic<bool> registered_for_notifications;
static inline stl::Atomic<bool> registered_for_notifications;
static inline HANDLE lowMemoryObject;
/**
@@ -240,9 +240,9 @@ namespace snmalloc
using WaitingWord = char;
template<class T>
static void wait_on_address(std::atomic<T>& addr, T expected)
static void wait_on_address(stl::Atomic<T>& addr, T expected)
{
while (addr.load(std::memory_order_relaxed) == expected)
while (addr.load(stl::memory_order_relaxed) == expected)
{
if (::WaitOnAddress(&addr, &expected, sizeof(T), INFINITE))
break;
@@ -250,13 +250,13 @@ namespace snmalloc
}
template<class T>
static void notify_one_on_address(std::atomic<T>& addr)
static void notify_one_on_address(stl::Atomic<T>& addr)
{
::WakeByAddressSingle(&addr);
}
template<class T>
static void notify_all_on_address(std::atomic<T>& addr)
static void notify_all_on_address(stl::Atomic<T>& addr)
{
::WakeByAddressAll(&addr);
}

4
src/snmalloc/stl/README.md Normal file
View File

@@ -0,0 +1,4 @@
# Standard Library Implementation
To support build environment without C++ STL, snmalloc can optionally use self-vendored STL functionalities.
To use self-vendored implementations, one need to set `SNMALLOC_USE_SELF_VENDORED_STL` to `ON` when configuring cmake.

9
src/snmalloc/stl/atomic.h Normal file
View File

@@ -0,0 +1,9 @@
#pragma once
#include "snmalloc/stl/common.h"
#if SNMALLOC_USE_SELF_VENDORED_STL
# include "snmalloc/stl/gnu/atomic.h"
#else
# include "snmalloc/stl/cxx/atomic.h"
#endif

15
src/snmalloc/stl/common.h Normal file
View File

@@ -0,0 +1,15 @@
#pragma once
#include "snmalloc/ds_core/defines.h"
// Default on using the system STL.
#ifndef SNMALLOC_USE_SELF_VENDORED_STL
# define SNMALLOC_USE_SELF_VENDORED_STL 0
#endif
// Check that the vendored STL is only used with GNU/Clang extensions.
#if SNMALLOC_USE_SELF_VENDORED_STL
# if !defined(__GNUC__) && !defined(__clang__)
# error "cannot use vendored STL without GNU/Clang extensions"
# endif
#endif

3
src/snmalloc/stl/cxx/README.md Normal file
View File

@@ -0,0 +1,3 @@
# CXX Standard Library
This directory provides an interface to use default STL.

22
src/snmalloc/stl/cxx/atomic.h Normal file
View File

@@ -0,0 +1,22 @@
#pragma once
#include <atomic>
namespace snmalloc
{
namespace stl
{
template<typename T>
using Atomic = std::atomic<T>;
constexpr auto memory_order_relaxed = std::memory_order_relaxed;
constexpr auto memory_order_consume = std::memory_order_consume;
constexpr auto memory_order_acquire = std::memory_order_acquire;
constexpr auto memory_order_release = std::memory_order_release;
constexpr auto memory_order_acq_rel = std::memory_order_acq_rel;
constexpr auto memory_order_seq_cst = std::memory_order_seq_cst;
using AtomicBool = std::atomic<bool>;
using MemoryOrder = std::memory_order;
} // namespace stl
} // namespace snmalloc

3
src/snmalloc/stl/gnu/README.md Normal file
View File

@@ -0,0 +1,3 @@
# Self-Vendored STL Using GNU Language Extensions
This directory contains implementations of STL functionalities using GNU extensions. Such extensions are also available with clang.

232
src/snmalloc/stl/gnu/atomic.h Normal file
View File

@@ -0,0 +1,232 @@
#pragma once
#include <stddef.h>
#include <type_traits> // TODO: switch the vendored headers when we have them.
namespace snmalloc
{
namespace stl
{
enum class MemoryOrder : int
{
RELAXED = __ATOMIC_RELAXED,
CONSUME = __ATOMIC_CONSUME,
ACQUIRE = __ATOMIC_ACQUIRE,
RELEASE = __ATOMIC_RELEASE,
ACQ_REL = __ATOMIC_ACQ_REL,
SEQ_CST = __ATOMIC_SEQ_CST
};
constexpr MemoryOrder memory_order_relaxed = MemoryOrder::RELAXED;
constexpr MemoryOrder memory_order_consume = MemoryOrder::CONSUME;
constexpr MemoryOrder memory_order_acquire = MemoryOrder::ACQUIRE;
constexpr MemoryOrder memory_order_release = MemoryOrder::RELEASE;
constexpr MemoryOrder memory_order_acq_rel = MemoryOrder::ACQ_REL;
constexpr MemoryOrder memory_order_seq_cst = MemoryOrder::SEQ_CST;
template<typename T>
class Atomic
{
static_assert(
std::is_trivially_copyable_v<T> && std::is_copy_constructible_v<T> &&
std::is_move_constructible_v<T> && std::is_copy_assignable_v<T> &&
std::is_move_assignable_v<T>,
"Atomic<T> requires T to be trivially copyable, copy "
"constructible, move constructible, copy assignable, "
"and move assignable.");
static_assert(
std::has_unique_object_representations_v<T>,
"vendored Atomic only supports types with unique object "
"representations");
// type conversion helper to avoid long c++ style casts
SNMALLOC_FAST_PATH static int order(MemoryOrder mem_ord)
{
return static_cast<int>(mem_ord);
}
SNMALLOC_FAST_PATH static int infer_failure_order(MemoryOrder mem_ord)
{
if (mem_ord == MemoryOrder::RELEASE)
return order(MemoryOrder::RELAXED);
if (mem_ord == MemoryOrder::ACQ_REL)
return order(MemoryOrder::ACQUIRE);
return order(mem_ord);
}
SNMALLOC_FAST_PATH static T* addressof(T& ref)
{
return __builtin_addressof(ref);
}
// From libc++:
// require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned
// to at least their size to be potentially
// used lock-free
static constexpr size_t MIN_ALIGNMENT =
(sizeof(T) & (sizeof(T) - 1)) || (sizeof(T) > 16) ? 0 : sizeof(T);
static constexpr size_t REQUIRED_ALIGNMENT =
alignof(T) > MIN_ALIGNMENT ? alignof(T) : MIN_ALIGNMENT;
alignas(REQUIRED_ALIGNMENT) T val;
public:
SNMALLOC_FAST_PATH constexpr Atomic() = default;
SNMALLOC_FAST_PATH constexpr Atomic(T v) : val(v) {}
SNMALLOC_FAST_PATH Atomic(const Atomic&) = delete;
SNMALLOC_FAST_PATH Atomic& operator=(const Atomic&) = delete;
// Atomic load.
SNMALLOC_FAST_PATH operator T()
{
return load();
}
SNMALLOC_FAST_PATH T load(MemoryOrder mem_ord = MemoryOrder::SEQ_CST)
{
T res;
__atomic_load(addressof(val), addressof(res), order(mem_ord));
return res;
}
// Atomic store.
// NOLINTNEXTLINE(misc-unconventional-assign-operator)
SNMALLOC_FAST_PATH
T operator=(T rhs)
{
store(rhs);
return rhs;
}
SNMALLOC_FAST_PATH void
store(T rhs, MemoryOrder mem_ord = MemoryOrder::SEQ_CST)
{
__atomic_store(addressof(val), addressof(rhs), order(mem_ord));
}
// Atomic compare exchange
SNMALLOC_FAST_PATH bool compare_exchange_strong(
T& expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST)
{
return __atomic_compare_exchange(
addressof(val),
addressof(expected),
addressof(desired),
false,
order(mem_ord),
infer_failure_order(mem_ord));
}
// Atomic compare exchange (separate success and failure memory orders)
SNMALLOC_FAST_PATH bool compare_exchange_strong(
T& expected,
T desired,
MemoryOrder success_order,
MemoryOrder failure_order)
{
return __atomic_compare_exchange(
addressof(val),
addressof(expected),
addressof(desired),
false,
order(success_order),
order(failure_order));
}
// Atomic compare exchange (weak version)
SNMALLOC_FAST_PATH bool compare_exchange_weak(
T& expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST)
{
return __atomic_compare_exchange(
addressof(val),
addressof(expected),
addressof(desired),
true,
order(mem_ord),
infer_failure_order(mem_ord));
}
// Atomic compare exchange (weak version with separate success and failure
// memory orders)
SNMALLOC_FAST_PATH bool compare_exchange_weak(
T& expected,
T desired,
MemoryOrder success_order,
MemoryOrder failure_order)
{
return __atomic_compare_exchange(
addressof(val),
addressof(expected),
addressof(desired),
true,
order(success_order),
order(failure_order));
}
SNMALLOC_FAST_PATH T
exchange(T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST)
{
T ret;
__atomic_exchange(
addressof(val), addressof(desired), addressof(ret), order(mem_ord));
return ret;
}
SNMALLOC_FAST_PATH T
fetch_add(T increment, MemoryOrder mem_ord = MemoryOrder::SEQ_CST)
{
static_assert(std::is_integral_v<T>, "T must be an integral type.");
return __atomic_fetch_add(addressof(val), increment, order(mem_ord));
}
SNMALLOC_FAST_PATH T
fetch_or(T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST)
{
static_assert(std::is_integral_v<T>, "T must be an integral type.");
return __atomic_fetch_or(addressof(val), mask, order(mem_ord));
}
SNMALLOC_FAST_PATH T
fetch_and(T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST)
{
static_assert(std::is_integral_v<T>, "T must be an integral type.");
return __atomic_fetch_and(addressof(val), mask, order(mem_ord));
}
SNMALLOC_FAST_PATH T
fetch_sub(T decrement, MemoryOrder mem_ord = MemoryOrder::SEQ_CST)
{
static_assert(std::is_integral_v<T>, "T must be an integral type.");
return __atomic_fetch_sub(addressof(val), decrement, order(mem_ord));
}
SNMALLOC_FAST_PATH T operator++()
{
static_assert(std::is_integral_v<T>, "T must be an integral type.");
return __atomic_add_fetch(
addressof(val), 1, order(MemoryOrder::SEQ_CST));
}
SNMALLOC_FAST_PATH const T operator++(int)
{
static_assert(std::is_integral_v<T>, "T must be an integral type.");
return __atomic_fetch_add(
addressof(val), 1, order(MemoryOrder::SEQ_CST));
}
SNMALLOC_FAST_PATH T operator-=(T decrement)
{
static_assert(std::is_integral_v<T>, "T must be an integral type.");
return __atomic_sub_fetch(
addressof(val), decrement, order(MemoryOrder::SEQ_CST));
}
};
using AtomicBool = Atomic<bool>;
} // namespace stl
} // namespace snmalloc

1
src/test/func/client_meta/client_meta.cc
View File

@@ -5,6 +5,7 @@
#include "test/setup.h"
#include <atomic>
#include <iostream>
#include <snmalloc/backend/globalconfig.h>
#include <snmalloc/snmalloc_core.h>

3
src/test/func/miracle_ptr/miracle_ptr.cc
View File

@@ -15,6 +15,7 @@ int main()
# include "test/setup.h"
# include <atomic>
# include <iostream>
# include <memory>
# include <snmalloc/backend/globalconfig.h>
@@ -201,4 +202,4 @@ int main()
# endif
return 0;
}
#endif
#endif

1
src/test/perf/low_memory/low-memory.cc
View File

@@ -1,3 +1,4 @@
#include <atomic>
#include <iostream>
#include <snmalloc/snmalloc.h>
#include <test/opt.h>

2
src/test/perf/msgpass/msgpass.cc
View File

@@ -275,7 +275,7 @@ int main(int argc, char** argv)
/* Spawn proxies */
for (size_t i = param.N_CONSUMER; i < param.N_QUEUE; i++)
{
queue_threads[i] = std::thread(proxy, &param, i);
queue_threads[i] = std::thread(::proxy, &param, i);
}
/* Spawn producers */