diff --git a/CMakeLists.txt b/CMakeLists.txt index e511dc3..29c9568 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -211,9 +211,7 @@ if(SNMALLOC_REMOTE_BATCH_PROCESS_SIZE) endif() if(SNMALLOC_USE_SELF_VENDORED_STL) - target_compile_definitions(snmalloc INTERFACE SNMALLOC_USE_SELF_VENDORED_STL=1) -else() - target_compile_definitions(snmalloc INTERFACE SNMALLOC_USE_SELF_VENDORED_STL=0) + target_compile_definitions(snmalloc INTERFACE SNMALLOC_USE_SELF_VENDORED_STL) endif() # https://learn.microsoft.com/en-us/cpp/build/reference/zc-cplusplus diff --git a/src/snmalloc/backend_helpers/buddy.h b/src/snmalloc/backend_helpers/buddy.h index d740646..58cafac 100644 --- a/src/snmalloc/backend_helpers/buddy.h +++ b/src/snmalloc/backend_helpers/buddy.h @@ -23,7 +23,7 @@ namespace snmalloc RBTree tree{}; }; - std::array entries{}; + stl::Array entries{}; // All RBtrees at or above this index should be empty. size_t empty_at_or_above{0}; @@ -167,7 +167,7 @@ namespace snmalloc { if (Rep::equal(Rep::null, addr) || Rep::compare(e, addr)) { - addr = std::exchange(e, addr); + addr = stl::exchange(e, addr); } } diff --git a/src/snmalloc/ds_core/defines.h b/src/snmalloc/ds_core/defines.h index fa85b68..8c83210 100644 --- a/src/snmalloc/ds_core/defines.h +++ b/src/snmalloc/ds_core/defines.h @@ -130,7 +130,8 @@ namespace snmalloc #define TOSTRING(expr) TOSTRING2(expr) #define TOSTRING2(expr) #expr -#ifdef __cpp_lib_source_location +#if defined(__cpp_lib_source_location) && \ + !defined(SNMALLOC_USE_SELF_VENDORED_STL) # include # define SNMALLOC_CURRENT_LINE std::source_location::current().line() # define SNMALLOC_CURRENT_FILE std::source_location::current().file_name() @@ -203,6 +204,13 @@ namespace snmalloc # define SNMALLOC_UNINITIALISED #endif +// Check that the vendored STL is only used with GNU/Clang extensions. +#ifdef SNMALLOC_USE_SELF_VENDORED_STL +# if !defined(__GNUC__) && !defined(__clang__) +# error "cannot use vendored STL without GNU/Clang extensions" +# endif +#endif + namespace snmalloc { /** diff --git a/src/snmalloc/ds_core/helpers.h b/src/snmalloc/ds_core/helpers.h index 2ff314b..3008da4 100644 --- a/src/snmalloc/ds_core/helpers.h +++ b/src/snmalloc/ds_core/helpers.h @@ -2,10 +2,10 @@ #include "bits.h" #include "snmalloc/ds_core/defines.h" +#include "snmalloc/stl/array.h" #include "snmalloc/stl/type_traits.h" #include "snmalloc/stl/utility.h" -#include #include namespace snmalloc @@ -50,7 +50,7 @@ namespace snmalloc }; static constexpr size_t rlength = bits::next_pow2_const(length); - std::array array; + stl::Array array; public: constexpr const T& operator[](const size_t i) const @@ -65,7 +65,7 @@ namespace snmalloc }; #else template - using ModArray = std::array; + using ModArray = stl::Array; #endif /** @@ -105,7 +105,7 @@ namespace snmalloc template< typename Fn, typename = - stl::enable_if_t, function_ref>>> + stl::enable_if_t, function_ref>>> function_ref(Fn&& fn) { data_ = static_cast(&fn); @@ -144,7 +144,7 @@ namespace snmalloc /** * The buffer that is used to store the formatted output. */ - std::array buffer; + stl::Array buffer; /** * Space in the buffer, excluding a trailing null terminator. @@ -208,7 +208,7 @@ namespace snmalloc /** * Append a nullptr */ - void append(std::nullptr_t) + void append(decltype(nullptr)) { append("(nullptr)"); } @@ -254,7 +254,7 @@ namespace snmalloc append_char('-'); s = 0 - s; } - std::array buf{{0}}; + stl::Array buf{{0}}; const char digits[] = "0123456789"; for (long i = static_cast(buf.size() - 1); i >= 0; i--) { @@ -284,7 +284,7 @@ namespace snmalloc { append_char('0'); append_char('x'); - std::array buf{{0}}; + stl::Array buf{{0}}; const char hexdigits[] = "0123456789abcdef"; // Length of string including null terminator static_assert(sizeof(hexdigits) == 0x11); diff --git a/src/snmalloc/ds_core/ptrwrap.h b/src/snmalloc/ds_core/ptrwrap.h index 8681e3d..1d6e41f 100644 --- a/src/snmalloc/ds_core/ptrwrap.h +++ b/src/snmalloc/ds_core/ptrwrap.h @@ -276,7 +276,7 @@ namespace snmalloc /** * nullptr is implicitly constructable at any bounds type */ - constexpr SNMALLOC_FAST_PATH CapPtr(const std::nullptr_t n) + constexpr SNMALLOC_FAST_PATH CapPtr(const decltype(nullptr) n) : unsafe_capptr(n) {} @@ -465,7 +465,7 @@ namespace snmalloc /** * nullptr is constructable at any bounds type */ - constexpr SNMALLOC_FAST_PATH AtomicCapPtr(const std::nullptr_t n) + constexpr SNMALLOC_FAST_PATH AtomicCapPtr(const decltype(nullptr) n) : unsafe_capptr(n) {} diff --git a/src/snmalloc/ds_core/redblacktree.h b/src/snmalloc/ds_core/redblacktree.h index 585c173..ec212fa 100644 --- a/src/snmalloc/ds_core/redblacktree.h +++ b/src/snmalloc/ds_core/redblacktree.h @@ -1,6 +1,7 @@ #pragma once -#include +#include "snmalloc/stl/array.h" + #include #include @@ -259,7 +260,7 @@ namespace snmalloc { friend class RBTree; - std::array path; + stl::Array path; size_t length = 0; RBPath(typename Rep::Handle root) diff --git a/src/snmalloc/global/memcpy.h b/src/snmalloc/global/memcpy.h index f52cea4..eaac1e9 100644 --- a/src/snmalloc/global/memcpy.h +++ b/src/snmalloc/global/memcpy.h @@ -156,7 +156,7 @@ namespace snmalloc */ SNMALLOC_UNUSED_FUNCTION static constexpr size_t LargestRegisterSize = - std::max(sizeof(uint64_t), sizeof(void*)); + bits::max(sizeof(uint64_t), sizeof(void*)); /** * Hook for architecture-specific optimisations. diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index a6481b6..f82ccc1 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -222,7 +222,7 @@ namespace snmalloc pointer_offset(curr, rsize).template as_static()) { size_t insert_index = entropy.sample(count); - curr->next = std::exchange( + curr->next = stl::exchange( pointer_offset(bumpptr, insert_index * rsize) .template as_static() ->next, diff --git a/src/snmalloc/mem/freelist.h b/src/snmalloc/mem/freelist.h index 7e5c9ca..fd91f6d 100644 --- a/src/snmalloc/mem/freelist.h +++ b/src/snmalloc/mem/freelist.h @@ -701,11 +701,11 @@ namespace snmalloc */ // Pointer to the first element. - std::array head{nullptr}; + stl::Array head{nullptr}; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - std::array end{nullptr}; + stl::Array end{nullptr}; [[nodiscard]] Object::BQueuePtr* cast_end(uint32_t ix) const { @@ -724,7 +724,7 @@ namespace snmalloc } SNMALLOC_NO_UNIQUE_ADDRESS - std::array length{}; + stl::Array length{}; public: constexpr Builder() = default; diff --git a/src/snmalloc/mem/remotecache.h b/src/snmalloc/mem/remotecache.h index 8a87ea1..3d5ed70 100644 --- a/src/snmalloc/mem/remotecache.h +++ b/src/snmalloc/mem/remotecache.h @@ -6,10 +6,9 @@ #include "metadata.h" #include "remoteallocator.h" #include "sizeclasstable.h" +#include "snmalloc/stl/array.h" #include "snmalloc/stl/atomic.h" -#include - namespace snmalloc { @@ -33,8 +32,8 @@ namespace snmalloc { static_assert(RINGS > 0); - std::array, RINGS> open_builder; - std::array open_meta = {0}; + stl::Array, RINGS> open_builder; + stl::Array open_meta = {0}; SNMALLOC_FAST_PATH size_t ring_set(typename Config::PagemapEntry::SlabMetadata* meta) @@ -191,7 +190,7 @@ namespace snmalloc template struct RemoteDeallocCache { - std::array, REMOTE_SLOTS> list; + stl::Array, REMOTE_SLOTS> list; RemoteDeallocCacheBatchingImpl batching; diff --git a/src/snmalloc/override/new.cc b/src/snmalloc/override/new.cc index 19aa9f5..5b383e5 100644 --- a/src/snmalloc/override/new.cc +++ b/src/snmalloc/override/new.cc @@ -16,6 +16,8 @@ # endif #endif +// only define these if we are not using the vendored STL +#ifndef SNMALLOC_USE_SELF_VENDORED_STL void* operator new(size_t size) { return snmalloc::libc::malloc(size); @@ -111,3 +113,4 @@ void operator delete[](void* p, size_t size, std::align_val_t val) EXCEPTSPEC size = snmalloc::aligned_size(size_t(val), size); snmalloc::libc::free_sized(p, size); } +#endif diff --git a/src/snmalloc/pal/pal_concept.h b/src/snmalloc/pal/pal_concept.h index b4b7d87..db45813 100644 --- a/src/snmalloc/pal/pal_concept.h +++ b/src/snmalloc/pal/pal_concept.h @@ -4,8 +4,6 @@ # include "pal_consts.h" # include "pal_ds.h" -# include - namespace snmalloc { /* diff --git a/src/snmalloc/pal/pal_linux.h b/src/snmalloc/pal/pal_linux.h index c65b765..ffa16eb 100644 --- a/src/snmalloc/pal/pal_linux.h +++ b/src/snmalloc/pal/pal_linux.h @@ -3,6 +3,7 @@ #if defined(__linux__) # include "../ds_core/ds_core.h" # include "pal_posix.h" +# include "snmalloc/stl/array.h" # include # include @@ -183,8 +184,8 @@ namespace snmalloc // protected routine. if (false == syscall_not_working.load(stl::memory_order_relaxed)) { - auto current = std::begin(buffer); - auto target = std::end(buffer); + auto current = stl::begin(buffer); + auto target = stl::end(buffer); while (auto length = target - current) { // Reading data via syscall from system entropy pool. diff --git a/src/snmalloc/pal/pal_posix.h b/src/snmalloc/pal/pal_posix.h index 1b1b3b8..463b710 100644 --- a/src/snmalloc/pal/pal_posix.h +++ b/src/snmalloc/pal/pal_posix.h @@ -5,6 +5,7 @@ #if defined(SNMALLOC_BACKTRACE_HEADER) # include SNMALLOC_BACKTRACE_HEADER #endif +#include "snmalloc/stl/array.h" #include "snmalloc/stl/utility.h" #include @@ -417,8 +418,8 @@ namespace snmalloc auto fd = open("/dev/urandom", flags, 0); if (fd > 0) { - auto current = std::begin(buffer); - auto target = std::end(buffer); + auto current = stl::begin(buffer); + auto target = stl::end(buffer); while (auto length = static_cast(target - current)) { ret = read(fd, current, length); diff --git a/src/snmalloc/stl/array.h b/src/snmalloc/stl/array.h new file mode 100644 index 0000000..8286fda --- /dev/null +++ b/src/snmalloc/stl/array.h @@ -0,0 +1,7 @@ +#pragma once + +#ifdef SNMALLOC_USE_SELF_VENDORED_STL +# include "snmalloc/stl/gnu/array.h" +#else +# include "snmalloc/stl/cxx/array.h" +#endif diff --git a/src/snmalloc/stl/atomic.h b/src/snmalloc/stl/atomic.h index a28def4..45dc18a 100644 --- a/src/snmalloc/stl/atomic.h +++ b/src/snmalloc/stl/atomic.h @@ -1,8 +1,6 @@ #pragma once -#include "snmalloc/stl/common.h" - -#if SNMALLOC_USE_SELF_VENDORED_STL +#ifdef SNMALLOC_USE_SELF_VENDORED_STL # include "snmalloc/stl/gnu/atomic.h" #else # include "snmalloc/stl/cxx/atomic.h" diff --git a/src/snmalloc/stl/common.h b/src/snmalloc/stl/common.h deleted file mode 100644 index 9b47e32..0000000 --- a/src/snmalloc/stl/common.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "snmalloc/ds_core/defines.h" - -// Default on using the system STL. -#ifndef SNMALLOC_USE_SELF_VENDORED_STL -# define SNMALLOC_USE_SELF_VENDORED_STL 0 -#endif - -// Check that the vendored STL is only used with GNU/Clang extensions. -#if SNMALLOC_USE_SELF_VENDORED_STL -# if !defined(__GNUC__) && !defined(__clang__) -# error "cannot use vendored STL without GNU/Clang extensions" -# endif -#endif diff --git a/src/snmalloc/stl/cxx/array.h b/src/snmalloc/stl/cxx/array.h new file mode 100644 index 0000000..f4dac24 --- /dev/null +++ b/src/snmalloc/stl/cxx/array.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace snmalloc +{ + namespace stl + { + template + using Array = std::array; + + using std::begin; + using std::end; + } // namespace stl +} // namespace snmalloc diff --git a/src/snmalloc/stl/cxx/utility.h b/src/snmalloc/stl/cxx/utility.h index 0853239..93ee5df 100644 --- a/src/snmalloc/stl/cxx/utility.h +++ b/src/snmalloc/stl/cxx/utility.h @@ -5,6 +5,7 @@ namespace snmalloc namespace stl { using std::declval; + using std::exchange; using std::forward; using std::move; template diff --git a/src/snmalloc/stl/gnu/array.h b/src/snmalloc/stl/gnu/array.h new file mode 100644 index 0000000..fd176fc --- /dev/null +++ b/src/snmalloc/stl/gnu/array.h @@ -0,0 +1,117 @@ +#pragma once + +#include "snmalloc/ds_core/defines.h" + +#include + +namespace snmalloc +{ + namespace stl + { + template + struct Array + { + // N is potentially 0, so we mark it with __extension__ attribute. + // Expose this to public to allow aggregate initialization + __extension__ T _storage[N]; + + [[nodiscard]] constexpr SNMALLOC_FAST_PATH size_t size() const + { + return N; + } + + constexpr T& operator[](size_t i) + { + return _storage[i]; + } + + constexpr const T& operator[](size_t i) const + { + return _storage[i]; + } + + using value_type = T; + using size_type = size_t; + using iterator = T*; + using const_iterator = const T*; + + [[nodiscard]] constexpr SNMALLOC_FAST_PATH iterator begin() + { + return &_storage[0]; + } + + [[nodiscard]] constexpr SNMALLOC_FAST_PATH const_iterator begin() const + { + return &_storage[0]; + } + + [[nodiscard]] constexpr SNMALLOC_FAST_PATH iterator end() + { + return &_storage[N]; + } + + [[nodiscard]] constexpr SNMALLOC_FAST_PATH const_iterator end() const + { + return &_storage[N]; + } + + [[nodiscard]] constexpr SNMALLOC_FAST_PATH T* data() + { + return &_storage[0]; + } + + [[nodiscard]] constexpr SNMALLOC_FAST_PATH const T* data() const + { + return &_storage[0]; + } + }; + + template + constexpr T* begin(Array& a) + { + return a.begin(); + } + + template + constexpr T* end(Array& a) + { + return a.end(); + } + + template + constexpr const T* begin(const Array& a) + { + return a.begin(); + } + + template + constexpr const T* end(const Array& a) + { + return a.end(); + } + + template + constexpr T* begin(T (&a)[N]) + { + return &a[0]; + } + + template + constexpr T* end(T (&a)[N]) + { + return &a[N]; + } + + template + constexpr const T* begin(const T (&a)[N]) + { + return &a[0]; + } + + template + constexpr const T* end(const T (&a)[N]) + { + return &a[N]; + } + } // namespace stl +} // namespace snmalloc diff --git a/src/snmalloc/stl/gnu/atomic.h b/src/snmalloc/stl/gnu/atomic.h index 28a2170..7e0e342 100644 --- a/src/snmalloc/stl/gnu/atomic.h +++ b/src/snmalloc/stl/gnu/atomic.h @@ -1,5 +1,6 @@ #pragma once +#include "snmalloc/ds_core/defines.h" #include "snmalloc/stl/type_traits.h" #include diff --git a/src/snmalloc/stl/gnu/utility.h b/src/snmalloc/stl/gnu/utility.h index 1936130..346b32c 100644 --- a/src/snmalloc/stl/gnu/utility.h +++ b/src/snmalloc/stl/gnu/utility.h @@ -1,5 +1,6 @@ #pragma once +#include "snmalloc/ds_core/defines.h" #include "snmalloc/stl/type_traits.h" // This is used by clang to provide better analysis of lifetimes. @@ -64,5 +65,13 @@ namespace snmalloc T1 first; T2 second; }; + + template + constexpr SNMALLOC_FAST_PATH A exchange(A& obj, B&& new_value) + { + A old_value = move(obj); + obj = forward(new_value); + return old_value; + } } // namespace stl } // namespace snmalloc diff --git a/src/snmalloc/stl/type_traits.h b/src/snmalloc/stl/type_traits.h index 7171221..002b173 100644 --- a/src/snmalloc/stl/type_traits.h +++ b/src/snmalloc/stl/type_traits.h @@ -1,8 +1,6 @@ #pragma once -#include "snmalloc/stl/common.h" - -#if SNMALLOC_USE_SELF_VENDORED_STL +#ifdef SNMALLOC_USE_SELF_VENDORED_STL # include "snmalloc/stl/gnu/type_traits.h" #else # include "snmalloc/stl/cxx/type_traits.h" diff --git a/src/snmalloc/stl/utility.h b/src/snmalloc/stl/utility.h index 78e3e6a..19e1a7c 100644 --- a/src/snmalloc/stl/utility.h +++ b/src/snmalloc/stl/utility.h @@ -1,8 +1,6 @@ #pragma once -#include "snmalloc/stl/common.h" - -#if SNMALLOC_USE_SELF_VENDORED_STL +#ifdef SNMALLOC_USE_SELF_VENDORED_STL # include "snmalloc/stl/gnu/utility.h" #else # include "snmalloc/stl/cxx/utility.h" diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc index f435b45..16efde3 100644 --- a/src/test/func/memcpy/func-memcpy.cc +++ b/src/test/func/memcpy/func-memcpy.cc @@ -30,6 +30,7 @@ int main() # include # include +# include # include # include # include diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index c85659e..6b52e89 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -5,6 +6,7 @@ #include #include #include + #if ((defined(__linux__) && !defined(__ANDROID__)) || defined(__sun)) || \ defined(__OpenBSD__) && !defined(SNMALLOC_QEMU_WORKAROUND) /* diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index 686c08d..ceb464c 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -2,6 +2,7 @@ # undef SNMALLOC_USE_PTHREAD_DESTRUCTORS #endif +#include #include #include