diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 27a528c..6a0ba72 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -287,7 +287,7 @@ jobs: git diff --exit-code - name: Run clang-tidy run: | - clang-tidy-9 src/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 + clang-tidy-9 src/snmalloc/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 if [ -f tidy.fail ] ; then cat tidy.fail exit 1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 339f4f6..db1e5c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,10 +12,10 @@ include(CMakeDependentOption) option(SNMALLOC_HEADER_ONLY_LIBRARY "Use snmalloc has a header-only library" OFF) # Options that apply globally -option(USE_SNMALLOC_STATS "Track allocation stats" OFF) option(SNMALLOC_CI_BUILD "Disable features not sensible for CI" OFF) option(SNMALLOC_QEMU_WORKAROUND "Disable using madvise(DONT_NEED) to zero memory on Linux" Off) option(SNMALLOC_USE_CXX17 "Build as C++17 for legacy support." OFF) +option(SNMALLOC_TRACING "Enable large quantities of debug output." OFF) option(SNMALLOC_NO_REALLOCARRAY "Build without reallocarray exported" ON) option(SNMALLOC_NO_REALLOCARR "Build without reallocarr exported" ON) # Options that apply only if we're not building the header-only library @@ -198,8 +198,8 @@ function(add_as_define FLAG) target_compile_definitions(snmalloc INTERFACE $<$:${FLAG}>) endfunction() -add_as_define(USE_SNMALLOC_STATS) add_as_define(SNMALLOC_QEMU_WORKAROUND) +add_as_define(SNMALLOC_TRACING) add_as_define(SNMALLOC_CI_BUILD) add_as_define(SNMALLOC_PLATFORM_HAS_GETENTROPY) if (SNMALLOC_NO_REALLOCARRAY) @@ -328,9 +328,9 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) endfunction() - set(SHIM_FILES src/override/new.cc) + set(SHIM_FILES src/snmalloc/override/new.cc) if (SNMALLOC_MEMCPY_BOUNDS) - list(APPEND SHIM_FILES src/override/memcpy.cc) + list(APPEND SHIM_FILES src/snmalloc/override/memcpy.cc) endif () if (SNMALLOC_STATIC_LIBRARY) @@ -346,8 +346,8 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) endif() if(SNMALLOC_RUST_SUPPORT) - add_shim(snmallocshim-rust STATIC src/override/rust.cc) - add_shim(snmallocshim-checks-rust STATIC src/override/rust.cc) + add_shim(snmallocshim-rust STATIC src/snmalloc/override/rust.cc) + add_shim(snmallocshim-checks-rust STATIC src/snmalloc/override/rust.cc) target_compile_definitions(snmallocshim-checks-rust PRIVATE SNMALLOC_CHECK_CLIENT) endif() @@ -426,9 +426,6 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) set_tests_properties(${TESTNAME} PROPERTIES PROCESSORS 4) endif() endif() - # if (${TEST_CATEGORY} MATCHES "func") - # target_compile_definitions(${TESTNAME} PRIVATE -DUSE_SNMALLOC_STATS) - # endif () endforeach() endforeach() endforeach() @@ -441,12 +438,12 @@ install(TARGETS snmalloc EXPORT snmallocConfig) install(TARGETS EXPORT snmallocConfig DESTINATION ${CMAKE_INSTALL_LIBDIR} PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/snmalloc) -install(DIRECTORY src/aal DESTINATION include/snmalloc) -install(DIRECTORY src/ds DESTINATION include/snmalloc) -install(DIRECTORY src/override DESTINATION include/snmalloc) -install(DIRECTORY src/backend DESTINATION include/snmalloc) -install(DIRECTORY src/mem DESTINATION include/snmalloc) -install(DIRECTORY src/pal DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/aal DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/ds DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/override DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/backend DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/mem DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/pal DESTINATION include/snmalloc) install(FILES src/test/measuretime.h src/test/opt.h @@ -455,7 +452,7 @@ install(FILES src/test/xoroshiro.h DESTINATION include/snmalloc/test ) -install(FILES src/snmalloc.h;src/snmalloc_core.h;src/snmalloc_front.h DESTINATION include/snmalloc) +install(FILES src/snmalloc/snmalloc.h;src/snmalloc/snmalloc_core.h;src/snmalloc/snmalloc_front.h DESTINATION include/snmalloc) install(EXPORT snmallocConfig FILE snmalloc-config.cmake diff --git a/src/ds/csv.h b/src/ds/csv.h deleted file mode 100644 index 63419ef..0000000 --- a/src/ds/csv.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once - -#include -#include - -namespace snmalloc -{ - class CSVStream - { - private: - std::ostream* out; - bool first = true; - - public: - class Endl - {}; - - Endl endl; - - CSVStream(std::ostream* o) : out(o) {} - - void preprint() - { - if (!first) - { - *out << ", "; - } - else - { - first = false; - } - } - - CSVStream& operator<<(const std::string& str) - { - preprint(); - *out << str; - return *this; - } - - CSVStream& operator<<(uint64_t u) - { - preprint(); - *out << u; - return *this; - } - - CSVStream& operator<<(Endl) - { - *out << std::endl; - first = true; - return *this; - } - }; -} // namespace snmalloc \ No newline at end of file diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h deleted file mode 100644 index 2d3ae45..0000000 --- a/src/mem/allocstats.h +++ /dev/null @@ -1,428 +0,0 @@ -#pragma once - -#include "../ds/bits.h" -#include "../mem/sizeclasstable.h" - -#include - -#ifdef USE_SNMALLOC_STATS -# include "../ds/csv.h" - -# include -# include -#endif - -namespace snmalloc -{ - template - struct AllocStats - { - constexpr AllocStats() = default; - - struct CurrentMaxPair - { - size_t current{0}; - size_t max{0}; - size_t used{0}; - - void inc() - { - current++; - used++; - if (current > max) - max++; - } - - void dec() - { - // Split stats means this is not true. - // TODO reestablish checks, when we sanitise the stats. - // SNMALLOC_ASSERT(current > 0); - current--; - } - - bool is_empty() - { - return current == 0; - } - - bool is_unused() - { - return max == 0; - } - - void add(CurrentMaxPair& that) - { - current += that.current; - max += that.max; - used += that.used; - } -#ifdef USE_SNMALLOC_STATS - void print(CSVStream& csv, size_t multiplier = 1) - { - csv << current * multiplier << max * multiplier << used * multiplier; - } -#endif - }; - - struct Stats - { - constexpr Stats() = default; - - CurrentMaxPair count; - CurrentMaxPair slab_count; - uint64_t time{0}; - uint64_t ticks{0}; - double online_average{0}; - - bool is_empty() - { - return count.is_empty(); - } - - void add(Stats& that) - { - count.add(that.count); - slab_count.add(that.slab_count); - } - - void addToRunningAverage() - { - uint64_t now = Aal::tick(); - - if (slab_count.current != 0) - { - double occupancy = static_cast(count.current) / - static_cast(slab_count.current); - uint64_t duration = now - time; - - if (ticks == 0) - online_average = occupancy; - else - online_average += - ((occupancy - online_average) * static_cast(duration)) / - static_cast(ticks); - - ticks += duration; - } - - time = now; - } - -#ifdef USE_SNMALLOC_STATS - void - print(CSVStream& csv, size_t multiplier = 1, size_t slab_multiplier = 1) - { - // Keep in sync with header lower down - count.print(csv, multiplier); - slab_count.print(csv, slab_multiplier); - size_t average = - static_cast(online_average * static_cast(multiplier)); - - csv << average << (slab_multiplier - average) * slab_count.max - << csv.endl; - } -#endif - }; - -#ifdef USE_SNMALLOC_STATS - static constexpr size_t BUCKETS_BITS = 4; - static constexpr size_t BUCKETS = 1 << BUCKETS_BITS; - static constexpr size_t TOTAL_BUCKETS = - bits::to_exp_mant_const( - bits::one_at_bit(bits::ADDRESS_BITS - 1)); - - Stats sizeclass[N]; - - size_t large_pop_count[LARGE_N] = {0}; - size_t large_push_count[LARGE_N] = {0}; - - size_t remote_freed = 0; - size_t remote_posted = 0; - size_t remote_received = 0; - size_t superslab_push_count = 0; - size_t superslab_pop_count = 0; - size_t superslab_fresh_count = 0; - size_t segment_count = 0; - size_t bucketed_requests[TOTAL_BUCKETS] = {}; -#endif - - void alloc_request(size_t size) - { - UNUSED(size); - -#ifdef USE_SNMALLOC_STATS - auto index = (size == 0) ? 0 : bits::to_exp_mant(size); - SNMALLOC_ASSERT(index < TOTAL_BUCKETS); - bucketed_requests[index]++; -#endif - } - - bool is_empty() - { -#ifdef USE_SNMALLOC_STATS - for (size_t i = 0; i < N; i++) - { - if (!sizeclass[i].is_empty()) - return false; - } - - for (size_t i = 0; i < LARGE_N; i++) - { - if (large_push_count[i] != large_pop_count[i]) - return false; - } - - return (remote_freed == remote_posted); -#else - return true; -#endif - } - - void sizeclass_alloc(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - sizeclass[sc].addToRunningAverage(); - sizeclass[sc].count.inc(); -#endif - } - - void sizeclass_dealloc(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - sizeclass[sc].addToRunningAverage(); - sizeclass[sc].count.dec(); -#endif - } - - void large_alloc(size_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - SNMALLOC_ASSUME(sc < LARGE_N); - large_pop_count[sc]++; -#endif - } - - void sizeclass_alloc_slab(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - sizeclass[sc].addToRunningAverage(); - sizeclass[sc].slab_count.inc(); -#endif - } - - void sizeclass_dealloc_slab(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - sizeclass[sc].addToRunningAverage(); - sizeclass[sc].slab_count.dec(); -#endif - } - - void large_dealloc(size_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - large_push_count[sc]++; -#endif - } - - void segment_create() - { -#ifdef USE_SNMALLOC_STATS - segment_count++; -#endif - } - - void superslab_pop() - { -#ifdef USE_SNMALLOC_STATS - superslab_pop_count++; -#endif - } - - void superslab_push() - { -#ifdef USE_SNMALLOC_STATS - superslab_push_count++; -#endif - } - - void superslab_fresh() - { -#ifdef USE_SNMALLOC_STATS - superslab_fresh_count++; -#endif - } - - void remote_free(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - remote_freed += sizeclass_to_size(sc); -#endif - } - - void remote_post() - { -#ifdef USE_SNMALLOC_STATS - remote_posted = remote_freed; -#endif - } - - void remote_receive(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - remote_received += sizeclass_to_size(sc); -#endif - } - - void add(AllocStats& that) - { - UNUSED(that); - -#ifdef USE_SNMALLOC_STATS - for (size_t i = 0; i < N; i++) - sizeclass[i].add(that.sizeclass[i]); - - for (size_t i = 0; i < LARGE_N; i++) - { - large_push_count[i] += that.large_push_count[i]; - large_pop_count[i] += that.large_pop_count[i]; - } - - for (size_t i = 0; i < TOTAL_BUCKETS; i++) - bucketed_requests[i] += that.bucketed_requests[i]; - - remote_freed += that.remote_freed; - remote_posted += that.remote_posted; - remote_received += that.remote_received; - superslab_pop_count += that.superslab_pop_count; - superslab_push_count += that.superslab_push_count; - superslab_fresh_count += that.superslab_fresh_count; - segment_count += that.segment_count; -#endif - } - -#ifdef USE_SNMALLOC_STATS - template - void print(std::ostream& o, uint64_t dumpid = 0, uint64_t allocatorid = 0) - { - UNUSED(o, dumpid, allocatorid); - - CSVStream csv(&o); - - if (dumpid == 0) - { - // Output headers for initial dump - // Keep in sync with data dump - csv << "GlobalStats" - << "DumpID" - << "AllocatorID" - << "Remote freed" - << "Remote posted" - << "Remote received" - << "Superslab pop" - << "Superslab push" - << "Superslab fresh" - << "Segments" << csv.endl; - - csv << "BucketedStats" - << "DumpID" - << "AllocatorID" - << "Size group" - << "Size" - << "Current count" - << "Max count" - << "Total Allocs" - << "Current Slab bytes" - << "Max Slab bytes" - << "Total slab allocs" - << "Average Slab Usage" - << "Average wasted space" << csv.endl; - - csv << "LargeBucketedStats" - << "DumpID" - << "AllocatorID" - << "Size group" - << "Size" - << "Push count" - << "Pop count" << csv.endl; - - csv << "AllocSizes" - << "DumpID" - << "AllocatorID" - << "ClassID" - << "Low size" - << "High size" - << "Count" << csv.endl; - } - - for (smallsizeclass_t i = 0; i < N; i++) - { - if (sizeclass[i].count.is_unused()) - continue; - - sizeclass[i].addToRunningAverage(); - - csv << "BucketedStats" << dumpid << allocatorid << i - << sizeclass_to_size(i); - - sizeclass[i].print(csv, sizeclass_to_size(i)); - } - - // for (uint8_t i = 0; i < LARGE_N; i++) - // { - // if ((large_push_count[i] == 0) && (large_pop_count[i] == 0)) - // continue; - - // csv << "LargeBucketedStats" << dumpid << allocatorid << (i + N) - // << large_sizeclass_to_size(i) << large_push_count[i] - // << large_pop_count[i] << csv.endl; - // } - - size_t low = 0; - size_t high = 0; - - for (size_t i = 0; i < TOTAL_BUCKETS; i++) - { - low = high + 1; - high = bits::from_exp_mant(i); - - if (bucketed_requests[i] == 0) - continue; - - csv << "AllocSizes" << dumpid << allocatorid << i << low << high - << bucketed_requests[i] << csv.endl; - } - - csv << "GlobalStats" << dumpid << allocatorid << remote_freed - << remote_posted << remote_received << superslab_pop_count - << superslab_push_count << superslab_fresh_count << segment_count - << csv.endl; - } -#endif - - void start() - { -#ifdef USE_SNMALLOC_STATS - for (size_t i = 0; i < N; i++) - sizeclass[i].time = Aal::tick(); -#endif - } - }; -} // namespace snmalloc diff --git a/src/override/override.h b/src/override/override.h deleted file mode 100644 index a9aedbe..0000000 --- a/src/override/override.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -// Core implementation of snmalloc independent of the configuration mode -#include "../snmalloc_core.h" - -#ifndef SNMALLOC_PROVIDE_OWN_CONFIG -# include "../backend/globalconfig.h" -// The default configuration for snmalloc is used if alternative not defined -namespace snmalloc -{ - using Alloc = snmalloc::LocalAllocator; -} // namespace snmalloc -#endif - -// User facing API surface, needs to know what `Alloc` is. -#include "../snmalloc_front.h" - -#ifndef SNMALLOC_EXPORT -# define SNMALLOC_EXPORT -#endif -#ifdef SNMALLOC_STATIC_LIBRARY_PREFIX -# define __SN_CONCAT(a, b) a##b -# define __SN_EVALUATE(a, b) __SN_CONCAT(a, b) -# define SNMALLOC_NAME_MANGLE(a) \ - __SN_EVALUATE(SNMALLOC_STATIC_LIBRARY_PREFIX, a) -#elif !defined(SNMALLOC_NAME_MANGLE) -# define SNMALLOC_NAME_MANGLE(a) a -#endif diff --git a/src/snmalloc.h b/src/snmalloc.h deleted file mode 100644 index e1d90d4..0000000 --- a/src/snmalloc.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -// Core implementation of snmalloc independent of the configuration mode -#include "snmalloc_core.h" - -// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own -// definition of `snmalloc::Alloc` and include `snmalloc_front.h` before -// including any files that include `snmalloc.h` and consume the global -// allocation APIs. -#ifndef SNMALLOC_PROVIDE_OWN_CONFIG -// Default implementation of global state -# include "backend/globalconfig.h" - -// The default configuration for snmalloc -namespace snmalloc -{ - using Alloc = snmalloc::LocalAllocator; -} - -// User facing API surface, needs to know what `Alloc` is. -# include "snmalloc_front.h" -#endif diff --git a/src/snmalloc/README.md b/src/snmalloc/README.md new file mode 100644 index 0000000..0366a5e --- /dev/null +++ b/src/snmalloc/README.md @@ -0,0 +1,40 @@ +Include hierarchy +----------------- + +The `snmalloc/` include path contains all of the snmalloc headers. +These are arranged in a hierarchy such that each of the directories may include ones below, in the following order, starting at the bottom: + + - `ds_core/` provides core data structures that depend on the C++ implementation and nothing else. + This directory includes a number of things that abstract over different language extensions (for example, different built-in function names in different compilers). + - `aal/` provides the architecture abstraction layer (AAL). + This layer provides abstractions over CPU-specific intrinsics and defines things such as the virtual address-space size. + There is a single AAL for an snmalloc instantiation. + - `pal/` provides the platform abstraction layer (PAL). + This exposes OS- or environment-specific abstractions into the rest of the code. + An snmalloc instantiation may use more than one PAL, including ones provided by the user. + - `ds/` includes data structures that may depend on platform services or on features specific to the current CPU. + - `mem/` provides the core allocator abstractions. + The code here is templated over a back-end, which defines a particular embedding of snmalloc. + - `backend_helpers/` provides helper classes for use in defining a back end. + This includes data structures such as pagemap implementations (efficient maps from a chunk address to associated metadata) and buddy allocators for managing address-space ranges. + - `backend/` provides some example implementations for snmalloc embeddings that provide a global memory allocator for an address space. + Users may ignore this entirely and use the types in `mem/` with a custom back end to expose an snmalloc instance with specific behaviour. + Layers above this can be used with a custom configuration by defining `SNMALLOC_PROVIDE_OWN_CONFIG` and exporting a type as `snmalloc::Alloc` that defines the type of an `snmalloc::LocalAllocator` template specialisation. + - `global/` provides some front-end components that assume that snmalloc is available in a global configuration. + - `override/` builds on top of `global/` to provide specific implementations with compatibility with external specifications (for example C `malloc`, C++ `operator new`, jemalloc's `*allocx`, or Rust's `std::alloc`). + +Each layer until `backend_helpers/` provides a single header with the same name as the directory. +Files in higher layers should depend only on the single-file version. +This allows specific files to be moved to a lower layer if appropriate, without too much code churn. + +There is only one exception to this rule: `backend/globalconfig.h`. +This file defines either the default configuration *or* nothing, depending on whether the user has defined `SNMALLOC_PROVIDE_OWN_CONFIG`. +The layers above the back end should include only this file, so that there is a single interception point for externally defined back ends. + +External code should include only the following files: + + - `snmalloc/snmalloc_core.h` includes everything up to `backend_helpers`. + This provides the building blocks required to assemble an snmalloc instance, but does not assume any global configuration. + - `snmalloc/snmalloc_front.h` assumes a global configuration (either user-provided or the default from `snmalloc/backend/globalconfig.h` and exposes all of the functionality that depends on both. + - `snmalloc/snmalloc.h` is a convenience wrapper that includes both of the above files. + - `snmalloc/override/*.cc` can be compiled as-is or included after `snmalloc/snmalloc_core.h` and a custom global allocator definition to provide specific languages' global memory allocator APIs with a custom snmalloc embedding. diff --git a/src/aal/aal.h b/src/snmalloc/aal/aal.h similarity index 95% rename from src/aal/aal.h rename to src/snmalloc/aal/aal.h index fcff4dc..413c284 100644 --- a/src/aal/aal.h +++ b/src/snmalloc/aal/aal.h @@ -1,7 +1,12 @@ +/** + * The snmalloc architecture abstraction layer. This defines + * CPU-architecture-specific functionality. + * + * Files in this directory may depend on `ds_core` and each other, but nothing + * else in snmalloc. + */ #pragma once -#include "../ds/concept.h" -#include "../ds/defines.h" -#include "../ds/ptrwrap.h" +#include "../ds_core/ds_core.h" #include "aal_concept.h" #include "aal_consts.h" @@ -242,10 +247,6 @@ namespace snmalloc constexpr static bool aal_supports = (AAL::aal_features & F) == F; } // namespace snmalloc -#if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_32) -# include -#endif - #ifdef __POINTER_WIDTH__ # if ((__POINTER_WIDTH__ == 64) && !defined(SNMALLOC_VA_BITS_64)) || \ ((__POINTER_WIDTH__ == 32) && !defined(SNMALLOC_VA_BITS_32)) @@ -262,3 +263,7 @@ static_assert(sizeof(size_t) == 4); #elif defined(SNMALLOC_VA_BITS_64) static_assert(sizeof(size_t) == 8); #endif + +// Included after the AAL has been defined, depends on the AAL's notion of an +// address +#include "address.h" diff --git a/src/aal/aal_arm.h b/src/snmalloc/aal/aal_arm.h similarity index 100% rename from src/aal/aal_arm.h rename to src/snmalloc/aal/aal_arm.h diff --git a/src/aal/aal_cheri.h b/src/snmalloc/aal/aal_cheri.h similarity index 98% rename from src/aal/aal_cheri.h rename to src/snmalloc/aal/aal_cheri.h index 1a7906e..4774dde 100644 --- a/src/aal/aal_cheri.h +++ b/src/snmalloc/aal/aal_cheri.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/defines.h" +#include "../ds_core/ds_core.h" #include diff --git a/src/aal/aal_concept.h b/src/snmalloc/aal/aal_concept.h similarity index 68% rename from src/aal/aal_concept.h rename to src/snmalloc/aal/aal_concept.h index 15eed19..6269d96 100644 --- a/src/aal/aal_concept.h +++ b/src/snmalloc/aal/aal_concept.h @@ -1,8 +1,7 @@ #pragma once #ifdef __cpp_concepts -# include "../ds/concept.h" -# include "../ds/ptrwrap.h" +# include "../ds_core/ds_core.h" # include "aal_consts.h" # include @@ -27,9 +26,12 @@ namespace snmalloc * AALs provide a prefetch operation. */ template - concept ConceptAAL_prefetch = requires(void *ptr) + concept ConceptAAL_prefetch = requires(void* ptr) { - { AAL::prefetch(ptr) } noexcept -> ConceptSame; + { + AAL::prefetch(ptr) + } + noexcept->ConceptSame; }; /** @@ -38,29 +40,31 @@ namespace snmalloc template concept ConceptAAL_tick = requires() { - { AAL::tick() } noexcept -> ConceptSame; + { + AAL::tick() + } + noexcept->ConceptSame; }; template concept ConceptAAL_capptr_methods = - requires(capptr::Chunk auth, capptr::AllocFull ret, size_t sz) + requires(capptr::Chunk auth, capptr::AllocFull ret, size_t sz) { /** * Produce a pointer with reduced authority from a more privilged pointer. * The resulting pointer will have base at auth's address and length of * exactly sz. auth+sz must not exceed auth's limit. */ - { AAL::template capptr_bound(auth, sz) } - noexcept - -> ConceptSame>; + { + AAL::template capptr_bound(auth, sz) + } + noexcept->ConceptSame>; }; template concept ConceptAAL = - ConceptAAL_static_members && - ConceptAAL_prefetch && - ConceptAAL_tick && - ConceptAAL_capptr_methods; + ConceptAAL_static_members&& ConceptAAL_prefetch&& + ConceptAAL_tick&& ConceptAAL_capptr_methods; } // namespace snmalloc #endif diff --git a/src/aal/aal_consts.h b/src/snmalloc/aal/aal_consts.h similarity index 100% rename from src/aal/aal_consts.h rename to src/snmalloc/aal/aal_consts.h diff --git a/src/aal/aal_powerpc.h b/src/snmalloc/aal/aal_powerpc.h similarity index 100% rename from src/aal/aal_powerpc.h rename to src/snmalloc/aal/aal_powerpc.h diff --git a/src/aal/aal_riscv.h b/src/snmalloc/aal/aal_riscv.h similarity index 100% rename from src/aal/aal_riscv.h rename to src/snmalloc/aal/aal_riscv.h diff --git a/src/aal/aal_sparc.h b/src/snmalloc/aal/aal_sparc.h similarity index 100% rename from src/aal/aal_sparc.h rename to src/snmalloc/aal/aal_sparc.h diff --git a/src/aal/aal_x86.h b/src/snmalloc/aal/aal_x86.h similarity index 100% rename from src/aal/aal_x86.h rename to src/snmalloc/aal/aal_x86.h diff --git a/src/aal/aal_x86_sgx.h b/src/snmalloc/aal/aal_x86_sgx.h similarity index 100% rename from src/aal/aal_x86_sgx.h rename to src/snmalloc/aal/aal_x86_sgx.h diff --git a/src/ds/address.h b/src/snmalloc/aal/address.h similarity index 99% rename from src/ds/address.h rename to src/snmalloc/aal/address.h index a6a161a..1f528f9 100644 --- a/src/ds/address.h +++ b/src/snmalloc/aal/address.h @@ -1,7 +1,5 @@ #pragma once -#include "../pal/pal_consts.h" -#include "bits.h" -#include "ptrwrap.h" +#include "../ds_core/ds_core.h" #include diff --git a/src/backend/backend.h b/src/snmalloc/backend/backend.h similarity index 96% rename from src/backend/backend.h rename to src/snmalloc/backend/backend.h index 6d33827..e791fe2 100644 --- a/src/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -1,19 +1,5 @@ #pragma once -#include "../mem/allocconfig.h" -#include "../mem/metadata.h" -#include "../pal/pal.h" -#include "commitrange.h" -#include "commonconfig.h" -#include "empty_range.h" -#include "globalrange.h" -#include "largebuddyrange.h" -#include "pagemap.h" -#include "pagemapregisterrange.h" -#include "palrange.h" -#include "range_helpers.h" -#include "smallbuddyrange.h" -#include "statsrange.h" -#include "subrange.h" +#include "../backend_helpers/backend_helpers.h" #if defined(SNMALLOC_CHECK_CLIENT) && !defined(OPEN_ENCLAVE) /** diff --git a/src/backend/fixedglobalconfig.h b/src/snmalloc/backend/fixedglobalconfig.h similarity index 95% rename from src/backend/fixedglobalconfig.h rename to src/snmalloc/backend/fixedglobalconfig.h index 5b9cbf5..f0e8a57 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/snmalloc/backend/fixedglobalconfig.h @@ -1,9 +1,6 @@ #pragma once #include "../backend/backend.h" -#include "../mem/corealloc.h" -#include "../mem/pool.h" -#include "commonconfig.h" namespace snmalloc { diff --git a/src/backend/globalconfig.h b/src/snmalloc/backend/globalconfig.h similarity index 79% rename from src/backend/globalconfig.h rename to src/snmalloc/backend/globalconfig.h index 3117662..6e88f17 100644 --- a/src/backend/globalconfig.h +++ b/src/snmalloc/backend/globalconfig.h @@ -1,19 +1,17 @@ #pragma once +// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own +// definition of `snmalloc::Alloc` before including any files that include +// `snmalloc.h` or consume the global allocation APIs. +#ifndef SNMALLOC_PROVIDE_OWN_CONFIG -#include "../backend/backend.h" -#include "../mem/corealloc.h" -#include "../mem/pool.h" -#include "commonconfig.h" +# include "../backend/backend.h" -#ifdef SNMALLOC_TRACING -# include -#endif namespace snmalloc { // Forward reference to thread local cleanup. void register_clean_up(); -#ifdef USE_SNMALLOC_STATS +# ifdef USE_SNMALLOC_STATS inline static void print_stats() { printf("No Stats yet!"); @@ -21,7 +19,7 @@ namespace snmalloc // current_alloc_pool()->aggregate_stats(s); // s.print(std::cout); } -#endif +# endif /** * The default configuration for a global snmalloc. This allocates memory @@ -59,9 +57,9 @@ namespace snmalloc static void ensure_init() { FlagLock lock{initialisation_lock}; -#ifdef SNMALLOC_TRACING +# ifdef SNMALLOC_TRACING message<1024>("Run init_impl"); -#endif +# endif if (initialised) return; @@ -74,9 +72,9 @@ namespace snmalloc // Need to initialise pagemap. Backend::init(); -#ifdef USE_SNMALLOC_STATS +# ifdef USE_SNMALLOC_STATS atexit(snmalloc::print_stats); -#endif +# endif initialised = true; } @@ -96,3 +94,10 @@ namespace snmalloc } }; } // namespace snmalloc + +// The default configuration for snmalloc +namespace snmalloc +{ + using Alloc = snmalloc::LocalAllocator; +} // namespace snmalloc +#endif diff --git a/src/snmalloc/backend_helpers/backend_helpers.h b/src/snmalloc/backend_helpers/backend_helpers.h new file mode 100644 index 0000000..06f0c92 --- /dev/null +++ b/src/snmalloc/backend_helpers/backend_helpers.h @@ -0,0 +1,14 @@ +#include "../mem/mem.h" +#include "buddy.h" +#include "commitrange.h" +#include "commonconfig.h" +#include "empty_range.h" +#include "globalrange.h" +#include "largebuddyrange.h" +#include "pagemap.h" +#include "pagemapregisterrange.h" +#include "palrange.h" +#include "range_helpers.h" +#include "smallbuddyrange.h" +#include "statsrange.h" +#include "subrange.h" diff --git a/src/backend/buddy.h b/src/snmalloc/backend_helpers/buddy.h similarity index 96% rename from src/backend/buddy.h rename to src/snmalloc/backend_helpers/buddy.h index 7e94c84..1c9b8e4 100644 --- a/src/backend/buddy.h +++ b/src/snmalloc/backend_helpers/buddy.h @@ -1,8 +1,6 @@ #pragma once -#include "../ds/address.h" -#include "../ds/bits.h" -#include "../ds/redblacktree.h" +#include "../ds/ds.h" namespace snmalloc { @@ -119,4 +117,4 @@ namespace snmalloc return bigger; } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/commitrange.h b/src/snmalloc/backend_helpers/commitrange.h similarity index 94% rename from src/backend/commitrange.h rename to src/snmalloc/backend_helpers/commitrange.h index a5b24d9..c9cdb39 100644 --- a/src/backend/commitrange.h +++ b/src/snmalloc/backend_helpers/commitrange.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/ptrwrap.h" -#include "../pal/pal_consts.h" +#include "../pal/pal.h" namespace snmalloc { diff --git a/src/backend/commonconfig.h b/src/snmalloc/backend_helpers/commonconfig.h similarity index 66% rename from src/backend/commonconfig.h rename to src/snmalloc/backend_helpers/commonconfig.h index 66314db..c8e55e1 100644 --- a/src/backend/commonconfig.h +++ b/src/snmalloc/backend_helpers/commonconfig.h @@ -1,8 +1,6 @@ #pragma once -#include "../backend/backend_concept.h" -#include "../ds/defines.h" -#include "../mem/remoteallocator.h" +#include "../mem/mem.h" namespace snmalloc { @@ -108,75 +106,5 @@ namespace snmalloc inline static RemoteAllocator unused_remote; }; - /** - * SFINAE helper. Matched only if `T` implements `is_initialised`. Calls - * it if it exists. - */ - template - SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) - -> decltype(T::is_initialised()) - { - return T::is_initialised(); - } - - /** - * SFINAE helper. Matched only if `T` does not implement `is_initialised`. - * Unconditionally returns true if invoked. - */ - template - SNMALLOC_FAST_PATH auto call_is_initialised(T*, long) - { - return true; - } - - namespace detail - { - /** - * SFINAE helper, calls capptr_domesticate in the backend if it exists. - */ - template< - SNMALLOC_CONCEPT(ConceptBackendDomestication) Backend, - typename T, - SNMALLOC_CONCEPT(capptr::ConceptBound) B> - SNMALLOC_FAST_PATH_INLINE auto - capptr_domesticate(typename Backend::LocalState* ls, CapPtr p, int) - -> decltype(Backend::capptr_domesticate(ls, p)) - { - return Backend::capptr_domesticate(ls, p); - } - - /** - * SFINAE helper. If the back end does not provide special handling for - * domestication then assume all wild pointers can be domesticated. - */ - template< - SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, - typename T, - SNMALLOC_CONCEPT(capptr::ConceptBound) B> - SNMALLOC_FAST_PATH_INLINE auto - capptr_domesticate(typename Backend::LocalState*, CapPtr p, long) - { - return CapPtr< - T, - typename B::template with_wildness>( - p.unsafe_ptr()); - } - } // namespace detail - - /** - * Wrapper that calls `Backend::capptr_domesticate` if and only if it is - * implemented. If it is not implemented then this assumes that any wild - * pointer can be domesticated. - */ - template< - SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, - typename T, - SNMALLOC_CONCEPT(capptr::ConceptBound) B> - SNMALLOC_FAST_PATH_INLINE auto - capptr_domesticate(typename Backend::LocalState* ls, CapPtr p) - { - return detail::capptr_domesticate(ls, p, 0); - } - } // namespace snmalloc #include "../mem/remotecache.h" diff --git a/src/backend/empty_range.h b/src/snmalloc/backend_helpers/empty_range.h similarity index 93% rename from src/backend/empty_range.h rename to src/snmalloc/backend_helpers/empty_range.h index 5287cd6..b5102a1 100644 --- a/src/backend/empty_range.h +++ b/src/snmalloc/backend_helpers/empty_range.h @@ -1,5 +1,5 @@ #pragma once -#include "../ds/ptrwrap.h" +#include "../ds_core/ds_core.h" namespace snmalloc { diff --git a/src/backend/globalrange.h b/src/snmalloc/backend_helpers/globalrange.h similarity index 91% rename from src/backend/globalrange.h rename to src/snmalloc/backend_helpers/globalrange.h index d8ae667..016701e 100644 --- a/src/backend/globalrange.h +++ b/src/snmalloc/backend_helpers/globalrange.h @@ -1,8 +1,6 @@ #pragma once -#include "../ds/defines.h" -#include "../ds/helpers.h" -#include "../ds/ptrwrap.h" +#include "../ds/ds.h" namespace snmalloc { @@ -53,4 +51,4 @@ namespace snmalloc parent->dealloc_range(base, size); } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/largebuddyrange.h b/src/snmalloc/backend_helpers/largebuddyrange.h similarity index 98% rename from src/backend/largebuddyrange.h rename to src/snmalloc/backend_helpers/largebuddyrange.h index a57ed69..e3b5ecc 100644 --- a/src/backend/largebuddyrange.h +++ b/src/snmalloc/backend_helpers/largebuddyrange.h @@ -1,10 +1,7 @@ #pragma once -#include "../ds/address.h" -#include "../ds/bits.h" -#include "../mem/allocconfig.h" -#include "../pal/pal.h" -#include "backend_concept.h" +#include "../ds/ds.h" +#include "../mem/mem.h" #include "buddy.h" #include "range_helpers.h" diff --git a/src/backend/pagemap.h b/src/snmalloc/backend_helpers/pagemap.h similarity index 98% rename from src/backend/pagemap.h rename to src/snmalloc/backend_helpers/pagemap.h index 6bb259d..ec7bbdf 100644 --- a/src/backend/pagemap.h +++ b/src/snmalloc/backend_helpers/pagemap.h @@ -1,9 +1,7 @@ #pragma once -#include "../ds/bits.h" -#include "../ds/helpers.h" -#include "../mem/entropy.h" -#include "../pal/pal.h" +#include "../ds/ds.h" +#include "../mem/mem.h" #include #include diff --git a/src/backend/pagemapregisterrange.h b/src/snmalloc/backend_helpers/pagemapregisterrange.h similarity index 94% rename from src/backend/pagemapregisterrange.h rename to src/snmalloc/backend_helpers/pagemapregisterrange.h index 4084eed..059cc15 100644 --- a/src/backend/pagemapregisterrange.h +++ b/src/snmalloc/backend_helpers/pagemapregisterrange.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/address.h" #include "../pal/pal.h" namespace snmalloc @@ -42,4 +41,4 @@ namespace snmalloc return base; } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/palrange.h b/src/snmalloc/backend_helpers/palrange.h similarity index 96% rename from src/backend/palrange.h rename to src/snmalloc/backend_helpers/palrange.h index 5fa3fe6..f447325 100644 --- a/src/backend/palrange.h +++ b/src/snmalloc/backend_helpers/palrange.h @@ -1,5 +1,4 @@ #pragma once -#include "../ds/address.h" #include "../pal/pal.h" namespace snmalloc @@ -61,4 +60,4 @@ namespace snmalloc } } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/range_helpers.h b/src/snmalloc/backend_helpers/range_helpers.h similarity index 95% rename from src/backend/range_helpers.h rename to src/snmalloc/backend_helpers/range_helpers.h index 1dd6ea4..a9dc43c 100644 --- a/src/backend/range_helpers.h +++ b/src/snmalloc/backend_helpers/range_helpers.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/ptrwrap.h" +#include "../ds_core/ds_core.h" namespace snmalloc { @@ -34,4 +34,4 @@ namespace snmalloc length -= align; } } -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/smallbuddyrange.h b/src/snmalloc/backend_helpers/smallbuddyrange.h similarity index 99% rename from src/backend/smallbuddyrange.h rename to src/snmalloc/backend_helpers/smallbuddyrange.h index 2500e1f..b3ccc45 100644 --- a/src/backend/smallbuddyrange.h +++ b/src/snmalloc/backend_helpers/smallbuddyrange.h @@ -1,7 +1,5 @@ #pragma once -#include "../ds/address.h" -#include "../mem/allocconfig.h" #include "../pal/pal.h" #include "range_helpers.h" diff --git a/src/backend/statsrange.h b/src/snmalloc/backend_helpers/statsrange.h similarity index 100% rename from src/backend/statsrange.h rename to src/snmalloc/backend_helpers/statsrange.h diff --git a/src/backend/subrange.h b/src/snmalloc/backend_helpers/subrange.h similarity index 96% rename from src/backend/subrange.h rename to src/snmalloc/backend_helpers/subrange.h index dd48243..44c1db9 100644 --- a/src/backend/subrange.h +++ b/src/snmalloc/backend_helpers/subrange.h @@ -1,5 +1,5 @@ #pragma once -#include "../mem/entropy.h" +#include "../mem/mem.h" namespace snmalloc { @@ -54,4 +54,4 @@ namespace snmalloc return pointer_offset(overblock, offset); } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/ds/aba.h b/src/snmalloc/ds/aba.h similarity index 99% rename from src/ds/aba.h rename to src/snmalloc/ds/aba.h index 64b0f99..51c4470 100644 --- a/src/ds/aba.h +++ b/src/snmalloc/ds/aba.h @@ -1,7 +1,6 @@ #pragma once -#include "bits.h" -#include "ptrwrap.h" +#include "../aal/aal.h" /** * This file contains an abstraction of ABA protection. This API should be diff --git a/src/mem/allocconfig.h b/src/snmalloc/ds/allocconfig.h similarity index 98% rename from src/mem/allocconfig.h rename to src/snmalloc/ds/allocconfig.h index a83d2a8..2186b78 100644 --- a/src/mem/allocconfig.h +++ b/src/snmalloc/ds/allocconfig.h @@ -1,8 +1,5 @@ #pragma once -#include "../ds/bits.h" -#include "../pal/pal.h" - namespace snmalloc { // 0 intermediate bits results in power of 2 small allocs. 1 intermediate diff --git a/src/snmalloc/ds/ds.h b/src/snmalloc/ds/ds.h new file mode 100644 index 0000000..432277d --- /dev/null +++ b/src/snmalloc/ds/ds.h @@ -0,0 +1,11 @@ +/** + * Data structures used by snmalloc. + * + */ +#pragma once +#include "../pal/pal.h" +#include "aba.h" +#include "allocconfig.h" +#include "flaglock.h" +#include "mpmcstack.h" +#include "singleton.h" diff --git a/src/ds/flaglock.h b/src/snmalloc/ds/flaglock.h similarity index 98% rename from src/ds/flaglock.h rename to src/snmalloc/ds/flaglock.h index c25f1ee..5fbbf0a 100644 --- a/src/ds/flaglock.h +++ b/src/snmalloc/ds/flaglock.h @@ -1,8 +1,9 @@ #pragma once -#include "bits.h" +#include "../aal/aal.h" #include +#include namespace snmalloc { diff --git a/src/ds/mpmcstack.h b/src/snmalloc/ds/mpmcstack.h similarity index 97% rename from src/ds/mpmcstack.h rename to src/snmalloc/ds/mpmcstack.h index 48af893..cd005e9 100644 --- a/src/ds/mpmcstack.h +++ b/src/snmalloc/ds/mpmcstack.h @@ -1,7 +1,8 @@ #pragma once +#include "../ds_core/ds_core.h" #include "aba.h" -#include "ptrwrap.h" +#include "allocconfig.h" #if defined(__has_feature) # if __has_feature(thread_sanitizer) diff --git a/src/snmalloc/ds/singleton.h b/src/snmalloc/ds/singleton.h new file mode 100644 index 0000000..c85635d --- /dev/null +++ b/src/snmalloc/ds/singleton.h @@ -0,0 +1,51 @@ +#pragma once + +#include "../ds_core/ds_core.h" +#include "flaglock.h" + +#include +#include +#include +#include + +namespace snmalloc +{ + /* + * In some use cases we need to run before any of the C++ runtime has been + * initialised. This singleton class is designed to not depend on the + * runtime. + */ + template + class Singleton + { + inline static FlagWord flag; + inline static std::atomic initialised{false}; + inline static Object obj; + + public: + /** + * If argument is non-null, then it is assigned the value + * true, if this is the first call to get. + * At most one call will be first. + */ + inline SNMALLOC_SLOW_PATH static Object& get(bool* first = nullptr) + { + // If defined should be initially false; + SNMALLOC_ASSERT(first == nullptr || *first == false); + + if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire))) + { + FlagLock lock(flag); + if (!initialised) + { + init(&obj); + initialised.store(true, std::memory_order_release); + if (first != nullptr) + *first = true; + } + } + return obj; + } + }; + +} // namespace snmalloc diff --git a/src/ds/bits.h b/src/snmalloc/ds_core/bits.h similarity index 87% rename from src/ds/bits.h rename to src/snmalloc/ds_core/bits.h index 455ec4a..f1dc4ff 100644 --- a/src/ds/bits.h +++ b/src/snmalloc/ds_core/bits.h @@ -5,13 +5,16 @@ // #define USE_LZCNT -#include "../aal/aal.h" -#include "../pal/pal_consts.h" #include "defines.h" #include +#include #include #include +#if defined(_MSC_VER) +# include +#endif + #ifdef pause # undef pause #endif @@ -30,7 +33,16 @@ namespace snmalloc namespace bits { - static constexpr size_t BITS = Aal::bits; + /** + * The number of bits in a `size_t`. Many of the functions in the + * `snmalloc::bits` namespace are defined to operate over `size_t`, mapping + * to the correct compiler builtins irrespective of the size. This size + * does *not* imply the number of bits of virtual address space that are + * actually allowed to be set. `Aal::bits` provides an + * architecture-specific definition of the number of bits of address space + * that exist. + */ + static constexpr size_t BITS = sizeof(size_t) * CHAR_BIT; /** * Returns a value of type T that has a single bit set, @@ -52,7 +64,7 @@ namespace snmalloc SNMALLOC_ASSERT(x != 0); // Calling with 0 is UB on some implementations #if defined(_MSC_VER) # ifdef USE_LZCNT -# ifdef SNMALLOC_VA_BITS_64 +# ifdef _WIN64 return __lzcnt64(x); # else return __lzcnt((uint32_t)x); @@ -60,10 +72,10 @@ namespace snmalloc # else unsigned long index; -# ifdef SNMALLOC_VA_BITS_64 - _BitScanReverse64(&index, x); +# ifdef _WIN64 + _BitScanReverse64(&index, static_cast(x)); # else - _BitScanReverse(&index, (unsigned long)x); + _BitScanReverse(&index, static_cast(x)); # endif return BITS - index - 1; @@ -101,10 +113,10 @@ namespace snmalloc inline size_t rotr(size_t x, size_t n) { #if defined(_MSC_VER) -# ifdef SNMALLOC_VA_BITS_64 - return _rotr64(x, (int)n); +# ifdef _WIN64 + return _rotr64(static_cast(x), static_cast(n)); # else - return _rotr((uint32_t)x, (int)n); + return _rotr(static_cast(x), static_cast(n)); # endif #else return rotr_const(x, n); @@ -114,10 +126,10 @@ namespace snmalloc inline size_t rotl(size_t x, size_t n) { #if defined(_MSC_VER) -# ifdef SNMALLOC_VA_BITS_64 - return _rotl64(x, (int)n); +# ifdef _WIN64 + return _rotl64(static_cast(x), static_cast(n)); # else - return _rotl((uint32_t)x, (int)n); + return _rotl(static_cast(x), static_cast(n)); # endif #else return rotl_const(x, n); @@ -145,11 +157,11 @@ namespace snmalloc { SNMALLOC_ASSERT(x != 0); // Calling with 0 is UB on some implementations -#if defined(_MSC_VER) -# ifdef SNMALLOC_VA_BITS_64 - return _tzcnt_u64(x); +#if defined(_MSC_VER) && !defined(__clang__) +# ifdef _WIN64 + return _tzcnt_u64(static_cast(x)); # else - return _tzcnt_u32((uint32_t)x); + return _tzcnt_u32(static_cast(x)); # endif #else if constexpr (std::is_same_v) @@ -191,14 +203,14 @@ namespace snmalloc overflow = __builtin_mul_overflow(x, y, &prod); return prod; #elif defined(_MSC_VER) -# if defined(SNMALLOC_VA_BITS_64) +# ifdef _WIN64 size_t high_prod; size_t prod = _umul128(x, y, &high_prod); overflow = high_prod != 0; return prod; # else - size_t prod; - overflow = S_OK != UIntMult(x, y, &prod); + UINT prod; + overflow = S_OK != UIntMult(UINT(x), UINT(y), &prod); return prod; # endif #else diff --git a/src/ds/concept.h b/src/snmalloc/ds_core/concept.h similarity index 100% rename from src/ds/concept.h rename to src/snmalloc/ds_core/concept.h diff --git a/src/ds/defines.h b/src/snmalloc/ds_core/defines.h similarity index 100% rename from src/ds/defines.h rename to src/snmalloc/ds_core/defines.h diff --git a/src/snmalloc/ds_core/ds_core.h b/src/snmalloc/ds_core/ds_core.h new file mode 100644 index 0000000..672f7d1 --- /dev/null +++ b/src/snmalloc/ds_core/ds_core.h @@ -0,0 +1,16 @@ +#pragma once +/** + * The core definitions for snmalloc. These provide some basic helpers that do + * not depend on anything except for a working C++ implementation. + * + * Files in this directory may not include anything from any other directory in + * snmalloc. + */ + +#include "bits.h" +#include "concept.h" +#include "defines.h" +#include "helpers.h" +#include "ptrwrap.h" +#include "redblacktree.h" +#include "seqset.h" diff --git a/src/ds/helpers.h b/src/snmalloc/ds_core/helpers.h similarity index 90% rename from src/ds/helpers.h rename to src/snmalloc/ds_core/helpers.h index f599aac..693b734 100644 --- a/src/ds/helpers.h +++ b/src/snmalloc/ds_core/helpers.h @@ -1,53 +1,15 @@ #pragma once #include "bits.h" -#include "flaglock.h" #include #include +#include #include #include namespace snmalloc { - /* - * In some use cases we need to run before any of the C++ runtime has been - * initialised. This singleton class is designed to not depend on the - * runtime. - */ - template - class Singleton - { - inline static FlagWord flag; - inline static std::atomic initialised{false}; - inline static Object obj; - - public: - /** - * If argument is non-null, then it is assigned the value - * true, if this is the first call to get. - * At most one call will be first. - */ - inline SNMALLOC_SLOW_PATH static Object& get(bool* first = nullptr) - { - // If defined should be initially false; - SNMALLOC_ASSERT(first == nullptr || *first == false); - - if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire))) - { - FlagLock lock(flag); - if (!initialised) - { - init(&obj); - initialised.store(true, std::memory_order_release); - if (first != nullptr) - *first = true; - } - } - return obj; - } - }; - /** * Wrapper for wrapping values. * @@ -477,4 +439,5 @@ namespace snmalloc */ struct Empty {}; + } // namespace snmalloc diff --git a/src/ds/ptrwrap.h b/src/snmalloc/ds_core/ptrwrap.h similarity index 99% rename from src/ds/ptrwrap.h rename to src/snmalloc/ds_core/ptrwrap.h index b94c90f..e7aa85c 100644 --- a/src/ds/ptrwrap.h +++ b/src/snmalloc/ds_core/ptrwrap.h @@ -1,7 +1,7 @@ #pragma once -#include "../ds/concept.h" -#include "../ds/defines.h" +#include "concept.h" +#include "defines.h" #include diff --git a/src/ds/redblacktree.h b/src/snmalloc/ds_core/redblacktree.h similarity index 99% rename from src/ds/redblacktree.h rename to src/snmalloc/ds_core/redblacktree.h index 3d87f9e..0d68469 100644 --- a/src/ds/redblacktree.h +++ b/src/snmalloc/ds_core/redblacktree.h @@ -1,7 +1,4 @@ #pragma once -#include "../pal/pal.h" -#include "concept.h" -#include "defines.h" #include #include diff --git a/src/ds/seqset.h b/src/snmalloc/ds_core/seqset.h similarity index 98% rename from src/ds/seqset.h rename to src/snmalloc/ds_core/seqset.h index 510961c..22a0fcd 100644 --- a/src/ds/seqset.h +++ b/src/snmalloc/ds_core/seqset.h @@ -1,8 +1,6 @@ #pragma once -#include "address.h" -#include "defines.h" -#include "ptrwrap.h" +#include "../ds_core/ds_core.h" #include #include diff --git a/src/mem/bounds_checks.h b/src/snmalloc/global/bounds_checks.h similarity index 98% rename from src/mem/bounds_checks.h rename to src/snmalloc/global/bounds_checks.h index 2843530..66b67ec 100644 --- a/src/mem/bounds_checks.h +++ b/src/snmalloc/global/bounds_checks.h @@ -1,5 +1,5 @@ #pragma once -#include "../snmalloc.h" +#include "threadalloc.h" namespace snmalloc { @@ -107,4 +107,4 @@ namespace snmalloc } } -} +} // namespace snmalloc diff --git a/src/snmalloc/global/global.h b/src/snmalloc/global/global.h new file mode 100644 index 0000000..a2f1159 --- /dev/null +++ b/src/snmalloc/global/global.h @@ -0,0 +1,4 @@ +#include "bounds_checks.h" +#include "memcpy.h" +#include "scopedalloc.h" +#include "threadalloc.h" diff --git a/src/mem/memcpy.h b/src/snmalloc/global/memcpy.h similarity index 99% rename from src/mem/memcpy.h rename to src/snmalloc/global/memcpy.h index 30eb7fb..beba37c 100644 --- a/src/mem/memcpy.h +++ b/src/snmalloc/global/memcpy.h @@ -1,5 +1,6 @@ #pragma once -#include "../mem/bounds_checks.h" +#include "../backend/globalconfig.h" +#include "bounds_checks.h" namespace snmalloc { @@ -328,4 +329,4 @@ namespace snmalloc Arch::copy(dst, src, len); return orig_dst; } -} // namespace +} // namespace snmalloc diff --git a/src/mem/scopedalloc.h b/src/snmalloc/global/scopedalloc.h similarity index 98% rename from src/mem/scopedalloc.h rename to src/snmalloc/global/scopedalloc.h index 345635a..cb9f0fc 100644 --- a/src/mem/scopedalloc.h +++ b/src/snmalloc/global/scopedalloc.h @@ -1,4 +1,5 @@ #pragma once +#include "../backend/globalconfig.h" /** * This header requires that Alloc has been defined. diff --git a/src/mem/threadalloc.h b/src/snmalloc/global/threadalloc.h similarity index 99% rename from src/mem/threadalloc.h rename to src/snmalloc/global/threadalloc.h index bbda32d..d900fb2 100644 --- a/src/mem/threadalloc.h +++ b/src/snmalloc/global/threadalloc.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/helpers.h" -#include "localalloc.h" +#include "../backend/globalconfig.h" #if defined(SNMALLOC_EXTERNAL_THREAD_ALLOC) # define SNMALLOC_THREAD_TEARDOWN_DEFINED diff --git a/src/backend/backend_concept.h b/src/snmalloc/mem/backend_concept.h similarity index 97% rename from src/backend/backend_concept.h rename to src/snmalloc/mem/backend_concept.h index 27aa8df..e29a2df 100644 --- a/src/backend/backend_concept.h +++ b/src/snmalloc/mem/backend_concept.h @@ -1,9 +1,7 @@ #pragma once #ifdef __cpp_concepts -# include "../ds/address.h" -# include "../ds/concept.h" -# include "../pal/pal_concept.h" +# include "../ds/ds.h" # include namespace snmalloc diff --git a/src/snmalloc/mem/backend_wrappers.h b/src/snmalloc/mem/backend_wrappers.h new file mode 100644 index 0000000..36657df --- /dev/null +++ b/src/snmalloc/mem/backend_wrappers.h @@ -0,0 +1,85 @@ +#pragma once +/** + * Several of the functions provided by the back end are optional. This file + * contains helpers that are templated on a back end and either call the + * corresponding function or do nothing. This allows the rest of the front end + * to assume that these functions always exist and avoid the need for `if + * constexpr` clauses everywhere. The no-op versions are always inlined and so + * will be optimised away. + */ + +#include "../ds_core/ds_core.h" + +namespace snmalloc +{ + /** + * SFINAE helper. Matched only if `T` implements `is_initialised`. Calls + * it if it exists. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) + -> decltype(T::is_initialised()) + { + return T::is_initialised(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `is_initialised`. + * Unconditionally returns true if invoked. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, long) + { + return true; + } + + namespace detail + { + /** + * SFINAE helper, calls capptr_domesticate in the backend if it exists. + */ + template< + SNMALLOC_CONCEPT(ConceptBackendDomestication) Backend, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + SNMALLOC_FAST_PATH_INLINE auto + capptr_domesticate(typename Backend::LocalState* ls, CapPtr p, int) + -> decltype(Backend::capptr_domesticate(ls, p)) + { + return Backend::capptr_domesticate(ls, p); + } + + /** + * SFINAE helper. If the back end does not provide special handling for + * domestication then assume all wild pointers can be domesticated. + */ + template< + SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + SNMALLOC_FAST_PATH_INLINE auto + capptr_domesticate(typename Backend::LocalState*, CapPtr p, long) + { + return CapPtr< + T, + typename B::template with_wildness>( + p.unsafe_ptr()); + } + } // namespace detail + + /** + * Wrapper that calls `Backend::capptr_domesticate` if and only if it is + * implemented. If it is not implemented then this assumes that any wild + * pointer can be domesticated. + */ + template< + SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + SNMALLOC_FAST_PATH_INLINE auto + capptr_domesticate(typename Backend::LocalState* ls, CapPtr p) + { + return detail::capptr_domesticate(ls, p, 0); + } + +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/snmalloc/mem/corealloc.h similarity index 99% rename from src/mem/corealloc.h rename to src/snmalloc/mem/corealloc.h index a1fc5c5..4baab6f 100644 --- a/src/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/defines.h" -#include "allocconfig.h" +#include "../ds/ds.h" #include "localcache.h" #include "metadata.h" #include "pool.h" diff --git a/src/mem/entropy.h b/src/snmalloc/mem/entropy.h similarity index 99% rename from src/mem/entropy.h rename to src/snmalloc/mem/entropy.h index 7f14919..1b59094 100644 --- a/src/mem/entropy.h +++ b/src/snmalloc/mem/entropy.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/address.h" #include "../pal/pal.h" #include diff --git a/src/mem/external_alloc.h b/src/snmalloc/mem/external_alloc.h similarity index 100% rename from src/mem/external_alloc.h rename to src/snmalloc/mem/external_alloc.h diff --git a/src/mem/freelist.h b/src/snmalloc/mem/freelist.h similarity index 99% rename from src/mem/freelist.h rename to src/snmalloc/mem/freelist.h index 9850b61..c310153 100644 --- a/src/mem/freelist.h +++ b/src/snmalloc/mem/freelist.h @@ -33,8 +33,7 @@ * and randomly deciding which list to add an element to. */ -#include "../ds/address.h" -#include "allocconfig.h" +#include "../ds/ds.h" #include "entropy.h" #include diff --git a/src/mem/globalalloc.h b/src/snmalloc/mem/globalalloc.h similarity index 72% rename from src/mem/globalalloc.h rename to src/snmalloc/mem/globalalloc.h index 0152f6c..b898eed 100644 --- a/src/mem/globalalloc.h +++ b/src/snmalloc/mem/globalalloc.h @@ -1,53 +1,10 @@ #pragma once -#include "../ds/helpers.h" +#include "../ds_core/ds_core.h" #include "localalloc.h" namespace snmalloc { - template - inline static void aggregate_stats(Stats& stats) - { - static_assert( - SharedStateHandle::Options.CoreAllocIsPoolAllocated, - "Global statistics are available only for pool-allocated configurations"); - auto* alloc = AllocPool::iterate(); - - while (alloc != nullptr) - { - auto a = alloc->attached_stats(); - if (a != nullptr) - stats.add(*a); - stats.add(alloc->stats()); - alloc = AllocPool::iterate(alloc); - } - } - -#ifdef USE_SNMALLOC_STATS - template - inline static void print_all_stats(std::ostream& o, uint64_t dumpid = 0) - { - static_assert( - SharedStateHandle::Options.CoreAllocIsPoolAllocated, - "Global statistics are available only for pool-allocated configurations"); - auto alloc = AllocPool::iterate(); - - while (alloc != nullptr) - { - auto stats = alloc->stats(); - if (stats != nullptr) - stats->template print(o, dumpid, alloc->id()); - alloc = AllocPool::iterate(alloc); - } - } -#else - template - inline static void print_all_stats(void*& o, uint64_t dumpid = 0) - { - UNUSED(o, dumpid); - } -#endif - template inline static void cleanup_unused() { diff --git a/src/mem/localalloc.h b/src/snmalloc/mem/localalloc.h similarity index 99% rename from src/mem/localalloc.h rename to src/snmalloc/mem/localalloc.h index bbe0781..086fd8a 100644 --- a/src/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -6,7 +6,7 @@ # define ALLOCATOR #endif -#include "../ds/ptrwrap.h" +#include "../ds/ds.h" #include "corealloc.h" #include "freelist.h" #include "localcache.h" @@ -18,9 +18,6 @@ # include "external_alloc.h" #endif -#ifdef SNMALLOC_TRACING -# include -#endif #include #include namespace snmalloc diff --git a/src/mem/localcache.h b/src/snmalloc/mem/localcache.h similarity index 90% rename from src/mem/localcache.h rename to src/snmalloc/mem/localcache.h index 9b3f174..2ae8ffd 100644 --- a/src/mem/localcache.h +++ b/src/snmalloc/mem/localcache.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/ptrwrap.h" -#include "allocstats.h" +#include "../ds/ds.h" #include "freelist.h" #include "remotecache.h" #include "sizeclasstable.h" @@ -10,8 +9,6 @@ namespace snmalloc { - using Stats = AllocStats; - inline static SNMALLOC_FAST_PATH capptr::Alloc finish_alloc_no_zero(freelist::HeadPtr p, smallsizeclass_t sizeclass) { @@ -51,10 +48,6 @@ namespace snmalloc // This is the entropy for a particular thread. LocalEntropy entropy; - // TODO: Minimal stats object for just the stats on this datastructure. - // This will be a zero-size structure if stats are not enabled. - Stats stats; - // Pointer to the remote allocator message_queue, used to check // if a deallocation is local. RemoteAllocator* remote_allocator; @@ -111,8 +104,6 @@ namespace snmalloc { auto& key = entropy.get_free_list_key(); smallsizeclass_t sizeclass = size_to_sizeclass(size); - stats.alloc_request(size); - stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; if (SNMALLOC_LIKELY(!fl.empty())) { diff --git a/src/snmalloc/mem/mem.h b/src/snmalloc/mem/mem.h new file mode 100644 index 0000000..9fb29a9 --- /dev/null +++ b/src/snmalloc/mem/mem.h @@ -0,0 +1,16 @@ +#include "backend_concept.h" +#include "backend_wrappers.h" +#include "corealloc.h" +#include "entropy.h" +#include "external_alloc.h" +#include "freelist.h" +#include "globalalloc.h" +#include "localalloc.h" +#include "localcache.h" +#include "metadata.h" +#include "pool.h" +#include "pooled.h" +#include "remoteallocator.h" +#include "remotecache.h" +#include "sizeclasstable.h" +#include "ticker.h" diff --git a/src/mem/metadata.h b/src/snmalloc/mem/metadata.h similarity index 99% rename from src/mem/metadata.h rename to src/snmalloc/mem/metadata.h index 765e298..ac56f0a 100644 --- a/src/mem/metadata.h +++ b/src/snmalloc/mem/metadata.h @@ -1,8 +1,6 @@ #pragma once -#include "../backend/backend_concept.h" -#include "../ds/helpers.h" -#include "../ds/seqset.h" +#include "../ds/ds.h" #include "freelist.h" #include "sizeclasstable.h" diff --git a/src/mem/pool.h b/src/snmalloc/mem/pool.h similarity index 98% rename from src/mem/pool.h rename to src/snmalloc/mem/pool.h index 141a14c..119777a 100644 --- a/src/mem/pool.h +++ b/src/snmalloc/mem/pool.h @@ -1,8 +1,6 @@ #pragma once -#include "../ds/flaglock.h" -#include "../ds/mpmcstack.h" -#include "../pal/pal_concept.h" +#include "../ds/ds.h" #include "pooled.h" #include diff --git a/src/mem/pooled.h b/src/snmalloc/mem/pooled.h similarity index 90% rename from src/mem/pooled.h rename to src/snmalloc/mem/pooled.h index 7b499fd..ac1af5d 100644 --- a/src/mem/pooled.h +++ b/src/snmalloc/mem/pooled.h @@ -1,6 +1,7 @@ #pragma once -#include "../ds/bits.h" +#include "../ds/ds.h" +#include "backend_concept.h" namespace snmalloc { diff --git a/src/mem/remoteallocator.h b/src/snmalloc/mem/remoteallocator.h similarity index 98% rename from src/mem/remoteallocator.h rename to src/snmalloc/mem/remoteallocator.h index e851a0a..2b92e9f 100644 --- a/src/mem/remoteallocator.h +++ b/src/snmalloc/mem/remoteallocator.h @@ -1,9 +1,9 @@ #pragma once -#include "../mem/allocconfig.h" -#include "../mem/freelist.h" -#include "../mem/metadata.h" -#include "../mem/sizeclasstable.h" +#include "../ds/ds.h" +#include "freelist.h" +#include "metadata.h" +#include "sizeclasstable.h" #include #include diff --git a/src/mem/remotecache.h b/src/snmalloc/mem/remotecache.h similarity index 99% rename from src/mem/remotecache.h rename to src/snmalloc/mem/remotecache.h index 1166656..01a2752 100644 --- a/src/mem/remotecache.h +++ b/src/snmalloc/mem/remotecache.h @@ -1,6 +1,7 @@ #pragma once -#include "allocconfig.h" +#include "../ds/ds.h" +#include "backend_wrappers.h" #include "freelist.h" #include "metadata.h" #include "remoteallocator.h" diff --git a/src/mem/sizeclasstable.h b/src/snmalloc/mem/sizeclasstable.h similarity index 99% rename from src/mem/sizeclasstable.h rename to src/snmalloc/mem/sizeclasstable.h index ae55545..b257d07 100644 --- a/src/mem/sizeclasstable.h +++ b/src/snmalloc/mem/sizeclasstable.h @@ -1,9 +1,6 @@ #pragma once -#include "../ds/bits.h" -#include "../ds/defines.h" -#include "../ds/helpers.h" -#include "allocconfig.h" +#include "../ds/ds.h" /** * This file contains all the code for transforming transforming sizes to diff --git a/src/mem/ticker.h b/src/snmalloc/mem/ticker.h similarity index 98% rename from src/mem/ticker.h rename to src/snmalloc/mem/ticker.h index 4dee55e..2bce041 100644 --- a/src/mem/ticker.h +++ b/src/snmalloc/mem/ticker.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/defines.h" +#include "../ds_core/ds_core.h" #include diff --git a/src/override/jemalloc_compat.cc b/src/snmalloc/override/jemalloc_compat.cc similarity index 100% rename from src/override/jemalloc_compat.cc rename to src/snmalloc/override/jemalloc_compat.cc diff --git a/src/override/malloc-extensions.cc b/src/snmalloc/override/malloc-extensions.cc similarity index 100% rename from src/override/malloc-extensions.cc rename to src/snmalloc/override/malloc-extensions.cc diff --git a/src/override/malloc-extensions.h b/src/snmalloc/override/malloc-extensions.h similarity index 100% rename from src/override/malloc-extensions.h rename to src/snmalloc/override/malloc-extensions.h diff --git a/src/override/malloc.cc b/src/snmalloc/override/malloc.cc similarity index 100% rename from src/override/malloc.cc rename to src/snmalloc/override/malloc.cc diff --git a/src/override/memcpy.cc b/src/snmalloc/override/memcpy.cc similarity index 90% rename from src/override/memcpy.cc rename to src/snmalloc/override/memcpy.cc index 5de742e..c2283ec 100644 --- a/src/override/memcpy.cc +++ b/src/snmalloc/override/memcpy.cc @@ -1,5 +1,3 @@ -#include "mem/memcpy.h" - #include "override.h" extern "C" diff --git a/src/override/new.cc b/src/snmalloc/override/new.cc similarity index 100% rename from src/override/new.cc rename to src/snmalloc/override/new.cc diff --git a/src/snmalloc/override/override.h b/src/snmalloc/override/override.h new file mode 100644 index 0000000..0ca70bc --- /dev/null +++ b/src/snmalloc/override/override.h @@ -0,0 +1,15 @@ +#pragma once + +#include "../global/global.h" + +#ifndef SNMALLOC_EXPORT +# define SNMALLOC_EXPORT +#endif +#ifdef SNMALLOC_STATIC_LIBRARY_PREFIX +# define __SN_CONCAT(a, b) a##b +# define __SN_EVALUATE(a, b) __SN_CONCAT(a, b) +# define SNMALLOC_NAME_MANGLE(a) \ + __SN_EVALUATE(SNMALLOC_STATIC_LIBRARY_PREFIX, a) +#elif !defined(SNMALLOC_NAME_MANGLE) +# define SNMALLOC_NAME_MANGLE(a) a +#endif diff --git a/src/override/rust.cc b/src/snmalloc/override/rust.cc similarity index 100% rename from src/override/rust.cc rename to src/snmalloc/override/rust.cc diff --git a/src/pal/pal.h b/src/snmalloc/pal/pal.h similarity index 89% rename from src/pal/pal.h rename to src/snmalloc/pal/pal.h index 8a0260f..697b22c 100644 --- a/src/pal/pal.h +++ b/src/snmalloc/pal/pal.h @@ -1,10 +1,22 @@ +/** + * The platform abstraction layer. This defines an abstraction that exposes + * services from the operating system or any equivalent environment. + * + * It is possible to have multiple PALs in a single snmalloc instance. For + * example, one may provide the base operating system functionality, another + * may hide some of this or provide its own abstractions for sandboxing or + * isolating heaps. + * + * Files in this directory may depend on the architecture abstraction and core + * layers (`aal` and `ds_core`, respectively) but nothing else in snmalloc. + */ #pragma once -#include "../ds/concept.h" +#include "../aal/aal.h" #include "pal_concept.h" #include "pal_consts.h" -// If simultating OE, then we need the underlying platform +// If simulating OE, then we need the underlying platform #if defined(OPEN_ENCLAVE) # include "pal_open_enclave.h" #endif diff --git a/src/pal/pal_apple.h b/src/snmalloc/pal/pal_apple.h similarity index 100% rename from src/pal/pal_apple.h rename to src/snmalloc/pal/pal_apple.h diff --git a/src/pal/pal_bsd.h b/src/snmalloc/pal/pal_bsd.h similarity index 100% rename from src/pal/pal_bsd.h rename to src/snmalloc/pal/pal_bsd.h diff --git a/src/pal/pal_bsd_aligned.h b/src/snmalloc/pal/pal_bsd_aligned.h similarity index 100% rename from src/pal/pal_bsd_aligned.h rename to src/snmalloc/pal/pal_bsd_aligned.h diff --git a/src/pal/pal_concept.h b/src/snmalloc/pal/pal_concept.h similarity index 58% rename from src/pal/pal_concept.h rename to src/snmalloc/pal/pal_concept.h index a476cce..78270f2 100644 --- a/src/pal/pal_concept.h +++ b/src/snmalloc/pal/pal_concept.h @@ -1,14 +1,13 @@ #pragma once #ifdef __cpp_concepts -# include "../ds/concept.h" # include "pal_consts.h" # include "pal_ds.h" + # include namespace snmalloc { - /* * These concepts enforce that these are indeed constants that fit in the * desired types. (This is subtly different from saying that they are the @@ -41,7 +40,10 @@ namespace snmalloc template concept ConceptPAL_error = requires(const char* const str) { - { PAL::error(str) } -> ConceptSame; + { + PAL::error(str) + } + ->ConceptSame; }; /** @@ -50,25 +52,40 @@ namespace snmalloc template concept ConceptPAL_memops = requires(void* vp, std::size_t sz) { - { PAL::notify_not_using(vp, sz) } noexcept -> ConceptSame; + { + PAL::notify_not_using(vp, sz) + } + noexcept->ConceptSame; - { PAL::template notify_using(vp, sz) } noexcept - -> ConceptSame; - { PAL::template notify_using(vp, sz) } noexcept - -> ConceptSame; + { + PAL::template notify_using(vp, sz) + } + noexcept->ConceptSame; + { + PAL::template notify_using(vp, sz) + } + noexcept->ConceptSame; - { PAL::template zero(vp, sz) } noexcept -> ConceptSame; - { PAL::template zero(vp, sz) } noexcept -> ConceptSame; + { + PAL::template zero(vp, sz) + } + noexcept->ConceptSame; + { + PAL::template zero(vp, sz) + } + noexcept->ConceptSame; }; /** * Absent any feature flags, the PAL must support a crude primitive allocator */ template - concept ConceptPAL_reserve = - requires(PAL p, std::size_t sz) + concept ConceptPAL_reserve = requires(PAL p, std::size_t sz) { - { PAL::reserve(sz) } noexcept -> ConceptSame; + { + PAL::reserve(sz) + } + noexcept->ConceptSame; }; /** @@ -77,9 +94,14 @@ namespace snmalloc template concept ConceptPAL_reserve_aligned = requires(std::size_t sz) { - { PAL::template reserve_aligned(sz) } noexcept -> ConceptSame; - { PAL::template reserve_aligned(sz) } noexcept - -> ConceptSame; + { + PAL::template reserve_aligned(sz) + } + noexcept->ConceptSame; + { + PAL::template reserve_aligned(sz) + } + noexcept->ConceptSame; }; /** @@ -88,14 +110,23 @@ namespace snmalloc template concept ConceptPAL_mem_low_notify = requires(PalNotificationObject* pno) { - { PAL::expensive_low_memory_check() } -> ConceptSame; - { PAL::register_for_low_memory_callback(pno) } -> ConceptSame; + { + PAL::expensive_low_memory_check() + } + ->ConceptSame; + { + PAL::register_for_low_memory_callback(pno) + } + ->ConceptSame; }; template concept ConceptPAL_get_entropy64 = requires() { - { PAL::get_entropy64() } -> ConceptSame; + { + PAL::get_entropy64() + } + ->ConceptSame; }; /** @@ -105,21 +136,17 @@ namespace snmalloc * are, naturally, not bound by the corresponding concept. */ template - concept ConceptPAL = - ConceptPAL_static_features && - ConceptPAL_static_sizes && - ConceptPAL_error && - ConceptPAL_memops && + concept ConceptPAL = ConceptPAL_static_features&& + ConceptPAL_static_sizes&& ConceptPAL_error&& + ConceptPAL_memops && (!pal_supports || - ConceptPAL_get_entropy64) && - (!pal_supports || - ConceptPAL_mem_low_notify) && - (pal_supports || - ( - (!pal_supports || - ConceptPAL_reserve_aligned) && - ConceptPAL_reserve) - ); + ConceptPAL_get_entropy64< + PAL>)&&(!pal_supports || + ConceptPAL_mem_low_notify< + PAL>)&&(pal_supports || + ((!pal_supports || + ConceptPAL_reserve_aligned< + PAL>)&&ConceptPAL_reserve)); } // namespace snmalloc #endif diff --git a/src/pal/pal_consts.h b/src/snmalloc/pal/pal_consts.h similarity index 98% rename from src/pal/pal_consts.h rename to src/snmalloc/pal/pal_consts.h index 31df547..8de4cb0 100644 --- a/src/pal/pal_consts.h +++ b/src/snmalloc/pal/pal_consts.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/defines.h" +#include "../ds_core/ds_core.h" #include #include diff --git a/src/pal/pal_dragonfly.h b/src/snmalloc/pal/pal_dragonfly.h similarity index 100% rename from src/pal/pal_dragonfly.h rename to src/snmalloc/pal/pal_dragonfly.h diff --git a/src/pal/pal_ds.h b/src/snmalloc/pal/pal_ds.h similarity index 98% rename from src/pal/pal_ds.h rename to src/snmalloc/pal/pal_ds.h index d6197d1..3da37cf 100644 --- a/src/pal/pal_ds.h +++ b/src/snmalloc/pal/pal_ds.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/defines.h" -#include "../ds/helpers.h" +#include "../ds_core/ds_core.h" #include #include diff --git a/src/pal/pal_freebsd.h b/src/snmalloc/pal/pal_freebsd.h similarity index 100% rename from src/pal/pal_freebsd.h rename to src/snmalloc/pal/pal_freebsd.h diff --git a/src/pal/pal_freebsd_kernel.h b/src/snmalloc/pal/pal_freebsd_kernel.h similarity index 98% rename from src/pal/pal_freebsd_kernel.h rename to src/snmalloc/pal/pal_freebsd_kernel.h index 2e25d91..11b951c 100644 --- a/src/pal/pal_freebsd_kernel.h +++ b/src/snmalloc/pal/pal_freebsd_kernel.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/bits.h" +#include "../ds_core/ds_core.h" #if defined(FreeBSD_KERNEL) extern "C" diff --git a/src/pal/pal_haiku.h b/src/snmalloc/pal/pal_haiku.h similarity index 100% rename from src/pal/pal_haiku.h rename to src/snmalloc/pal/pal_haiku.h diff --git a/src/pal/pal_linux.h b/src/snmalloc/pal/pal_linux.h similarity index 99% rename from src/pal/pal_linux.h rename to src/snmalloc/pal/pal_linux.h index aa28ded..66240fa 100644 --- a/src/pal/pal_linux.h +++ b/src/snmalloc/pal/pal_linux.h @@ -1,7 +1,7 @@ #pragma once #if defined(__linux__) -# include "../ds/bits.h" +# include "../ds_core/ds_core.h" # include "pal_posix.h" # include diff --git a/src/pal/pal_netbsd.h b/src/snmalloc/pal/pal_netbsd.h similarity index 100% rename from src/pal/pal_netbsd.h rename to src/snmalloc/pal/pal_netbsd.h diff --git a/src/pal/pal_noalloc.h b/src/snmalloc/pal/pal_noalloc.h similarity index 100% rename from src/pal/pal_noalloc.h rename to src/snmalloc/pal/pal_noalloc.h diff --git a/src/pal/pal_open_enclave.h b/src/snmalloc/pal/pal_open_enclave.h similarity index 100% rename from src/pal/pal_open_enclave.h rename to src/snmalloc/pal/pal_open_enclave.h diff --git a/src/pal/pal_openbsd.h b/src/snmalloc/pal/pal_openbsd.h similarity index 100% rename from src/pal/pal_openbsd.h rename to src/snmalloc/pal/pal_openbsd.h diff --git a/src/pal/pal_plain.h b/src/snmalloc/pal/pal_plain.h similarity index 95% rename from src/pal/pal_plain.h rename to src/snmalloc/pal/pal_plain.h index 27230f2..005ef98 100644 --- a/src/pal/pal_plain.h +++ b/src/snmalloc/pal/pal_plain.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/bits.h" +#include "../ds_core/ds_core.h" #include "pal_timer_default.h" namespace snmalloc diff --git a/src/pal/pal_posix.h b/src/snmalloc/pal/pal_posix.h similarity index 99% rename from src/pal/pal_posix.h rename to src/snmalloc/pal/pal_posix.h index add7712..f71755d 100644 --- a/src/pal/pal_posix.h +++ b/src/snmalloc/pal/pal_posix.h @@ -1,9 +1,6 @@ #pragma once -#ifdef SNMALLOC_TRACING -# include -#endif -#include "../ds/address.h" +#include "../aal/aal.h" #include "pal_timer_default.h" #if defined(SNMALLOC_BACKTRACE_HEADER) # include SNMALLOC_BACKTRACE_HEADER diff --git a/src/pal/pal_solaris.h b/src/snmalloc/pal/pal_solaris.h similarity index 100% rename from src/pal/pal_solaris.h rename to src/snmalloc/pal/pal_solaris.h diff --git a/src/pal/pal_timer_default.h b/src/snmalloc/pal/pal_timer_default.h similarity index 100% rename from src/pal/pal_timer_default.h rename to src/snmalloc/pal/pal_timer_default.h diff --git a/src/pal/pal_windows.h b/src/snmalloc/pal/pal_windows.h similarity index 99% rename from src/pal/pal_windows.h rename to src/snmalloc/pal/pal_windows.h index 6d64a25..2d6f822 100644 --- a/src/pal/pal_windows.h +++ b/src/snmalloc/pal/pal_windows.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/address.h" -#include "../ds/bits.h" +#include "../aal/aal.h" #include "pal_timer_default.h" #ifdef _WIN32 diff --git a/src/snmalloc/snmalloc.h b/src/snmalloc/snmalloc.h new file mode 100644 index 0000000..47bd6e7 --- /dev/null +++ b/src/snmalloc/snmalloc.h @@ -0,0 +1,10 @@ +#pragma once + +// Core implementation of snmalloc independent of the configuration mode +#include "snmalloc_core.h" + +// If the user has defined SNMALLOC_PROVIDE_OWN_CONFIG, this include does +// nothing. Otherwise, it provide a default configuration of snmalloc::Alloc. +#include "backend/globalconfig.h" +// User facing API surface, needs to know what `Alloc` is. +#include "snmalloc_front.h" diff --git a/src/snmalloc/snmalloc_core.h b/src/snmalloc/snmalloc_core.h new file mode 100644 index 0000000..633b7ab --- /dev/null +++ b/src/snmalloc/snmalloc_core.h @@ -0,0 +1,3 @@ +#pragma once + +#include "backend_helpers/backend_helpers.h" diff --git a/src/snmalloc/snmalloc_front.h b/src/snmalloc/snmalloc_front.h new file mode 100644 index 0000000..4c5aa60 --- /dev/null +++ b/src/snmalloc/snmalloc_front.h @@ -0,0 +1 @@ +#include "global/global.h" diff --git a/src/snmalloc_core.h b/src/snmalloc_core.h deleted file mode 100644 index 5cd224f..0000000 --- a/src/snmalloc_core.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "backend/commitrange.h" -#include "backend/commonconfig.h" -#include "backend/empty_range.h" -#include "backend/globalrange.h" -#include "backend/largebuddyrange.h" -#include "backend/pagemap.h" -#include "backend/pagemapregisterrange.h" -#include "backend/palrange.h" -#include "backend/range_helpers.h" -#include "backend/smallbuddyrange.h" -#include "backend/statsrange.h" -#include "backend/subrange.h" -#include "mem/globalalloc.h" diff --git a/src/snmalloc_front.h b/src/snmalloc_front.h deleted file mode 100644 index 00167dd..0000000 --- a/src/snmalloc_front.h +++ /dev/null @@ -1,2 +0,0 @@ -#include "mem/scopedalloc.h" -#include "mem/threadalloc.h" \ No newline at end of file diff --git a/src/test/func/bits/bits.cc b/src/test/func/bits/bits.cc index 874c5fb..0046516 100644 --- a/src/test/func/bits/bits.cc +++ b/src/test/func/bits/bits.cc @@ -3,7 +3,7 @@ */ #include -#include +#include #include void test_ctz() @@ -41,4 +41,4 @@ int main(int argc, char** argv) test_clz(); test_ctz(); -} \ No newline at end of file +} diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index 64c74dd..26c033a 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -10,8 +10,8 @@ int main() // # define SNMALLOC_TRACING -# include -# include +# include +# include // Specify type of allocator # define SNMALLOC_PROVIDE_OWN_CONFIG @@ -98,7 +98,7 @@ namespace snmalloc } # define SNMALLOC_NAME_MANGLE(a) test_##a -# include "../../../override/malloc.cc" +# include int main() { diff --git a/src/test/func/external_pagemap/external_pagemap.cc b/src/test/func/external_pagemap/external_pagemap.cc index 3d139a4..5e79c6a 100644 --- a/src/test/func/external_pagemap/external_pagemap.cc +++ b/src/test/func/external_pagemap/external_pagemap.cc @@ -9,7 +9,7 @@ int main() } #else # define SNMALLOC_EXPOSE_PAGEMAP 1 -# include +# include using ExternalChunkmap = ExternalGlobalPagemapTemplate; diff --git a/src/test/func/first_operation/first_operation.cc b/src/test/func/first_operation/first_operation.cc index aa22635..629027f 100644 --- a/src/test/func/first_operation/first_operation.cc +++ b/src/test/func/first_operation/first_operation.cc @@ -8,7 +8,7 @@ #include "test/setup.h" #include -#include +#include #include void alloc1(size_t size) diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 5273772..0a99627 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -1,8 +1,8 @@ -#include "backend/fixedglobalconfig.h" #include "test/setup.h" #include -#include +#include +#include #ifdef assert # undef assert diff --git a/src/test/func/jemalloc/jemalloc.cc b/src/test/func/jemalloc/jemalloc.cc index 1d0271e..5baddd7 100644 --- a/src/test/func/jemalloc/jemalloc.cc +++ b/src/test/func/jemalloc/jemalloc.cc @@ -9,8 +9,8 @@ #define SNMALLOC_BOOTSTRAP_ALLOCATOR #define SNMALLOC_JEMALLOC3_EXPERIMENTAL #define SNMALLOC_JEMALLOC_NONSTANDARD -#include "../../../override/jemalloc_compat.cc" -#include "../../../override/malloc.cc" +#include +#include #if __has_include() # include diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 4d7ae28..04b8839 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -6,7 +6,7 @@ #undef SNMALLOC_NO_REALLOCARRAY #undef SNMALLOC_NO_REALLOCARR #define SNMALLOC_BOOTSTRAP_ALLOCATOR -#include "../../../override/malloc.cc" +#include using namespace snmalloc; diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc index ad8ea17..ff1856f 100644 --- a/src/test/func/memcpy/func-memcpy.cc +++ b/src/test/func/memcpy/func-memcpy.cc @@ -19,14 +19,13 @@ int main() # endif # define SNMALLOC_FAIL_FAST false # define SNMALLOC_STATIC_LIBRARY_PREFIX my_ -# include "ds/defines.h" # ifndef SNMALLOC_PASS_THROUGH -# include "override/malloc.cc" +# include "snmalloc/override/malloc.cc" # else # define my_malloc(x) malloc(x) # define my_free(x) free(x) # endif -# include "override/memcpy.cc" +# include "snmalloc/override/memcpy.cc" # include "test/helpers.h" # include diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 8db19e3..b010f2a 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/test/func/memory_usage/memory_usage.cc b/src/test/func/memory_usage/memory_usage.cc index 98e3548..6d7dc40 100644 --- a/src/test/func/memory_usage/memory_usage.cc +++ b/src/test/func/memory_usage/memory_usage.cc @@ -7,8 +7,8 @@ #include #define SNMALLOC_NAME_MANGLE(a) our_##a -#include "../../../override/malloc-extensions.cc" -#include "../../../override/malloc.cc" +#include "../../../snmalloc/override/malloc-extensions.cc" +#include "../../../snmalloc/override/malloc.cc" using namespace snmalloc; diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index 74f14c4..1f00ff8 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -6,10 +6,8 @@ * but no examples were using multiple levels of pagemap. */ -#include -#include #include -#include +#include #include using namespace snmalloc; diff --git a/src/test/func/pool/pool.cc b/src/test/func/pool/pool.cc index 7d63e9e..215c393 100644 --- a/src/test/func/pool/pool.cc +++ b/src/test/func/pool/pool.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/test/func/redblack/redblack.cc b/src/test/func/redblack/redblack.cc index 208acea..f13c72e 100644 --- a/src/test/func/redblack/redblack.cc +++ b/src/test/func/redblack/redblack.cc @@ -8,10 +8,11 @@ #include #include -#define SNMALLOC_TRACING +#ifndef SNMALLOC_TRACING +# define SNMALLOC_TRACING +#endif // Redblack tree needs some libraries with trace enabled. -#include "ds/redblacktree.h" -#include "snmalloc.h" +#include "snmalloc/snmalloc.h" struct NodeRef { diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index 7b1a2d7..f954133 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -1,5 +1,5 @@ #include -#include +#include #include using namespace snmalloc; diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 5f3c4aa..d42794e 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -1,5 +1,5 @@ #include -#include +#include #include NOINLINE @@ -118,4 +118,4 @@ int main(int, char**) abort(); test_align_size(); -} \ No newline at end of file +} diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index 156612c..d83dd33 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -1,4 +1,4 @@ -#include +#include int main() { diff --git a/src/test/func/teardown/teardown.cc b/src/test/func/teardown/teardown.cc index b3d2645..f68ed4d 100644 --- a/src/test/func/teardown/teardown.cc +++ b/src/test/func/teardown/teardown.cc @@ -8,7 +8,7 @@ #include "test/setup.h" #include -#include +#include #include void trigger_teardown() diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index bc0952e..b8b1b23 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -2,13 +2,13 @@ # undef SNMALLOC_USE_PTHREAD_DESTRUCTORS #endif -#include +#include #include // Specify using own #define SNMALLOC_EXTERNAL_THREAD_ALLOC -#include "backend/globalconfig.h" +#include namespace snmalloc { @@ -32,7 +32,7 @@ public: } }; -#include +#include void allocator_thread_init(void) { diff --git a/src/test/func/two_alloc_types/alloc1.cc b/src/test/func/two_alloc_types/alloc1.cc index 8a21c80..8bfe413 100644 --- a/src/test/func/two_alloc_types/alloc1.cc +++ b/src/test/func/two_alloc_types/alloc1.cc @@ -1,10 +1,12 @@ -#define SNMALLOC_TRACING +#ifndef SNMALLOC_TRACING +# define SNMALLOC_TRACING +#endif // Redefine the namespace, so we can have two versions. #define snmalloc snmalloc_enclave -#include -#include +#include +#include // Specify type of allocator #define SNMALLOC_PROVIDE_OWN_CONFIG @@ -15,7 +17,7 @@ namespace snmalloc } #define SNMALLOC_NAME_MANGLE(a) enclave_##a -#include "../../../override/malloc.cc" +#include extern "C" void oe_allocator_init(void* base, void* end) { diff --git a/src/test/func/two_alloc_types/alloc2.cc b/src/test/func/two_alloc_types/alloc2.cc index 9bdfbf8..f25b5dc 100644 --- a/src/test/func/two_alloc_types/alloc2.cc +++ b/src/test/func/two_alloc_types/alloc2.cc @@ -1,6 +1,8 @@ -#define SNMALLOC_TRACING +#ifndef SNMALLOC_TRACING +# define SNMALLOC_TRACING +#endif #define SNMALLOC_NAME_MANGLE(a) host_##a // Redefine the namespace, so we can have two versions. #define snmalloc snmalloc_host -#include "../../../override/malloc.cc" +#include diff --git a/src/test/func/two_alloc_types/main.cc b/src/test/func/two_alloc_types/main.cc index 3cea54c..b7f6ded 100644 --- a/src/test/func/two_alloc_types/main.cc +++ b/src/test/func/two_alloc_types/main.cc @@ -1,6 +1,5 @@ -#include "../../../snmalloc.h" - #include +#include #include #include #include diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 74f9cd2..6f6bd39 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index b589124..b2509a6 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index aa024d4..fa2997f 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -164,4 +164,4 @@ int main(int argc, char** argv) // std::cout << "Release test only." << std::endl; // #endif return 0; -} \ No newline at end of file +} diff --git a/src/test/perf/memcpy/memcpy.cc b/src/test/perf/memcpy/memcpy.cc index 62f5da3..7642d60 100644 --- a/src/test/perf/memcpy/memcpy.cc +++ b/src/test/perf/memcpy/memcpy.cc @@ -1,4 +1,4 @@ -#include "mem/memcpy.h" +#include "snmalloc/global/memcpy.h" #include #include diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index cd15a12..2959523 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/test/setup.h b/src/test/setup.h index 2b440a6..642720d 100644 --- a/src/test/setup.h +++ b/src/test/setup.h @@ -1,10 +1,9 @@ #if defined(SNMALLOC_CI_BUILD) -# include +# include # if defined(WIN32) -# include # include -# include # include +# include # include // Has to come after the PAL. # include