From 7eabea01d66424c803619f79dfaa0ed8d15657a2 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Wed, 10 Jul 2019 10:42:59 +0100 Subject: [PATCH] Add an Architecture Abstraction Layer. Currently, we support one architecture, but this provides a layer for adding other architectures without adding more nested `#ifdef`s. Fixes #42 --- src/aal/aal.h | 86 +++++++++++++++++++ src/aal/aal_x86.h | 112 +++++++++++++++++++++++++ src/ds/aba.h | 2 +- src/ds/bits.h | 90 ++------------------ src/ds/defines.h | 23 ----- src/ds/flaglock.h | 2 +- src/ds/mpscq.h | 2 +- src/mem/allocstats.h | 4 +- src/mem/pagemap.h | 2 +- src/test/perf/contention/contention.cc | 6 +- 10 files changed, 214 insertions(+), 115 deletions(-) create mode 100644 src/aal/aal.h create mode 100644 src/aal/aal_x86.h diff --git a/src/aal/aal.h b/src/aal/aal.h new file mode 100644 index 0000000..24c046e --- /dev/null +++ b/src/aal/aal.h @@ -0,0 +1,86 @@ +#pragma once +#include "ds/defines.h" + +#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \ + defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) +# define PLATFORM_IS_X86 +#endif + +namespace snmalloc +{ + /** + * Architecture Abstraction Layer. Includes default implementations of some + * functions using compiler builtins. Falls back to the definitions in the + * platform's AAL if the builtin does not exist. + */ + template + struct AAL_Generic : Arch + { + /** + * Prefetch a specific address. + * + * If the compiler provides a portable prefetch builtin, use it directly, + * otherwise delegate to the architecture-specific layer. This allows new + * architectures to avoid needing to implement a custom `prefetch` method + * if they are used only with a compiler that provides the builtin. + */ + static inline void prefetch(void* ptr) + { +#if __has_builtin(__builtin_prefetch) && !defined(SNMALLOC_NO_AAL_BUILTINS) + __builtin_prefetch(ptr); +#else + Arch::prefetch(ptr); +#endif + } + + /** + * Return an architecture-specific cycle counter. + * + * If the compiler provides a portable prefetch builtin, use it directly, + * otherwise delegate to the architecture-specific layer. This allows new + * architectures to avoid needing to implement a custom `tick` method + * if they are used only with a compiler that provides the builtin. + */ + static inline uint64_t tick() + { +#if __has_builtin(__builtin_readcyclecounter) && \ + !defined(SNMALLOC_NO_AAL_BUILTINS) + return __builtin_readcyclecounter(); +#else + return Arch::tick(); +#endif + } + }; + +} + +#ifdef PLATFORM_IS_X86 +# include "aal_x86.h" +#endif + +namespace snmalloc +{ + using AAL = AAL_Generic; +} + +#if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_32) +# include +#endif + +#ifdef __POINTER_WIDTH__ +# if ((__POINTER_WIDTH__ == 64) && !defined(SNMALLOC_VA_BITS_64)) || \ + ((__POINTER_WIDTH__ == 32) && !defined(SNMALLOC_VA_BITS_32)) +# error Compiler and PAL define inconsistent bit widths +# endif +#endif + +#if defined(SNMALLOC_VA_BITS_32) && defined(SNMALLOC_VA_BITS_64) +# error Only one of SNMALLOC_VA_BITS_64 and SNMALLOC_VA_BITS_32 may be defined! +#endif + +#ifdef SNMALLOC_VA_BITS_32 +static_assert(sizeof(size_t) == 4); +#elif defined(SNMALLOC_VA_BITS_64) +static_assert(sizeof(size_t) == 8); +#endif diff --git a/src/aal/aal_x86.h b/src/aal/aal_x86.h new file mode 100644 index 0000000..69fe9e7 --- /dev/null +++ b/src/aal/aal_x86.h @@ -0,0 +1,112 @@ +#pragma once + +#ifdef _MSC_VER +# include +# include +#else +# include +# include +#endif + +#if defined(__linux__) && !defined(OPEN_ENCLAVE) +# include +#endif + +#if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) +# define SNMALLOC_VA_BITS_64 +#else +# define SNMALLOC_VA_BITS_32 +#endif + +namespace snmalloc +{ + /** + * x86-specific architecture abstraction layer. + */ + class AAL_x86 + { + /** + * Read the timestamp counter, guaranteeing that all previous instructions + * have been retired. + */ + static inline uint64_t tickp() + { + unsigned int aux; +#if defined(_MSC_VER) + return __rdtscp(&aux); +#else + return __builtin_ia32_rdtscp(&aux); +#endif + } + + /** + * Issue a fully serialising instruction. + */ + static inline void halt_out_of_order() + { +#if defined(_MSC_VER) + int cpu_info[4]; + __cpuid(cpu_info, 0); +#else + unsigned int eax, ebx, ecx, edx; + __get_cpuid(0, &eax, &ebx, &ecx, &edx); +#endif + } + + public: + /** + * On pipelined processors, notify the core that we are in a spin loop and + * that speculative execution past this point may not be a performance gain. + */ + static inline void pause() + { + _mm_pause(); + } + + /** + * Issue a prefetch hint at the specified address. + */ + static inline void prefetch(void* ptr) + { + _mm_prefetch(reinterpret_cast(ptr), _MM_HINT_T0); + } + + /** + * Return a cycle counter value. + */ + static inline uint64_t tick() + { +#if defined(_MSC_VER) + return __rdtsc(); +#else + return __builtin_ia32_rdtsc(); +#endif + } + + /** + * Return the cycle counter value that can be used to indicate the start of + * a benchmark run. This is responsible for ensuring any serialisation + * that the pipeline may require. + */ + static inline uint64_t benchmark_time_start() + { + halt_out_of_order(); + return AAL_Generic::tick(); + } + + /** + * Return the cycle counter value that can be used to indicate the end of a + * benchmark run. This is responsible for ensuring any serialisation that + * the pipeline may require. + */ + static inline uint64_t benchmark_time_end() + { + uint64_t t = tickp(); + halt_out_of_order(); + return t; + } + }; + + using AAL_Arch = AAL_x86; +} diff --git a/src/ds/aba.h b/src/ds/aba.h index 4275ad5..72c14ff 100644 --- a/src/ds/aba.h +++ b/src/ds/aba.h @@ -89,7 +89,7 @@ namespace snmalloc bool compare_exchange(Cmp& expect, T* value) { #ifdef PLATFORM_IS_X86 -# if defined(_MSC_VER) && defined(PLATFORM_BITS_64) +# if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_64) return _InterlockedCompareExchange128( (volatile __int64*)&linked, expect.aba + 1, diff --git a/src/ds/bits.h b/src/ds/bits.h index dff0b2a..6c3d734 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -3,9 +3,9 @@ #include #include - // #define USE_LZCNT +#include "aal/aal.h" #include "address.h" #include "defines.h" @@ -55,87 +55,11 @@ namespace snmalloc static constexpr size_t ADDRESS_BITS = is64() ? 48 : 32; - inline void pause() - { -#if defined(PLATFORM_IS_X86) - _mm_pause(); -#else -# warning "Missing pause intrinsic" -#endif - } - - inline void prefetch(void* ptr) - { -#if defined(PLATFORM_IS_X86) - _mm_prefetch(reinterpret_cast(ptr), _MM_HINT_T0); -#else -# warning "Missing prefetch intrinsic" -#endif - } - - inline uint64_t tick() - { -#if defined(PLATFORM_IS_X86) -# if defined(_MSC_VER) - return __rdtsc(); -# elif defined(__clang__) - return __builtin_readcyclecounter(); -# else - return __builtin_ia32_rdtsc(); -# endif -#else -# error Define CPU tick for this platform -#endif - } - - inline uint64_t tickp() - { -#if defined(PLATFORM_IS_X86) -# if defined(_MSC_VER) - unsigned int aux; - return __rdtscp(&aux); -# else - unsigned aux; - return __builtin_ia32_rdtscp(&aux); -# endif -#else -# error Define CPU tick for this platform -#endif - } - - inline void halt_out_of_order() - { -#if defined(PLATFORM_IS_X86) -# if defined(_MSC_VER) - int cpu_info[4]; - __cpuid(cpu_info, 0); -# else - unsigned int eax, ebx, ecx, edx; - __get_cpuid(0, &eax, &ebx, &ecx, &edx); -# endif -#else -# error Define CPU benchmark start time for this platform -#endif - } - - inline uint64_t benchmark_time_start() - { - halt_out_of_order(); - return tick(); - } - - inline uint64_t benchmark_time_end() - { - uint64_t t = tickp(); - halt_out_of_order(); - return t; - } - inline size_t clz(size_t x) { #if defined(_MSC_VER) # ifdef USE_LZCNT -# ifdef PLATFORM_BITS_64 +# ifdef SNMALLOC_VA_BITS_64 return __lzcnt64(x); # else return __lzcnt((uint32_t)x); @@ -143,7 +67,7 @@ namespace snmalloc # else unsigned long index; -# ifdef PLATFORM_BITS_64 +# ifdef SNMALLOC_VA_BITS_64 _BitScanReverse64(&index, x); # else _BitScanReverse(&index, (unsigned long)x); @@ -173,7 +97,7 @@ namespace snmalloc inline size_t rotr(size_t x, size_t n) { #if defined(_MSC_VER) -# ifdef PLATFORM_BITS_64 +# ifdef SNMALLOC_VA_BITS_64 return _rotr64(x, (int)n); # else return _rotr((uint32_t)x, (int)n); @@ -186,7 +110,7 @@ namespace snmalloc inline size_t rotl(size_t x, size_t n) { #if defined(_MSC_VER) -# ifdef PLATFORM_BITS_64 +# ifdef SNMALLOC_VA_BITS_64 return _rotl64(x, (int)n); # else return _rotl((uint32_t)x, (int)n); @@ -216,7 +140,7 @@ namespace snmalloc inline size_t ctz(size_t x) { #if defined(_MSC_VER) -# ifdef PLATFORM_BITS_64 +# ifdef SNMALLOC_VA_BITS_64 return _tzcnt_u64(x); # else return _tzcnt_u32((uint32_t)x); @@ -250,7 +174,7 @@ namespace snmalloc overflow = __builtin_mul_overflow(x, y, &prod); return prod; #elif defined(_MSC_VER) -# if defined(PLATFORM_BITS_64) +# if defined(SNMALLOC_VA_BITS_64) size_t high_prod; size_t prod = _umul128(x, y, &high_prod); overflow = high_prod != 0; diff --git a/src/ds/defines.h b/src/ds/defines.h index ab20ad6..1e80a5f 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -1,8 +1,6 @@ #pragma once #ifdef _MSC_VER -# include -# include # define ALWAYSINLINE __forceinline # define NOINLINE __declspec(noinline) # define HEADER_GLOBAL __declspec(selectany) @@ -14,8 +12,6 @@ #else # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) -# include -# include # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) # define SNMALLOC_SLOW_PATH NOINLINE @@ -30,25 +26,6 @@ # endif #endif -#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \ - defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) -# define PLATFORM_IS_X86 -# if defined(__linux__) && !defined(OPEN_ENCLAVE) -# include -# endif -# if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) -# define PLATFORM_BITS_64 -# else -# define PLATFORM_BITS_32 -# endif -#endif - -#if defined(_MSC_VER) && defined(PLATFORM_BITS_32) -# include -#endif - #ifndef __has_builtin # define __has_builtin(x) 0 #endif diff --git a/src/ds/flaglock.h b/src/ds/flaglock.h index 341471c..06d5337 100644 --- a/src/ds/flaglock.h +++ b/src/ds/flaglock.h @@ -13,7 +13,7 @@ namespace snmalloc FlagLock(std::atomic_flag& lock) : lock(lock) { while (lock.test_and_set(std::memory_order_acquire)) - bits::pause(); + AAL::pause(); } ~FlagLock() diff --git a/src/ds/mpscq.h b/src/ds/mpscq.h index 1b34a91..9785ff9 100644 --- a/src/ds/mpscq.h +++ b/src/ds/mpscq.h @@ -70,7 +70,7 @@ namespace snmalloc if (next != nullptr) { front = next; - bits::prefetch(&(next->next)); + AAL::prefetch(&(next->next)); assert(front); std::atomic_thread_fence(std::memory_order_acquire); invariant(); diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h index 83e0b2e..5519985 100644 --- a/src/mem/allocstats.h +++ b/src/mem/allocstats.h @@ -65,7 +65,7 @@ namespace snmalloc { CurrentMaxPair count; CurrentMaxPair slab_count; - uint64_t time = bits::tick(); + uint64_t time = AAL::tick(); uint64_t ticks = 0; double online_average = 0; @@ -82,7 +82,7 @@ namespace snmalloc void addToRunningAverage() { - uint64_t now = bits::tick(); + uint64_t now = AAL::tick(); if (slab_count.current != 0) { diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index 0c264c4..d69b951 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -154,7 +154,7 @@ namespace snmalloc while (address_cast(e->load(std::memory_order_relaxed)) == LOCKED_ENTRY) { - bits::pause(); + AAL::pause(); } value = e->load(std::memory_order_acquire); } diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index a17999e..11cd631 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -30,18 +30,18 @@ private: auto prev = ready.fetch_add(1); if (prev + 1 == cores) { - start = bits::tick(); + start = AAL::tick(); flag = true; } while (!flag) - bits::pause(); + AAL::pause(); f(id); prev = complete.fetch_add(1); if (prev + 1 == cores) { - end = bits::tick(); + end = AAL::tick(); } }