Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #69 from microsoft/aal

Browse Source 
Aal
This commit is contained in:
David Chisnall
2019-07-10 18:36:42 +01:00
committed by GitHub
parent c35a3941fe bc34e5abe5
commit 1e3f95152e
11 changed files with 269 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
azure-pipelines.yml
View File

@@ -95,22 +95,27 @@ jobs:
64-bit Debug:
BuildType: Debug
CMakeArgs: '-G"Visual Studio 15 2017 Win64"'
JFlag: -j 4
64-bit Release:
BuildType: Release
CMakeArgs: '-G"Visual Studio 15 2017 Win64"'
JFlag: '-j 4'
64-bit Release Windows8Compat:
BuildType: Release
CMakeArgs: '-G"Visual Studio 15 2017 Win64" -DWIN8COMPAT=TRUE'
JFlag: '-j 2'
32-bit Debug:
BuildType: Debug
CMakeArgs: '-G"Visual Studio 15 2017"'
JFlag: '-j 4'
32-bit Release:
BuildType: Release
CMakeArgs: '-G"Visual Studio 15 2017"'
JFlag: '-j 4'
steps:
- task: CMake@1
@@ -124,7 +129,7 @@ jobs:
solution: build/snmalloc.sln
msbuildArguments: '/m /p:Configuration=$(BuildType)'
- script: 'ctest -j 4 --interactive-debug-mode 0 --output-on-failure -C $(BuildType)'
- script: 'ctest $(JFlag) --interactive-debug-mode 0 --output-on-failure -C $(BuildType)'
workingDirectory: build
displayName: 'Run Ctest'

88
src/aal/aal.h Normal file
View File

@@ -0,0 +1,88 @@
#pragma once
#include "../ds/defines.h"
#include <cstdint>
#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \
defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
defined(_M_AMD64)
# define PLATFORM_IS_X86
#endif
namespace snmalloc
{
/**
* Architecture Abstraction Layer. Includes default implementations of some
* functions using compiler builtins. Falls back to the definitions in the
* platform's AAL if the builtin does not exist.
*/
template<class Arch>
struct AAL_Generic : Arch
{
/**
* Prefetch a specific address.
*
* If the compiler provides a portable prefetch builtin, use it directly,
* otherwise delegate to the architecture-specific layer. This allows new
* architectures to avoid needing to implement a custom `prefetch` method
* if they are used only with a compiler that provides the builtin.
*/
static inline void prefetch(void* ptr)
{
#if __has_builtin(__builtin_prefetch) && !defined(SNMALLOC_NO_AAL_BUILTINS)
__builtin_prefetch(ptr);
#else
Arch::prefetch(ptr);
#endif
}
/**
* Return an architecture-specific cycle counter.
*
* If the compiler provides a portable prefetch builtin, use it directly,
* otherwise delegate to the architecture-specific layer. This allows new
* architectures to avoid needing to implement a custom `tick` method
* if they are used only with a compiler that provides the builtin.
*/
static inline uint64_t tick()
{
#if __has_builtin(__builtin_readcyclecounter) && \
!defined(SNMALLOC_NO_AAL_BUILTINS)
return __builtin_readcyclecounter();
#else
return Arch::tick();
#endif
}
};
} // namespace snmalloc
#ifdef PLATFORM_IS_X86
# include "aal_x86.h"
#endif
namespace snmalloc
{
using AAL = AAL_Generic<AAL_Arch>;
} // namespace snmalloc
#if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_32)
# include <intsafe.h>
#endif
#ifdef __POINTER_WIDTH__
# if ((__POINTER_WIDTH__ == 64) && !defined(SNMALLOC_VA_BITS_64)) || \
((__POINTER_WIDTH__ == 32) && !defined(SNMALLOC_VA_BITS_32))
# error Compiler and PAL define inconsistent bit widths
# endif
#endif
#if defined(SNMALLOC_VA_BITS_32) && defined(SNMALLOC_VA_BITS_64)
# error Only one of SNMALLOC_VA_BITS_64 and SNMALLOC_VA_BITS_32 may be defined!
#endif
#ifdef SNMALLOC_VA_BITS_32
static_assert(sizeof(size_t) == 4);
#elif defined(SNMALLOC_VA_BITS_64)
static_assert(sizeof(size_t) == 8);
#endif

112
src/aal/aal_x86.h Normal file
View File

@@ -0,0 +1,112 @@
#pragma once
#ifdef _MSC_VER
# include <immintrin.h>
# include <intrin.h>
#else
# include <cpuid.h>
# include <emmintrin.h>
#endif
#if defined(__linux__) && !defined(OPEN_ENCLAVE)
# include <x86intrin.h>
#endif
#if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
defined(_M_AMD64)
# define SNMALLOC_VA_BITS_64
#else
# define SNMALLOC_VA_BITS_32
#endif
namespace snmalloc
{
/**
* x86-specific architecture abstraction layer.
*/
class AAL_x86
{
/**
* Read the timestamp counter, guaranteeing that all previous instructions
* have been retired.
*/
static inline uint64_t tickp()
{
unsigned int aux;
#if defined(_MSC_VER)
return __rdtscp(&aux);
#else
return __builtin_ia32_rdtscp(&aux);
#endif
}
/**
* Issue a fully serialising instruction.
*/
static inline void halt_out_of_order()
{
#if defined(_MSC_VER)
int cpu_info[4];
__cpuid(cpu_info, 0);
#else
unsigned int eax, ebx, ecx, edx;
__get_cpuid(0, &eax, &ebx, &ecx, &edx);
#endif
}
public:
/**
* On pipelined processors, notify the core that we are in a spin loop and
* that speculative execution past this point may not be a performance gain.
*/
static inline void pause()
{
_mm_pause();
}
/**
* Issue a prefetch hint at the specified address.
*/
static inline void prefetch(void* ptr)
{
_mm_prefetch(reinterpret_cast<const char*>(ptr), _MM_HINT_T0);
}
/**
* Return a cycle counter value.
*/
static inline uint64_t tick()
{
#if defined(_MSC_VER)
return __rdtsc();
#else
return __builtin_ia32_rdtsc();
#endif
}
/**
* Return the cycle counter value that can be used to indicate the start of
* a benchmark run. This is responsible for ensuring any serialisation
* that the pipeline may require.
*/
static inline uint64_t benchmark_time_start()
{
halt_out_of_order();
return AAL_Generic<AAL_x86>::tick();
}
/**
* Return the cycle counter value that can be used to indicate the end of a
* benchmark run. This is responsible for ensuring any serialisation that
* the pipeline may require.
*/
static inline uint64_t benchmark_time_end()
{
uint64_t t = tickp();
halt_out_of_order();
return t;
}
};
using AAL_Arch = AAL_x86;
} // namespace snmalloc

2
src/ds/aba.h
View File

@@ -89,7 +89,7 @@ namespace snmalloc
bool compare_exchange(Cmp& expect, T* value)
{
#ifdef PLATFORM_IS_X86
# if defined(_MSC_VER) && defined(PLATFORM_BITS_64)
# if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_64)
return _InterlockedCompareExchange128(
(volatile __int64*)&linked,
expect.aba + 1,

158
src/ds/bits.h
View File

@@ -3,77 +3,11 @@
#include <limits>
#include <stddef.h>
#ifdef _MSC_VER
# include <immintrin.h>
# include <intrin.h>
# define ALWAYSINLINE __forceinline
# define NOINLINE __declspec(noinline)
# define HEADER_GLOBAL __declspec(selectany)
# define likely(x) !!(x)
# define unlikely(x) !!(x)
# define SNMALLOC_SLOW_PATH NOINLINE
# define SNMALLOC_FAST_PATH ALWAYSINLINE
# define SNMALLOC_PURE
#else
# define likely(x) __builtin_expect(!!(x), 1)
# define unlikely(x) __builtin_expect(!!(x), 0)
# include <cpuid.h>
# include <emmintrin.h>
# define ALWAYSINLINE __attribute__((always_inline))
# define NOINLINE __attribute__((noinline))
# define SNMALLOC_SLOW_PATH NOINLINE
# define SNMALLOC_FAST_PATH inline ALWAYSINLINE
# define SNMALLOC_PURE __attribute__((const))
# ifdef __clang__
# define HEADER_GLOBAL __attribute__((selectany))
# else
// GCC does not support selectany, weak is almost the correct
// attribute, but leaves the global variable preemptible.
# define HEADER_GLOBAL __attribute__((weak))
# endif
#endif
#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \
defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
defined(_M_AMD64)
# define PLATFORM_IS_X86
# if defined(__linux__) && !defined(OPEN_ENCLAVE)
# include <x86intrin.h>
# endif
# if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
defined(_M_AMD64)
# define PLATFORM_BITS_64
# else
# define PLATFORM_BITS_32
# endif
#endif
#if defined(_MSC_VER) && defined(PLATFORM_BITS_32)
# include <intsafe.h>
#endif
#ifndef __has_builtin
# define __has_builtin(x) 0
#endif
#define UNUSED(x) ((void)(x))
#ifndef NDEBUG
# define SNMALLOC_ASSUME(x) assert(x)
#else
# if __has_builtin(__builtin_assume)
# define SNMALLOC_ASSUME(x) __builtin_assume((x))
# else
# define SNMALLOC_ASSUME(x) \
do \
{ \
} while (0)
# endif
#endif
// #define USE_LZCNT
#include "../aal/aal.h"
#include "address.h"
#include "defines.h"
#include <atomic>
#include <cassert>
@@ -121,87 +55,11 @@ namespace snmalloc
static constexpr size_t ADDRESS_BITS = is64() ? 48 : 32;
inline void pause()
{
#if defined(PLATFORM_IS_X86)
_mm_pause();
#else
# warning "Missing pause intrinsic"
#endif
}
inline void prefetch(void* ptr)
{
#if defined(PLATFORM_IS_X86)
_mm_prefetch(reinterpret_cast<const char*>(ptr), _MM_HINT_T0);
#else
# warning "Missing prefetch intrinsic"
#endif
}
inline uint64_t tick()
{
#if defined(PLATFORM_IS_X86)
# if defined(_MSC_VER)
return __rdtsc();
# elif defined(__clang__)
return __builtin_readcyclecounter();
# else
return __builtin_ia32_rdtsc();
# endif
#else
# error Define CPU tick for this platform
#endif
}
inline uint64_t tickp()
{
#if defined(PLATFORM_IS_X86)
# if defined(_MSC_VER)
unsigned int aux;
return __rdtscp(&aux);
# else
unsigned aux;
return __builtin_ia32_rdtscp(&aux);
# endif
#else
# error Define CPU tick for this platform
#endif
}
inline void halt_out_of_order()
{
#if defined(PLATFORM_IS_X86)
# if defined(_MSC_VER)
int cpu_info[4];
__cpuid(cpu_info, 0);
# else
unsigned int eax, ebx, ecx, edx;
__get_cpuid(0, &eax, &ebx, &ecx, &edx);
# endif
#else
# error Define CPU benchmark start time for this platform
#endif
}
inline uint64_t benchmark_time_start()
{
halt_out_of_order();
return tick();
}
inline uint64_t benchmark_time_end()
{
uint64_t t = tickp();
halt_out_of_order();
return t;
}
inline size_t clz(size_t x)
{
#if defined(_MSC_VER)
# ifdef USE_LZCNT
# ifdef PLATFORM_BITS_64
# ifdef SNMALLOC_VA_BITS_64
return __lzcnt64(x);
# else
return __lzcnt((uint32_t)x);
@@ -209,7 +67,7 @@ namespace snmalloc
# else
unsigned long index;
# ifdef PLATFORM_BITS_64
# ifdef SNMALLOC_VA_BITS_64
_BitScanReverse64(&index, x);
# else
_BitScanReverse(&index, (unsigned long)x);
@@ -239,7 +97,7 @@ namespace snmalloc
inline size_t rotr(size_t x, size_t n)
{
#if defined(_MSC_VER)
# ifdef PLATFORM_BITS_64
# ifdef SNMALLOC_VA_BITS_64
return _rotr64(x, (int)n);
# else
return _rotr((uint32_t)x, (int)n);
@@ -252,7 +110,7 @@ namespace snmalloc
inline size_t rotl(size_t x, size_t n)
{
#if defined(_MSC_VER)
# ifdef PLATFORM_BITS_64
# ifdef SNMALLOC_VA_BITS_64
return _rotl64(x, (int)n);
# else
return _rotl((uint32_t)x, (int)n);
@@ -282,7 +140,7 @@ namespace snmalloc
inline size_t ctz(size_t x)
{
#if defined(_MSC_VER)
# ifdef PLATFORM_BITS_64
# ifdef SNMALLOC_VA_BITS_64
return _tzcnt_u64(x);
# else
return _tzcnt_u32((uint32_t)x);
@@ -316,7 +174,7 @@ namespace snmalloc
overflow = __builtin_mul_overflow(x, y, &prod);
return prod;
#elif defined(_MSC_VER)
# if defined(PLATFORM_BITS_64)
# if defined(SNMALLOC_VA_BITS_64)
size_t high_prod;
size_t prod = _umul128(x, y, &high_prod);
overflow = high_prod != 0;

46
src/ds/defines.h Normal file
View File

@@ -0,0 +1,46 @@
#pragma once
#ifdef _MSC_VER
# define ALWAYSINLINE __forceinline
# define NOINLINE __declspec(noinline)
# define HEADER_GLOBAL __declspec(selectany)
# define likely(x) !!(x)
# define unlikely(x) !!(x)
# define SNMALLOC_SLOW_PATH NOINLINE
# define SNMALLOC_FAST_PATH ALWAYSINLINE
# define SNMALLOC_PURE
#else
# define likely(x) __builtin_expect(!!(x), 1)
# define unlikely(x) __builtin_expect(!!(x), 0)
# define ALWAYSINLINE __attribute__((always_inline))
# define NOINLINE __attribute__((noinline))
# define SNMALLOC_SLOW_PATH NOINLINE
# define SNMALLOC_FAST_PATH inline ALWAYSINLINE
# define SNMALLOC_PURE __attribute__((const))
# ifdef __clang__
# define HEADER_GLOBAL __attribute__((selectany))
# else
// GCC does not support selectany, weak is almost the correct
// attribute, but leaves the global variable preemptible.
# define HEADER_GLOBAL __attribute__((weak))
# endif
#endif
#ifndef __has_builtin
# define __has_builtin(x) 0
#endif
#define UNUSED(x) ((void)(x))
#ifndef NDEBUG
# define SNMALLOC_ASSUME(x) assert(x)
#else
# if __has_builtin(__builtin_assume)
# define SNMALLOC_ASSUME(x) __builtin_assume((x))
# else
# define SNMALLOC_ASSUME(x) \
do \
{ \
} while (0)
# endif
#endif

2
src/ds/flaglock.h
View File

@@ -13,7 +13,7 @@ namespace snmalloc
FlagLock(std::atomic_flag& lock) : lock(lock)
{
while (lock.test_and_set(std::memory_order_acquire))
bits::pause();
AAL::pause();
}
~FlagLock()

2
src/ds/mpscq.h
View File

@@ -70,7 +70,7 @@ namespace snmalloc
if (next != nullptr)
{
front = next;
bits::prefetch(&(next->next));
AAL::prefetch(&(next->next));
assert(front);
std::atomic_thread_fence(std::memory_order_acquire);
invariant();

4
src/mem/allocstats.h
View File

@@ -65,7 +65,7 @@ namespace snmalloc
{
CurrentMaxPair count;
CurrentMaxPair slab_count;
uint64_t time = bits::tick();
uint64_t time = AAL::tick();
uint64_t ticks = 0;
double online_average = 0;
@@ -82,7 +82,7 @@ namespace snmalloc
void addToRunningAverage()
{
uint64_t now = bits::tick();
uint64_t now = AAL::tick();
if (slab_count.current != 0)
{

2
src/mem/pagemap.h
View File

@@ -154,7 +154,7 @@ namespace snmalloc
while (address_cast(e->load(std::memory_order_relaxed)) ==
LOCKED_ENTRY)
{
bits::pause();
AAL::pause();
}
value = e->load(std::memory_order_acquire);
}

6
src/test/perf/contention/contention.cc
View File

@@ -30,18 +30,18 @@ private:
auto prev = ready.fetch_add(1);
if (prev + 1 == cores)
{
start = bits::tick();
start = AAL::tick();
flag = true;
}
while (!flag)
bits::pause();
AAL::pause();
f(id);
prev = complete.fetch_add(1);
if (prev + 1 == cores)
{
end = bits::tick();
end = AAL::tick();
}
}