Add an Architecture Abstraction Layer.

Currently, we support one architecture, but this provides a layer for
adding other architectures without adding more nested `#ifdef`s.

Fixes #42

src/aal/aal.h

@@ -0,0 +1,86 @@
+#pragma once
+#include "ds/defines.h"
+
+#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \
+  defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
+  defined(_M_AMD64)
+#  define PLATFORM_IS_X86
+#endif
+
+namespace snmalloc
+{
+  /**
+   * Architecture Abstraction Layer. Includes default implementations of some
+   * functions using compiler builtins.  Falls back to the definitions in the
+   * platform's AAL if the builtin does not exist.
+   */
+  template<class Arch>
+  struct AAL_Generic : Arch
+  {
+    /**
+     * Prefetch a specific address.
+     *
+     * If the compiler provides a portable prefetch builtin, use it directly,
+     * otherwise delegate to the architecture-specific layer.  This allows new
+     * architectures to avoid needing to implement a custom `prefetch` method
+     * if they are used only with a compiler that provides the builtin.
+     */
+    static inline void prefetch(void* ptr)
+    {
+#if __has_builtin(__builtin_prefetch) && !defined(SNMALLOC_NO_AAL_BUILTINS)
+      __builtin_prefetch(ptr);
+#else
+      Arch::prefetch(ptr);
+#endif
+    }
+
+    /**
+     * Return an architecture-specific cycle counter.
+     *
+     * If the compiler provides a portable prefetch builtin, use it directly,
+     * otherwise delegate to the architecture-specific layer.  This allows new
+     * architectures to avoid needing to implement a custom `tick` method
+     * if they are used only with a compiler that provides the builtin.
+     */
+    static inline uint64_t tick()
+    {
+#if __has_builtin(__builtin_readcyclecounter) && \
+  !defined(SNMALLOC_NO_AAL_BUILTINS)
+      return __builtin_readcyclecounter();
+#else
+      return Arch::tick();
+#endif
+    }
+  };
+
+}
+
+#ifdef PLATFORM_IS_X86
+#  include "aal_x86.h"
+#endif
+
+namespace snmalloc
+{
+  using AAL = AAL_Generic<AAL_Arch>;
+}
+
+#if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_32)
+#  include <intsafe.h>
+#endif
+
+#ifdef __POINTER_WIDTH__
+#  if ((__POINTER_WIDTH__ == 64) && !defined(SNMALLOC_VA_BITS_64)) || \
+    ((__POINTER_WIDTH__ == 32) && !defined(SNMALLOC_VA_BITS_32))
+#    error Compiler and PAL define inconsistent bit widths
+#  endif
+#endif
+
+#if defined(SNMALLOC_VA_BITS_32) && defined(SNMALLOC_VA_BITS_64)
+#  error Only one of SNMALLOC_VA_BITS_64 and SNMALLOC_VA_BITS_32 may be defined!
+#endif
+
+#ifdef SNMALLOC_VA_BITS_32
+static_assert(sizeof(size_t) == 4);
+#elif defined(SNMALLOC_VA_BITS_64)
+static_assert(sizeof(size_t) == 8);
+#endif

src/aal/aal_x86.h

@@ -0,0 +1,112 @@
+#pragma once
+
+#ifdef _MSC_VER
+#  include <immintrin.h>
+#  include <intrin.h>
+#else
+#  include <cpuid.h>
+#  include <emmintrin.h>
+#endif
+
+#if defined(__linux__) && !defined(OPEN_ENCLAVE)
+#  include <x86intrin.h>
+#endif
+
+#if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
+  defined(_M_AMD64)
+#  define SNMALLOC_VA_BITS_64
+#else
+#  define SNMALLOC_VA_BITS_32
+#endif
+
+namespace snmalloc
+{
+  /**
+   * x86-specific architecture abstraction layer.
+   */
+  class AAL_x86
+  {
+    /**
+     * Read the timestamp counter, guaranteeing that all previous instructions
+     * have been retired.
+     */
+    static inline uint64_t tickp()
+    {
+      unsigned int aux;
+#if defined(_MSC_VER)
+      return __rdtscp(&aux);
+#else
+      return __builtin_ia32_rdtscp(&aux);
+#endif
+    }
+
+    /**
+     * Issue a fully serialising instruction.
+     */
+    static inline void halt_out_of_order()
+    {
+#if defined(_MSC_VER)
+      int cpu_info[4];
+      __cpuid(cpu_info, 0);
+#else
+      unsigned int eax, ebx, ecx, edx;
+      __get_cpuid(0, &eax, &ebx, &ecx, &edx);
+#endif
+    }
+
+  public:
+    /**
+     * On pipelined processors, notify the core that we are in a spin loop and
+     * that speculative execution past this point may not be a performance gain.
+     */
+    static inline void pause()
+    {
+      _mm_pause();
+    }
+
+    /**
+     * Issue a prefetch hint at the specified address.
+     */
+    static inline void prefetch(void* ptr)
+    {
+      _mm_prefetch(reinterpret_cast<const char*>(ptr), _MM_HINT_T0);
+    }
+
+    /**
+     * Return a cycle counter value.
+     */
+    static inline uint64_t tick()
+    {
+#if defined(_MSC_VER)
+      return __rdtsc();
+#else
+      return __builtin_ia32_rdtsc();
+#endif
+    }
+
+    /**
+     * Return the cycle counter value that can be used to indicate the start of
+     * a benchmark run.  This is responsible for ensuring any serialisation
+     * that the pipeline may require.
+     */
+    static inline uint64_t benchmark_time_start()
+    {
+      halt_out_of_order();
+      return AAL_Generic<AAL_x86>::tick();
+    }
+
+    /**
+     * Return the cycle counter value that can be used to indicate the end of a
+     * benchmark run.  This is responsible for ensuring any serialisation that
+     * the pipeline may require.
+     */
+    static inline uint64_t benchmark_time_end()
+    {
+      uint64_t t = tickp();
+      halt_out_of_order();
+      return t;
+    }
+  };
+
+  using AAL_Arch = AAL_x86;
+}

src/ds/aba.h

@@ -89,7 +89,7 @@ namespace snmalloc
     bool compare_exchange(Cmp& expect, T* value)
     {
 #ifdef PLATFORM_IS_X86
-#  if defined(_MSC_VER) && defined(PLATFORM_BITS_64)
+#  if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_64)
       return _InterlockedCompareExchange128(
         (volatile __int64*)&linked,
         expect.aba + 1,

src/ds/bits.h

@@ -3,9 +3,9 @@
 #include <limits>
 #include <stddef.h>
 
-
 // #define USE_LZCNT
 
+#include "aal/aal.h"
 #include "address.h"
 #include "defines.h"
 
@@ -55,87 +55,11 @@ namespace snmalloc
 
     static constexpr size_t ADDRESS_BITS = is64() ? 48 : 32;
 
-    inline void pause()
-    {
-#if defined(PLATFORM_IS_X86)
-      _mm_pause();
-#else
-#  warning "Missing pause intrinsic"
-#endif
-    }
-
-    inline void prefetch(void* ptr)
-    {
-#if defined(PLATFORM_IS_X86)
-      _mm_prefetch(reinterpret_cast<const char*>(ptr), _MM_HINT_T0);
-#else
-#  warning "Missing prefetch intrinsic"
-#endif
-    }
-
-    inline uint64_t tick()
-    {
-#if defined(PLATFORM_IS_X86)
-#  if defined(_MSC_VER)
-      return __rdtsc();
-#  elif defined(__clang__)
-      return __builtin_readcyclecounter();
-#  else
-      return __builtin_ia32_rdtsc();
-#  endif
-#else
-#  error Define CPU tick for this platform
-#endif
-    }
-
-    inline uint64_t tickp()
-    {
-#if defined(PLATFORM_IS_X86)
-#  if defined(_MSC_VER)
-      unsigned int aux;
-      return __rdtscp(&aux);
-#  else
-      unsigned aux;
-      return __builtin_ia32_rdtscp(&aux);
-#  endif
-#else
-#  error Define CPU tick for this platform
-#endif
-    }
-
-    inline void halt_out_of_order()
-    {
-#if defined(PLATFORM_IS_X86)
-#  if defined(_MSC_VER)
-      int cpu_info[4];
-      __cpuid(cpu_info, 0);
-#  else
-      unsigned int eax, ebx, ecx, edx;
-      __get_cpuid(0, &eax, &ebx, &ecx, &edx);
-#  endif
-#else
-#  error Define CPU benchmark start time for this platform
-#endif
-    }
-
-    inline uint64_t benchmark_time_start()
-    {
-      halt_out_of_order();
-      return tick();
-    }
-
-    inline uint64_t benchmark_time_end()
-    {
-      uint64_t t = tickp();
-      halt_out_of_order();
-      return t;
-    }
-
     inline size_t clz(size_t x)
     {
 #if defined(_MSC_VER)
 #  ifdef USE_LZCNT
-#    ifdef PLATFORM_BITS_64
+#    ifdef SNMALLOC_VA_BITS_64
       return __lzcnt64(x);
 #    else
       return __lzcnt((uint32_t)x);
@@ -143,7 +67,7 @@ namespace snmalloc
 #  else
       unsigned long index;
 
-#    ifdef PLATFORM_BITS_64
+#    ifdef SNMALLOC_VA_BITS_64
       _BitScanReverse64(&index, x);
 #    else
       _BitScanReverse(&index, (unsigned long)x);
@@ -173,7 +97,7 @@ namespace snmalloc
     inline size_t rotr(size_t x, size_t n)
     {
 #if defined(_MSC_VER)
-#  ifdef PLATFORM_BITS_64
+#  ifdef SNMALLOC_VA_BITS_64
       return _rotr64(x, (int)n);
 #  else
       return _rotr((uint32_t)x, (int)n);
@@ -186,7 +110,7 @@ namespace snmalloc
     inline size_t rotl(size_t x, size_t n)
     {
 #if defined(_MSC_VER)
-#  ifdef PLATFORM_BITS_64
+#  ifdef SNMALLOC_VA_BITS_64
       return _rotl64(x, (int)n);
 #  else
       return _rotl((uint32_t)x, (int)n);
@@ -216,7 +140,7 @@ namespace snmalloc
     inline size_t ctz(size_t x)
     {
 #if defined(_MSC_VER)
-#  ifdef PLATFORM_BITS_64
+#  ifdef SNMALLOC_VA_BITS_64
       return _tzcnt_u64(x);
 #  else
       return _tzcnt_u32((uint32_t)x);
@@ -250,7 +174,7 @@ namespace snmalloc
       overflow = __builtin_mul_overflow(x, y, &prod);
       return prod;
 #elif defined(_MSC_VER)
-#  if defined(PLATFORM_BITS_64)
+#  if defined(SNMALLOC_VA_BITS_64)
       size_t high_prod;
       size_t prod = _umul128(x, y, &high_prod);
       overflow = high_prod != 0;

src/ds/defines.h

@@ -1,8 +1,6 @@
 #pragma once
 
 #ifdef _MSC_VER
-#  include <immintrin.h>
-#  include <intrin.h>
 #  define ALWAYSINLINE __forceinline
 #  define NOINLINE __declspec(noinline)
 #  define HEADER_GLOBAL __declspec(selectany)
@@ -14,8 +12,6 @@
 #else
 #  define likely(x) __builtin_expect(!!(x), 1)
 #  define unlikely(x) __builtin_expect(!!(x), 0)
-#  include <cpuid.h>
-#  include <emmintrin.h>
 #  define ALWAYSINLINE __attribute__((always_inline))
 #  define NOINLINE __attribute__((noinline))
 #  define SNMALLOC_SLOW_PATH NOINLINE
@@ -30,25 +26,6 @@
 #  endif
 #endif
 
-#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \
-  defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
-  defined(_M_AMD64)
-#  define PLATFORM_IS_X86
-#  if defined(__linux__) && !defined(OPEN_ENCLAVE)
-#    include <x86intrin.h>
-#  endif
-#  if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
-    defined(_M_AMD64)
-#    define PLATFORM_BITS_64
-#  else
-#    define PLATFORM_BITS_32
-#  endif
-#endif
-
-#if defined(_MSC_VER) && defined(PLATFORM_BITS_32)
-#  include <intsafe.h>
-#endif
-
 #ifndef __has_builtin
 #  define __has_builtin(x) 0
 #endif

src/ds/flaglock.h

@@ -13,7 +13,7 @@ namespace snmalloc
     FlagLock(std::atomic_flag& lock) : lock(lock)
     {
       while (lock.test_and_set(std::memory_order_acquire))
-        bits::pause();
+        AAL::pause();
     }
 
     ~FlagLock()

src/ds/mpscq.h

@@ -70,7 +70,7 @@ namespace snmalloc
       if (next != nullptr)
       {
         front = next;
-        bits::prefetch(&(next->next));
+        AAL::prefetch(&(next->next));
         assert(front);
         std::atomic_thread_fence(std::memory_order_acquire);
         invariant();

src/mem/allocstats.h

@@ -65,7 +65,7 @@ namespace snmalloc
     {
       CurrentMaxPair count;
       CurrentMaxPair slab_count;
-      uint64_t time = bits::tick();
+      uint64_t time = AAL::tick();
       uint64_t ticks = 0;
       double online_average = 0;
 
@@ -82,7 +82,7 @@ namespace snmalloc
 
       void addToRunningAverage()
       {
-        uint64_t now = bits::tick();
+        uint64_t now = AAL::tick();
 
         if (slab_count.current != 0)
         {

src/mem/pagemap.h

@@ -154,7 +154,7 @@ namespace snmalloc
           while (address_cast(e->load(std::memory_order_relaxed)) ==
                  LOCKED_ENTRY)
           {
-            bits::pause();
+            AAL::pause();
           }
           value = e->load(std::memory_order_acquire);
         }

src/test/perf/contention/contention.cc

@@ -30,18 +30,18 @@ private:
     auto prev = ready.fetch_add(1);
     if (prev + 1 == cores)
     {
-      start = bits::tick();
+      start = AAL::tick();
       flag = true;
     }
     while (!flag)
-      bits::pause();
+      AAL::pause();
 
     f(id);
 
     prev = complete.fetch_add(1);
     if (prev + 1 == cores)
     {
-      end = bits::tick();
+      end = AAL::tick();
     }
   }