From c73846e41808fa73552f12331a02b6b146a960fa Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Wed, 2 Mar 2022 18:11:57 +0000 Subject: [PATCH] Improve the `memcpy` implementation. (#466) This now either outperforms, or performs as well within the region of measurement noise as, FreeBSD's libc memcpy, which is hand-written assembly. This uses a jump table for small copies with a sequence of power-of-two loads and stores for each, a vector-register copy with an overlapping copy for the last chunk for medium copies and, on x86, rep movsb for large copies. The checked version still incurs a noticeable overhead. --- src/override/memcpy.cc | 214 +++++++++++++++++++++++++++++++++++------ 1 file changed, 186 insertions(+), 28 deletions(-) diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 9d96eba..617f7ec 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -5,23 +5,6 @@ using namespace snmalloc; namespace { - /** - * The largest register size that we can use for loads and stores. These - * types are expected to work for overlapping copies: we can always load them - * into a register and store them. Note that this is at the C abstract - * machine level: the compiler may spill temporaries to the stack, just not - * to the source or destination object. - */ - static constexpr size_t LargestRegisterSize = -#ifdef __AVX__ - 32 -#elif defined(__SSE__) - 16 -#else - sizeof(uint64_t) -#endif - ; - /** * Copy a single element of a specified size. Uses a compiler builtin that * expands to a single load and store. @@ -51,7 +34,7 @@ namespace * Copy a block using the specified size. This copies as many complete * chunks of size `Size` as are possible from `len`. */ - template + template SNMALLOC_FAST_PATH_INLINE void block_copy(void* dst, const void* src, size_t len) { @@ -79,18 +62,188 @@ namespace * aligned to be copied as aligned chunks of `Size` bytes. */ template - SNMALLOC_FAST_PATH bool is_aligned_memcpy(void* dst, const void* src) + SNMALLOC_FAST_PATH_INLINE bool is_aligned_memcpy(void* dst, const void* src) { return (pointer_align_down(const_cast(src)) == src) && (pointer_align_down(dst) == dst); } /** - * Snmalloc checked memcpy. + * Copy a small size (`Size` bytes) as a sequence of power-of-two-sized loads + * and stores of decreasing size. `Word` is the largest size to attempt for a + * single copy. */ - template + template + SNMALLOC_FAST_PATH_INLINE void small_copy(void* dst, const void* src) + { + static_assert(bits::is_pow2(Word), "Word size must be a power of two!"); + if constexpr (Size != 0) + { + if constexpr (Size >= Word) + { + copy_one(dst, src); + small_copy( + pointer_offset(dst, Word), pointer_offset(src, Word)); + } + else + { + small_copy(dst, src); + } + } + else + { + UNUSED(src); + UNUSED(dst); + } + } + + /** + * Generate small copies for all sizes up to `Size`, using `WordSize` as the + * largest size to copy in a single operation. + */ + template + SNMALLOC_FAST_PATH_INLINE void + small_copies(void* dst, const void* src, size_t len) + { + if (len == Size) + { + small_copy(dst, src); + } + if constexpr (Size > 0) + { + small_copies(dst, src, len); + } + } + + /** + * If the source and destination are the same displacement away from being + * aligned on a `BlockSize` boundary, do a small copy to ensure alignment and + * update `src`, `dst`, and `len` to reflect the remainder that needs + * copying. + * + * Note that this, like memcpy, requires that the source and destination do + * not overlap. It unconditionally copies `BlockSize` bytes, so a subsequent + * copy may not do the right thing. + */ + template + SNMALLOC_FAST_PATH_INLINE void + unaligned_start(void*& dst, const void*& src, size_t& len) + { + constexpr size_t block_mask = BlockSize - 1; + size_t src_addr = static_cast(reinterpret_cast(src)); + size_t dst_addr = static_cast(reinterpret_cast(dst)); + size_t src_offset = src_addr & block_mask; + if ((src_offset > 0) && (src_offset == (dst_addr & block_mask))) + { + size_t disp = BlockSize - src_offset; + small_copies(dst, src, disp); + src = pointer_offset(src, disp); + dst = pointer_offset(dst, disp); + len -= disp; + } + } + + /** + * Default architecture definition. Provides sane defaults. + */ + struct GenericArch + { + /** + * The largest register size that we can use for loads and stores. These + * types are expected to work for overlapping copies: we can always load + * them into a register and store them. Note that this is at the C abstract + * machine level: the compiler may spill temporaries to the stack, just not + * to the source or destination object. + */ + SNMALLOC_UNUSED_FUNCTION + static constexpr size_t LargestRegisterSize = + std::max(sizeof(uint64_t), sizeof(void*)); + + /** + * Hook for architecture-specific optimisations. Does nothing in the + * default case. + */ + static SNMALLOC_FAST_PATH_INLINE bool copy(void*, const void*, size_t) + { + return false; + } + }; + +#if defined(__x86_64__) || defined(_M_X64) + /** + * x86-64 architecture. Prefers SSE registers for small and medium copies + * and uses `rep movsb` for large ones. + */ + struct X86_64Arch + { + /** + * The largest register size that we can use for loads and stores. These + * types are expected to work for overlapping copies: we can always load + * them into a register and store them. Note that this is at the C abstract + * machine level: the compiler may spill temporaries to the stack, just not + * to the source or destination object. + * + * We set this to 16 unconditionally for now because using AVX registers + * imposes stronger alignment requirements that seem to not be a net win. + */ + static constexpr size_t LargestRegisterSize = 16; + + /** + * Platform-specific copy hook. For large copies, use `rep movsb`. + */ + static SNMALLOC_FAST_PATH_INLINE bool + copy(void* dst, const void* src, size_t len) + { + // The Intel optimisation manual recommends doing this for sizes >256 + // bytes on modern systems and for all sizes on very modern systems. + // Testing shows that this is somewhat overly optimistic. + if (SNMALLOC_UNLIKELY(len >= 512)) + { + // Align to cache-line boundaries if possible. + unaligned_start<64, LargestRegisterSize>(dst, src, len); + // Bulk copy. This is aggressively optimised on modern x86 cores. +# ifdef __GNUC__ + asm volatile("rep movsb" + : "+S"(src), "+D"(dst), "+c"(len) + : + : "memory"); +# elif defined(_MSC_VER) + __movsb( + static_cast(dst), + static_cast(src), + len); +# else +# error No inline assembly or rep movsb intrinsic for this compiler. +# endif + return true; + } + return false; + } + }; +#endif + + using DefaultArch = +#ifdef __x86_64__ + X86_64Arch +#else + GenericArch +#endif + ; + + /** + * Snmalloc checked memcpy. The `Arch` parameter must provide: + * + * - A `size_t` value `LargestRegisterSize`, describing the largest size to + * use for single copies. + * - A `copy` function that takes (optionally, references to) the arguments + * of `memcpy` and returns `true` if it performs a copy, `false` + * otherwise. This can be used to special-case some or all sizes for a + * particular architecture. + */ + template void* memcpy(void* dst, const void* src, size_t len) { + auto orig_dst = dst; // 0 is a very common size for memcpy and we don't need to do external // pointer checks if we hit it. It's also the fastest case, to encourage // the compiler to favour the other cases. @@ -107,15 +260,20 @@ namespace check_bounds( src, len, "memcpy with source out of bounds of heap allocation"); } - // If this is a small size, do byte-by-byte copies. - if (len < LargestRegisterSize) + + // If this is a small size, use a jump table for small sizes. + if (len <= Arch::LargestRegisterSize) { - block_copy<1>(dst, src, len); - return dst; + small_copies(dst, src, len); } - block_copy(dst, src, len); - copy_end(dst, src, len); - return dst; + // If there's an architecture-specific hook, try using it. Otherwise do a + // simple bulk copy loop. + else if (!Arch::copy(dst, src, len)) + { + block_copy(dst, src, len); + copy_end(dst, src, len); + } + return orig_dst; } } // namespace