Improve the memcpy implementation. (#466)
This now either outperforms, or performs as well within the region of measurement noise as, FreeBSD's libc memcpy, which is hand-written assembly. This uses a jump table for small copies with a sequence of power-of-two loads and stores for each, a vector-register copy with an overlapping copy for the last chunk for medium copies and, on x86, rep movsb for large copies. The checked version still incurs a noticeable overhead.
This commit is contained in:
David Chisnall
@@ -5,23 +5,6 @@ using namespace snmalloc;
|
||||
|
||||
namespace
|
||||
{
|
||||
/**
|
||||
* The largest register size that we can use for loads and stores. These
|
||||
* types are expected to work for overlapping copies: we can always load them
|
||||
* into a register and store them. Note that this is at the C abstract
|
||||
* machine level: the compiler may spill temporaries to the stack, just not
|
||||
* to the source or destination object.
|
||||
*/
|
||||
static constexpr size_t LargestRegisterSize =
|
||||
#ifdef __AVX__
|
||||
32
|
||||
#elif defined(__SSE__)
|
||||
16
|
||||
#else
|
||||
sizeof(uint64_t)
|
||||
#endif
|
||||
;
|
||||
|
||||
/**
|
||||
* Copy a single element of a specified size. Uses a compiler builtin that
|
||||
* expands to a single load and store.
|
||||
@@ -51,7 +34,7 @@ namespace
|
||||
* Copy a block using the specified size. This copies as many complete
|
||||
* chunks of size `Size` as are possible from `len`.
|
||||
*/
|
||||
template<size_t Size>
|
||||
template<size_t Size, size_t PrefetchOffset = 0>
|
||||
SNMALLOC_FAST_PATH_INLINE void
|
||||
block_copy(void* dst, const void* src, size_t len)
|
||||
{
|
||||
@@ -79,18 +62,188 @@ namespace
|
||||
* aligned to be copied as aligned chunks of `Size` bytes.
|
||||
*/
|
||||
template<size_t Size>
|
||||
SNMALLOC_FAST_PATH bool is_aligned_memcpy(void* dst, const void* src)
|
||||
SNMALLOC_FAST_PATH_INLINE bool is_aligned_memcpy(void* dst, const void* src)
|
||||
{
|
||||
return (pointer_align_down<Size>(const_cast<void*>(src)) == src) &&
|
||||
(pointer_align_down<Size>(dst) == dst);
|
||||
}
|
||||
|
||||
/**
|
||||
* Snmalloc checked memcpy.
|
||||
* Copy a small size (`Size` bytes) as a sequence of power-of-two-sized loads
|
||||
* and stores of decreasing size. `Word` is the largest size to attempt for a
|
||||
* single copy.
|
||||
*/
|
||||
template<bool checked>
|
||||
template<size_t Size, size_t Word>
|
||||
SNMALLOC_FAST_PATH_INLINE void small_copy(void* dst, const void* src)
|
||||
{
|
||||
static_assert(bits::is_pow2(Word), "Word size must be a power of two!");
|
||||
if constexpr (Size != 0)
|
||||
{
|
||||
if constexpr (Size >= Word)
|
||||
{
|
||||
copy_one<Word>(dst, src);
|
||||
small_copy<Size - Word, Word>(
|
||||
pointer_offset(dst, Word), pointer_offset(src, Word));
|
||||
}
|
||||
else
|
||||
{
|
||||
small_copy<Size, Word / 2>(dst, src);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
UNUSED(src);
|
||||
UNUSED(dst);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate small copies for all sizes up to `Size`, using `WordSize` as the
|
||||
* largest size to copy in a single operation.
|
||||
*/
|
||||
template<size_t Size, size_t WordSize = Size>
|
||||
SNMALLOC_FAST_PATH_INLINE void
|
||||
small_copies(void* dst, const void* src, size_t len)
|
||||
{
|
||||
if (len == Size)
|
||||
{
|
||||
small_copy<Size, WordSize>(dst, src);
|
||||
}
|
||||
if constexpr (Size > 0)
|
||||
{
|
||||
small_copies<Size - 1, WordSize>(dst, src, len);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* If the source and destination are the same displacement away from being
|
||||
* aligned on a `BlockSize` boundary, do a small copy to ensure alignment and
|
||||
* update `src`, `dst`, and `len` to reflect the remainder that needs
|
||||
* copying.
|
||||
*
|
||||
* Note that this, like memcpy, requires that the source and destination do
|
||||
* not overlap. It unconditionally copies `BlockSize` bytes, so a subsequent
|
||||
* copy may not do the right thing.
|
||||
*/
|
||||
template<size_t BlockSize, size_t WordSize>
|
||||
SNMALLOC_FAST_PATH_INLINE void
|
||||
unaligned_start(void*& dst, const void*& src, size_t& len)
|
||||
{
|
||||
constexpr size_t block_mask = BlockSize - 1;
|
||||
size_t src_addr = static_cast<size_t>(reinterpret_cast<uintptr_t>(src));
|
||||
size_t dst_addr = static_cast<size_t>(reinterpret_cast<uintptr_t>(dst));
|
||||
size_t src_offset = src_addr & block_mask;
|
||||
if ((src_offset > 0) && (src_offset == (dst_addr & block_mask)))
|
||||
{
|
||||
size_t disp = BlockSize - src_offset;
|
||||
small_copies<BlockSize, WordSize>(dst, src, disp);
|
||||
src = pointer_offset(src, disp);
|
||||
dst = pointer_offset(dst, disp);
|
||||
len -= disp;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Default architecture definition. Provides sane defaults.
|
||||
*/
|
||||
struct GenericArch
|
||||
{
|
||||
/**
|
||||
* The largest register size that we can use for loads and stores. These
|
||||
* types are expected to work for overlapping copies: we can always load
|
||||
* them into a register and store them. Note that this is at the C abstract
|
||||
* machine level: the compiler may spill temporaries to the stack, just not
|
||||
* to the source or destination object.
|
||||
*/
|
||||
SNMALLOC_UNUSED_FUNCTION
|
||||
static constexpr size_t LargestRegisterSize =
|
||||
std::max(sizeof(uint64_t), sizeof(void*));
|
||||
|
||||
/**
|
||||
* Hook for architecture-specific optimisations. Does nothing in the
|
||||
* default case.
|
||||
*/
|
||||
static SNMALLOC_FAST_PATH_INLINE bool copy(void*, const void*, size_t)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
/**
|
||||
* x86-64 architecture. Prefers SSE registers for small and medium copies
|
||||
* and uses `rep movsb` for large ones.
|
||||
*/
|
||||
struct X86_64Arch
|
||||
{
|
||||
/**
|
||||
* The largest register size that we can use for loads and stores. These
|
||||
* types are expected to work for overlapping copies: we can always load
|
||||
* them into a register and store them. Note that this is at the C abstract
|
||||
* machine level: the compiler may spill temporaries to the stack, just not
|
||||
* to the source or destination object.
|
||||
*
|
||||
* We set this to 16 unconditionally for now because using AVX registers
|
||||
* imposes stronger alignment requirements that seem to not be a net win.
|
||||
*/
|
||||
static constexpr size_t LargestRegisterSize = 16;
|
||||
|
||||
/**
|
||||
* Platform-specific copy hook. For large copies, use `rep movsb`.
|
||||
*/
|
||||
static SNMALLOC_FAST_PATH_INLINE bool
|
||||
copy(void* dst, const void* src, size_t len)
|
||||
{
|
||||
// The Intel optimisation manual recommends doing this for sizes >256
|
||||
// bytes on modern systems and for all sizes on very modern systems.
|
||||
// Testing shows that this is somewhat overly optimistic.
|
||||
if (SNMALLOC_UNLIKELY(len >= 512))
|
||||
{
|
||||
// Align to cache-line boundaries if possible.
|
||||
unaligned_start<64, LargestRegisterSize>(dst, src, len);
|
||||
// Bulk copy. This is aggressively optimised on modern x86 cores.
|
||||
# ifdef __GNUC__
|
||||
asm volatile("rep movsb"
|
||||
: "+S"(src), "+D"(dst), "+c"(len)
|
||||
:
|
||||
: "memory");
|
||||
# elif defined(_MSC_VER)
|
||||
__movsb(
|
||||
static_cast<unsigned char*>(dst),
|
||||
static_cast<const unsigned char*>(src),
|
||||
len);
|
||||
# else
|
||||
# error No inline assembly or rep movsb intrinsic for this compiler.
|
||||
# endif
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
using DefaultArch =
|
||||
#ifdef __x86_64__
|
||||
X86_64Arch
|
||||
#else
|
||||
GenericArch
|
||||
#endif
|
||||
;
|
||||
|
||||
/**
|
||||
* Snmalloc checked memcpy. The `Arch` parameter must provide:
|
||||
*
|
||||
* - A `size_t` value `LargestRegisterSize`, describing the largest size to
|
||||
* use for single copies.
|
||||
* - A `copy` function that takes (optionally, references to) the arguments
|
||||
* of `memcpy` and returns `true` if it performs a copy, `false`
|
||||
* otherwise. This can be used to special-case some or all sizes for a
|
||||
* particular architecture.
|
||||
*/
|
||||
template<bool checked, typename Arch = DefaultArch>
|
||||
void* memcpy(void* dst, const void* src, size_t len)
|
||||
{
|
||||
auto orig_dst = dst;
|
||||
// 0 is a very common size for memcpy and we don't need to do external
|
||||
// pointer checks if we hit it. It's also the fastest case, to encourage
|
||||
// the compiler to favour the other cases.
|
||||
@@ -107,15 +260,20 @@ namespace
|
||||
check_bounds<CheckDirection::Read>(
|
||||
src, len, "memcpy with source out of bounds of heap allocation");
|
||||
}
|
||||
// If this is a small size, do byte-by-byte copies.
|
||||
if (len < LargestRegisterSize)
|
||||
|
||||
// If this is a small size, use a jump table for small sizes.
|
||||
if (len <= Arch::LargestRegisterSize)
|
||||
{
|
||||
block_copy<1>(dst, src, len);
|
||||
return dst;
|
||||
small_copies<Arch::LargestRegisterSize>(dst, src, len);
|
||||
}
|
||||
block_copy<LargestRegisterSize>(dst, src, len);
|
||||
copy_end<LargestRegisterSize>(dst, src, len);
|
||||
return dst;
|
||||
// If there's an architecture-specific hook, try using it. Otherwise do a
|
||||
// simple bulk copy loop.
|
||||
else if (!Arch::copy(dst, src, len))
|
||||
{
|
||||
block_copy<Arch::LargestRegisterSize>(dst, src, len);
|
||||
copy_end<Arch::LargestRegisterSize>(dst, src, len);
|
||||
}
|
||||
return orig_dst;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
|
||||
Reference in New Issue
Block a user