diff --git a/src/mem/bounds_checks.h b/src/mem/bounds_checks.h index bbd1d7b..2843530 100644 --- a/src/mem/bounds_checks.h +++ b/src/mem/bounds_checks.h @@ -77,11 +77,13 @@ namespace snmalloc * The template parameter indicates whether this is a read. If so, this * function is a no-op when `CheckReads` is false. */ - template + template< + CheckDirection Direction = CheckDirection::Write, + bool CheckBoth = CheckReads> SNMALLOC_FAST_PATH_INLINE void check_bounds(const void* ptr, size_t len, const char* msg = "") { - if constexpr ((Direction == CheckDirection::Write) || CheckReads) + if constexpr ((Direction == CheckDirection::Write) || CheckBoth) { auto& alloc = ThreadAlloc::get(); void* p = const_cast(ptr); diff --git a/src/mem/memcpy.h b/src/mem/memcpy.h new file mode 100644 index 0000000..40eea5e --- /dev/null +++ b/src/mem/memcpy.h @@ -0,0 +1,279 @@ +#pragma once +#include "../mem/bounds_checks.h" + +namespace snmalloc +{ + /** + * Copy a single element of a specified size. Uses a compiler builtin that + * expands to a single load and store. + */ + template + SNMALLOC_FAST_PATH_INLINE void copy_one(void* dst, const void* src) + { +#if __has_builtin(__builtin_memcpy_inline) + __builtin_memcpy_inline(dst, src, Size); +#else + // Define a structure of size `Size` that has alignment 1 and a default + // copy-assignment operator. We can then copy the data as this type. The + // compiler knows the exact width and so will generate the correct wide + // instruction for us (clang 10 and gcc 12 both generate movups for the + // 16-byte version of this when targeting SSE. + struct Block + { + char data[Size]; + }; + auto* d = static_cast(dst); + auto* s = static_cast(src); + *d = *s; +#endif + } + + /** + * Copy a block using the specified size. This copies as many complete + * chunks of size `Size` as are possible from `len`. + */ + template + SNMALLOC_FAST_PATH_INLINE void + block_copy(void* dst, const void* src, size_t len) + { + for (size_t i = 0; (i + Size) <= len; i += Size) + { + copy_one(pointer_offset(dst, i), pointer_offset(src, i)); + } + } + + /** + * Perform an overlapping copy of the end. This will copy one (potentially + * unaligned) `T` from the end of the source to the end of the destination. + * This may overlap other bits of the copy. + */ + template + SNMALLOC_FAST_PATH_INLINE void + copy_end(void* dst, const void* src, size_t len) + { + copy_one( + pointer_offset(dst, len - Size), pointer_offset(src, len - Size)); + } + + /** + * Predicate indicating whether the source and destination are sufficiently + * aligned to be copied as aligned chunks of `Size` bytes. + */ + template + SNMALLOC_FAST_PATH_INLINE bool is_aligned_memcpy(void* dst, const void* src) + { + return (pointer_align_down(const_cast(src)) == src) && + (pointer_align_down(dst) == dst); + } + + /** + * Copy a small size (`Size` bytes) as a sequence of power-of-two-sized loads + * and stores of decreasing size. `Word` is the largest size to attempt for a + * single copy. + */ + template + SNMALLOC_FAST_PATH_INLINE void small_copy(void* dst, const void* src) + { + static_assert(bits::is_pow2(Word), "Word size must be a power of two!"); + if constexpr (Size != 0) + { + if constexpr (Size >= Word) + { + copy_one(dst, src); + small_copy( + pointer_offset(dst, Word), pointer_offset(src, Word)); + } + else + { + small_copy(dst, src); + } + } + else + { + UNUSED(src); + UNUSED(dst); + } + } + + /** + * Generate small copies for all sizes up to `Size`, using `WordSize` as the + * largest size to copy in a single operation. + */ + template + SNMALLOC_FAST_PATH_INLINE void + small_copies(void* dst, const void* src, size_t len) + { + if (len == Size) + { + small_copy(dst, src); + } + if constexpr (Size > 0) + { + small_copies(dst, src, len); + } + } + + /** + * If the source and destination are the same displacement away from being + * aligned on a `BlockSize` boundary, do a small copy to ensure alignment and + * update `src`, `dst`, and `len` to reflect the remainder that needs + * copying. + * + * Note that this, like memcpy, requires that the source and destination do + * not overlap. It unconditionally copies `BlockSize` bytes, so a subsequent + * copy may not do the right thing. + */ + template + SNMALLOC_FAST_PATH_INLINE void + unaligned_start(void*& dst, const void*& src, size_t& len) + { + constexpr size_t block_mask = BlockSize - 1; + size_t src_addr = static_cast(reinterpret_cast(src)); + size_t dst_addr = static_cast(reinterpret_cast(dst)); + size_t src_offset = src_addr & block_mask; + if ((src_offset > 0) && (src_offset == (dst_addr & block_mask))) + { + size_t disp = BlockSize - src_offset; + small_copies(dst, src, disp); + src = pointer_offset(src, disp); + dst = pointer_offset(dst, disp); + len -= disp; + } + } + + /** + * Default architecture definition. Provides sane defaults. + */ + struct GenericArch + { + /** + * The largest register size that we can use for loads and stores. These + * types are expected to work for overlapping copies: we can always load + * them into a register and store them. Note that this is at the C abstract + * machine level: the compiler may spill temporaries to the stack, just not + * to the source or destination object. + */ + SNMALLOC_UNUSED_FUNCTION + static constexpr size_t LargestRegisterSize = + std::max(sizeof(uint64_t), sizeof(void*)); + + /** + * Hook for architecture-specific optimisations. Does nothing in the + * default case. + */ + static SNMALLOC_FAST_PATH_INLINE bool copy(void*, const void*, size_t) + { + return false; + } + }; + +#if defined(__x86_64__) || defined(_M_X64) + /** + * x86-64 architecture. Prefers SSE registers for small and medium copies + * and uses `rep movsb` for large ones. + */ + struct X86_64Arch + { + /** + * The largest register size that we can use for loads and stores. These + * types are expected to work for overlapping copies: we can always load + * them into a register and store them. Note that this is at the C abstract + * machine level: the compiler may spill temporaries to the stack, just not + * to the source or destination object. + * + * We set this to 16 unconditionally for now because using AVX registers + * imposes stronger alignment requirements that seem to not be a net win. + */ + static constexpr size_t LargestRegisterSize = 16; + + /** + * Platform-specific copy hook. For large copies, use `rep movsb`. + */ + static SNMALLOC_FAST_PATH_INLINE bool + copy(void* dst, const void* src, size_t len) + { + // The Intel optimisation manual recommends doing this for sizes >256 + // bytes on modern systems and for all sizes on very modern systems. + // Testing shows that this is somewhat overly optimistic. + if (SNMALLOC_UNLIKELY(len >= 512)) + { + // Align to cache-line boundaries if possible. + unaligned_start<64, LargestRegisterSize>(dst, src, len); + // Bulk copy. This is aggressively optimised on modern x86 cores. +# ifdef __GNUC__ + asm volatile("rep movsb" + : "+S"(src), "+D"(dst), "+c"(len) + : + : "memory"); +# elif defined(_MSC_VER) + __movsb( + static_cast(dst), + static_cast(src), + len); +# else +# error No inline assembly or rep movsb intrinsic for this compiler. +# endif + return true; + } + return false; + } + }; +#endif + + using DefaultArch = +#ifdef __x86_64__ + X86_64Arch +#else + GenericArch +#endif + ; + + /** + * Snmalloc checked memcpy. The `Arch` parameter must provide: + * + * - A `size_t` value `LargestRegisterSize`, describing the largest size to + * use for single copies. + * - A `copy` function that takes (optionally, references to) the arguments + * of `memcpy` and returns `true` if it performs a copy, `false` + * otherwise. This can be used to special-case some or all sizes for a + * particular architecture. + */ + template< + bool Checked, + bool ReadsChecked = CheckReads, + typename Arch = DefaultArch> + SNMALLOC_FAST_PATH_INLINE void* memcpy(void* dst, const void* src, size_t len) + { + auto orig_dst = dst; + // 0 is a very common size for memcpy and we don't need to do external + // pointer checks if we hit it. It's also the fastest case, to encourage + // the compiler to favour the other cases. + if (SNMALLOC_UNLIKELY(len == 0)) + { + return dst; + } + + if constexpr (Checked) + { + // Check the bounds of the arguments. + check_bounds( + dst, len, "memcpy with destination out of bounds of heap allocation"); + check_bounds( + src, len, "memcpy with source out of bounds of heap allocation"); + } + + // If this is a small size, use a jump table for small sizes. + if (len <= Arch::LargestRegisterSize) + { + small_copies(dst, src, len); + } + // If there's an architecture-specific hook, try using it. Otherwise do a + // simple bulk copy loop. + else if (!Arch::copy(dst, src, len)) + { + block_copy(dst, src, len); + copy_end(dst, src, len); + } + return orig_dst; + } +} // namespace diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 617f7ec..5de742e 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -1,282 +1,7 @@ -#include "../mem/bounds_checks.h" +#include "mem/memcpy.h" + #include "override.h" -using namespace snmalloc; - -namespace -{ - /** - * Copy a single element of a specified size. Uses a compiler builtin that - * expands to a single load and store. - */ - template - SNMALLOC_FAST_PATH_INLINE void copy_one(void* dst, const void* src) - { -#if __has_builtin(__builtin_memcpy_inline) - __builtin_memcpy_inline(dst, src, Size); -#else - // Define a structure of size `Size` that has alignment 1 and a default - // copy-assignment operator. We can then copy the data as this type. The - // compiler knows the exact width and so will generate the correct wide - // instruction for us (clang 10 and gcc 12 both generate movups for the - // 16-byte version of this when targeting SSE. - struct Block - { - char data[Size]; - }; - auto* d = static_cast(dst); - auto* s = static_cast(src); - *d = *s; -#endif - } - - /** - * Copy a block using the specified size. This copies as many complete - * chunks of size `Size` as are possible from `len`. - */ - template - SNMALLOC_FAST_PATH_INLINE void - block_copy(void* dst, const void* src, size_t len) - { - for (size_t i = 0; (i + Size) <= len; i += Size) - { - copy_one(pointer_offset(dst, i), pointer_offset(src, i)); - } - } - - /** - * Perform an overlapping copy of the end. This will copy one (potentially - * unaligned) `T` from the end of the source to the end of the destination. - * This may overlap other bits of the copy. - */ - template - SNMALLOC_FAST_PATH_INLINE void - copy_end(void* dst, const void* src, size_t len) - { - copy_one( - pointer_offset(dst, len - Size), pointer_offset(src, len - Size)); - } - - /** - * Predicate indicating whether the source and destination are sufficiently - * aligned to be copied as aligned chunks of `Size` bytes. - */ - template - SNMALLOC_FAST_PATH_INLINE bool is_aligned_memcpy(void* dst, const void* src) - { - return (pointer_align_down(const_cast(src)) == src) && - (pointer_align_down(dst) == dst); - } - - /** - * Copy a small size (`Size` bytes) as a sequence of power-of-two-sized loads - * and stores of decreasing size. `Word` is the largest size to attempt for a - * single copy. - */ - template - SNMALLOC_FAST_PATH_INLINE void small_copy(void* dst, const void* src) - { - static_assert(bits::is_pow2(Word), "Word size must be a power of two!"); - if constexpr (Size != 0) - { - if constexpr (Size >= Word) - { - copy_one(dst, src); - small_copy( - pointer_offset(dst, Word), pointer_offset(src, Word)); - } - else - { - small_copy(dst, src); - } - } - else - { - UNUSED(src); - UNUSED(dst); - } - } - - /** - * Generate small copies for all sizes up to `Size`, using `WordSize` as the - * largest size to copy in a single operation. - */ - template - SNMALLOC_FAST_PATH_INLINE void - small_copies(void* dst, const void* src, size_t len) - { - if (len == Size) - { - small_copy(dst, src); - } - if constexpr (Size > 0) - { - small_copies(dst, src, len); - } - } - - /** - * If the source and destination are the same displacement away from being - * aligned on a `BlockSize` boundary, do a small copy to ensure alignment and - * update `src`, `dst`, and `len` to reflect the remainder that needs - * copying. - * - * Note that this, like memcpy, requires that the source and destination do - * not overlap. It unconditionally copies `BlockSize` bytes, so a subsequent - * copy may not do the right thing. - */ - template - SNMALLOC_FAST_PATH_INLINE void - unaligned_start(void*& dst, const void*& src, size_t& len) - { - constexpr size_t block_mask = BlockSize - 1; - size_t src_addr = static_cast(reinterpret_cast(src)); - size_t dst_addr = static_cast(reinterpret_cast(dst)); - size_t src_offset = src_addr & block_mask; - if ((src_offset > 0) && (src_offset == (dst_addr & block_mask))) - { - size_t disp = BlockSize - src_offset; - small_copies(dst, src, disp); - src = pointer_offset(src, disp); - dst = pointer_offset(dst, disp); - len -= disp; - } - } - - /** - * Default architecture definition. Provides sane defaults. - */ - struct GenericArch - { - /** - * The largest register size that we can use for loads and stores. These - * types are expected to work for overlapping copies: we can always load - * them into a register and store them. Note that this is at the C abstract - * machine level: the compiler may spill temporaries to the stack, just not - * to the source or destination object. - */ - SNMALLOC_UNUSED_FUNCTION - static constexpr size_t LargestRegisterSize = - std::max(sizeof(uint64_t), sizeof(void*)); - - /** - * Hook for architecture-specific optimisations. Does nothing in the - * default case. - */ - static SNMALLOC_FAST_PATH_INLINE bool copy(void*, const void*, size_t) - { - return false; - } - }; - -#if defined(__x86_64__) || defined(_M_X64) - /** - * x86-64 architecture. Prefers SSE registers for small and medium copies - * and uses `rep movsb` for large ones. - */ - struct X86_64Arch - { - /** - * The largest register size that we can use for loads and stores. These - * types are expected to work for overlapping copies: we can always load - * them into a register and store them. Note that this is at the C abstract - * machine level: the compiler may spill temporaries to the stack, just not - * to the source or destination object. - * - * We set this to 16 unconditionally for now because using AVX registers - * imposes stronger alignment requirements that seem to not be a net win. - */ - static constexpr size_t LargestRegisterSize = 16; - - /** - * Platform-specific copy hook. For large copies, use `rep movsb`. - */ - static SNMALLOC_FAST_PATH_INLINE bool - copy(void* dst, const void* src, size_t len) - { - // The Intel optimisation manual recommends doing this for sizes >256 - // bytes on modern systems and for all sizes on very modern systems. - // Testing shows that this is somewhat overly optimistic. - if (SNMALLOC_UNLIKELY(len >= 512)) - { - // Align to cache-line boundaries if possible. - unaligned_start<64, LargestRegisterSize>(dst, src, len); - // Bulk copy. This is aggressively optimised on modern x86 cores. -# ifdef __GNUC__ - asm volatile("rep movsb" - : "+S"(src), "+D"(dst), "+c"(len) - : - : "memory"); -# elif defined(_MSC_VER) - __movsb( - static_cast(dst), - static_cast(src), - len); -# else -# error No inline assembly or rep movsb intrinsic for this compiler. -# endif - return true; - } - return false; - } - }; -#endif - - using DefaultArch = -#ifdef __x86_64__ - X86_64Arch -#else - GenericArch -#endif - ; - - /** - * Snmalloc checked memcpy. The `Arch` parameter must provide: - * - * - A `size_t` value `LargestRegisterSize`, describing the largest size to - * use for single copies. - * - A `copy` function that takes (optionally, references to) the arguments - * of `memcpy` and returns `true` if it performs a copy, `false` - * otherwise. This can be used to special-case some or all sizes for a - * particular architecture. - */ - template - void* memcpy(void* dst, const void* src, size_t len) - { - auto orig_dst = dst; - // 0 is a very common size for memcpy and we don't need to do external - // pointer checks if we hit it. It's also the fastest case, to encourage - // the compiler to favour the other cases. - if (SNMALLOC_UNLIKELY(len == 0)) - { - return dst; - } - - if constexpr (checked) - { - // Check the bounds of the arguments. - check_bounds( - dst, len, "memcpy with destination out of bounds of heap allocation"); - check_bounds( - src, len, "memcpy with source out of bounds of heap allocation"); - } - - // If this is a small size, use a jump table for small sizes. - if (len <= Arch::LargestRegisterSize) - { - small_copies(dst, src, len); - } - // If there's an architecture-specific hook, try using it. Otherwise do a - // simple bulk copy loop. - else if (!Arch::copy(dst, src, len)) - { - block_copy(dst, src, len); - copy_end(dst, src, len); - } - return orig_dst; - } -} // namespace - extern "C" { /** @@ -285,6 +10,6 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(memcpy)(void* dst, const void* src, size_t len) { - return memcpy(dst, src, len); + return snmalloc::memcpy(dst, src, len); } } diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc index 03ce761..e99569e 100644 --- a/src/test/func/memcpy/func-memcpy.cc +++ b/src/test/func/memcpy/func-memcpy.cc @@ -35,6 +35,8 @@ int main() # include # include +using namespace snmalloc; + /** * Jump buffer used to jump out of `abort()` for recoverable errors. */ @@ -84,7 +86,8 @@ void check_size(size_t size) { dst[i] = 0; } - my_memcpy(dst, src, sz); + void* ret = my_memcpy(dst, src, sz); + EXPECT(ret == dst, "Return value should be {}, was {}", dst, ret); for (size_t i = 0; i < sz; ++i) { if (dst[i] != static_cast(i)) diff --git a/src/test/perf/memcpy/memcpy.cc b/src/test/perf/memcpy/memcpy.cc index bfa6cb4..b42c474 100644 --- a/src/test/perf/memcpy/memcpy.cc +++ b/src/test/perf/memcpy/memcpy.cc @@ -1,11 +1,11 @@ +#include "mem/memcpy.h" + #include #include - -#define SNMALLOC_NAME_MANGLE(a) our_##a -#include "override/memcpy.cc" - #include +using namespace snmalloc; + struct Shape { void* object;