diff --git a/src/ds/defines.h b/src/ds/defines.h index b0aab14..3019493 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -1,12 +1,20 @@ #pragma once #if defined(_MSC_VER) && !defined(__clang__) +// 28 is FAST_FAIL_INVALID_BUFFER_ACCESS. Not using the symbolic constant to +// avoid depending on winnt.h +# define SNMALLOC_FAST_FAIL() __fastfail(28) # define ALWAYSINLINE __forceinline # define NOINLINE __declspec(noinline) # define likely(x) !!(x) # define unlikely(x) !!(x) # define SNMALLOC_SLOW_PATH NOINLINE # define SNMALLOC_FAST_PATH ALWAYSINLINE +/** + * Fast path with inline linkage. MSVC assumes that `__forceinline` implies + * `inline` and complains if you specify `SNMALLOC_FAST_PATH` and `inline`. + */ +# define SNMALLOC_FAST_PATH_INLINE ALWAYSINLINE # if _MSC_VER >= 1927 && !defined(SNMALLOC_USE_CXX17) # define SNMALLOC_FAST_PATH_LAMBDA [[msvc::forceinline]] # else @@ -17,12 +25,23 @@ # define SNMALLOC_REQUIRE_CONSTINIT # define SNMALLOC_UNUSED_FUNCTION #else +# define SNMALLOC_FAST_FAIL() __builtin_trap() # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) # define SNMALLOC_SLOW_PATH NOINLINE # define SNMALLOC_FAST_PATH ALWAYSINLINE +/** + * Fast path with inline linkage. GCC assumes that + * `__attribute__((always_inline))` is orthogonal to `inline` and complains if + * you specify `SNMALLOC_FAST_PATH` and don't specify `inline` in places where + * `inline` would be required for the one definition rule. The error message + * in this case is confusing: always-inline function may not be inlined. If + * you see this error message when using `SNMALLOC_FAST_PATH` then switch to + * `SNMALLOC_FAST_PATH_INLINE`. + */ +# define SNMALLOC_FAST_PATH_INLINE ALWAYSINLINE inline # define SNMALLOC_FAST_PATH_LAMBDA SNMALLOC_FAST_PATH # define SNMALLOC_PURE __attribute__((const)) # define SNMALLOC_COLD __attribute__((cold)) diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 4d41bf3..82bbf9a 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -16,7 +16,7 @@ using namespace snmalloc; // Windows has it with an underscore prefix #elif defined(_MSC_VER) # define snprintf_l(buf, size, loc, msg, ...) \ - _snprintf_l(buf, size, msg, loc, __VA_ARGS__) + _snprintf_s_l(buf, size, _TRUNCATE, msg, loc, __VA_ARGS__) #endif namespace @@ -79,7 +79,7 @@ namespace * expands to a single load and store. */ template - SNMALLOC_FAST_PATH inline void copy_one(void* dst, const void* src) + SNMALLOC_FAST_PATH_INLINE void copy_one(void* dst, const void* src) { #if __has_builtin(__builtin_memcpy_inline) __builtin_memcpy_inline(dst, src, Size); @@ -128,7 +128,7 @@ namespace * function is a no-op when `CheckReads` is false. */ template - SNMALLOC_FAST_PATH inline void + SNMALLOC_FAST_PATH_INLINE void check_bounds(const void* ptr, size_t len, const char* msg = "") { if constexpr (!IsRead || CheckReads) @@ -144,7 +144,7 @@ namespace UNUSED(ptr); UNUSED(len); UNUSED(msg); - __builtin_trap(); + SNMALLOC_FAST_FAIL(); } else { @@ -165,7 +165,7 @@ namespace * chunks of size `Size` as are possible from `len`. */ template - SNMALLOC_FAST_PATH inline void + SNMALLOC_FAST_PATH_INLINE void block_copy(void* dst, const void* src, size_t len) { for (size_t i = 0; (i + Size) <= len; i += Size) @@ -180,7 +180,7 @@ namespace * This may overlap other bits of the copy. */ template - SNMALLOC_FAST_PATH inline void + SNMALLOC_FAST_PATH_INLINE void copy_end(void* dst, const void* src, size_t len) { copy_one(