diff --git a/src/snmalloc/aal/address.h b/src/snmalloc/aal/address.h index d10474c..6c1a760 100644 --- a/src/snmalloc/aal/address.h +++ b/src/snmalloc/aal/address.h @@ -14,7 +14,8 @@ namespace snmalloc /** * Perform arithmetic on a uintptr_t. */ - inline uintptr_t pointer_offset(uintptr_t base, size_t diff) + SNMALLOC_FAST_PATH_INLINE uintptr_t + pointer_offset(uintptr_t base, size_t diff) { return base + diff; } @@ -23,7 +24,7 @@ namespace snmalloc * Perform pointer arithmetic and return the adjusted pointer. */ template - inline U* pointer_offset(T* base, size_t diff) + SNMALLOC_FAST_PATH_INLINE U* pointer_offset(T* base, size_t diff) { SNMALLOC_ASSERT(base != nullptr); /* Avoid UB */ return unsafe_from_uintptr( @@ -31,7 +32,7 @@ namespace snmalloc } template - inline CapPtr + SNMALLOC_FAST_PATH_INLINE CapPtr pointer_offset(CapPtr base, size_t diff) { return CapPtr::unsafe_from( @@ -42,14 +43,14 @@ namespace snmalloc * Perform pointer arithmetic and return the adjusted pointer. */ template - inline U* pointer_offset_signed(T* base, ptrdiff_t diff) + SNMALLOC_FAST_PATH_INLINE U* pointer_offset_signed(T* base, ptrdiff_t diff) { SNMALLOC_ASSERT(base != nullptr); /* Avoid UB */ return reinterpret_cast(reinterpret_cast(base) + diff); } template - inline CapPtr + SNMALLOC_FAST_PATH_INLINE CapPtr pointer_offset_signed(CapPtr base, ptrdiff_t diff) { return CapPtr::unsafe_from( @@ -60,7 +61,7 @@ namespace snmalloc * Cast from a pointer type to an address. */ template - inline SNMALLOC_FAST_PATH address_t address_cast(T* ptr) + SNMALLOC_FAST_PATH_INLINE address_t address_cast(T* ptr) { return reinterpret_cast(ptr); } @@ -73,12 +74,12 @@ namespace snmalloc * capptr_bound. */ template - inline SNMALLOC_FAST_PATH address_t address_cast(CapPtr a) + SNMALLOC_FAST_PATH_INLINE address_t address_cast(CapPtr a) { return address_cast(a.unsafe_ptr()); } - inline SNMALLOC_FAST_PATH address_t address_cast(uintptr_t a) + SNMALLOC_FAST_PATH_INLINE address_t address_cast(uintptr_t a) { return static_cast(a); } @@ -88,7 +89,7 @@ namespace snmalloc * two. */ template - static inline bool is_aligned_block(address_t p, size_t size) + SNMALLOC_FAST_PATH_INLINE bool is_aligned_block(address_t p, size_t size) { static_assert(bits::is_pow2(alignment)); @@ -96,7 +97,7 @@ namespace snmalloc } template - static inline bool is_aligned_block(void* p, size_t size) + SNMALLOC_FAST_PATH_INLINE bool is_aligned_block(void* p, size_t size) { return is_aligned_block(address_cast(p), size); } @@ -106,7 +107,7 @@ namespace snmalloc * a power of two. */ template - inline uintptr_t pointer_align_down(uintptr_t p) + SNMALLOC_FAST_PATH_INLINE uintptr_t pointer_align_down(uintptr_t p) { static_assert(alignment > 0); static_assert(bits::is_pow2(alignment)); @@ -127,7 +128,7 @@ namespace snmalloc * power of two. */ template - inline T* pointer_align_down(void* p) + SNMALLOC_FAST_PATH_INLINE T* pointer_align_down(void* p) { return unsafe_from_uintptr( pointer_align_down(unsafe_to_uintptr(p))); @@ -137,14 +138,15 @@ namespace snmalloc size_t alignment, typename T, SNMALLOC_CONCEPT(capptr::IsBound) bounds> - inline CapPtr pointer_align_down(CapPtr p) + SNMALLOC_FAST_PATH_INLINE CapPtr + pointer_align_down(CapPtr p) { return CapPtr::unsafe_from( pointer_align_down(p.unsafe_ptr())); } template - inline address_t address_align_down(address_t p) + SNMALLOC_FAST_PATH_INLINE address_t address_align_down(address_t p) { return bits::align_down(p, alignment); } @@ -154,7 +156,7 @@ namespace snmalloc * power of two. */ template - inline T* pointer_align_up(void* p) + SNMALLOC_FAST_PATH_INLINE T* pointer_align_up(void* p) { static_assert(alignment > 0); static_assert(bits::is_pow2(alignment)); @@ -175,14 +177,15 @@ namespace snmalloc size_t alignment, typename T = void, SNMALLOC_CONCEPT(capptr::IsBound) bounds> - inline CapPtr pointer_align_up(CapPtr p) + SNMALLOC_FAST_PATH_INLINE CapPtr + pointer_align_up(CapPtr p) { return CapPtr::unsafe_from( pointer_align_up(p.unsafe_ptr())); } template - inline address_t address_align_up(address_t p) + SNMALLOC_FAST_PATH_INLINE address_t address_align_up(address_t p) { return bits::align_up(p, alignment); } @@ -192,7 +195,7 @@ namespace snmalloc * a power of two. */ template - inline T* pointer_align_down(void* p, size_t alignment) + SNMALLOC_FAST_PATH_INLINE T* pointer_align_down(void* p, size_t alignment) { SNMALLOC_ASSERT(alignment > 0); SNMALLOC_ASSERT(bits::is_pow2(alignment)); @@ -205,7 +208,7 @@ namespace snmalloc } template - inline CapPtr + SNMALLOC_FAST_PATH_INLINE CapPtr pointer_align_down(CapPtr p, size_t alignment) { return CapPtr::unsafe_from( @@ -217,7 +220,7 @@ namespace snmalloc * be a power of two. */ template - inline T* pointer_align_up(void* p, size_t alignment) + SNMALLOC_FAST_PATH_INLINE T* pointer_align_up(void* p, size_t alignment) { SNMALLOC_ASSERT(alignment > 0); SNMALLOC_ASSERT(bits::is_pow2(alignment)); @@ -230,7 +233,7 @@ namespace snmalloc } template - inline CapPtr + SNMALLOC_FAST_PATH_INLINE CapPtr pointer_align_up(CapPtr p, size_t alignment) { return CapPtr::unsafe_from( @@ -242,7 +245,8 @@ namespace snmalloc * expected to point to the base of some (sub)allocation into which cursor * points; would-be negative answers trip an assertion in debug builds. */ - inline size_t pointer_diff(const void* base, const void* cursor) + SNMALLOC_FAST_PATH_INLINE size_t + pointer_diff(const void* base, const void* cursor) { SNMALLOC_ASSERT(cursor >= base); return static_cast( @@ -254,7 +258,8 @@ namespace snmalloc typename U = void, SNMALLOC_CONCEPT(capptr::IsBound) Tbounds, SNMALLOC_CONCEPT(capptr::IsBound) Ubounds> - inline size_t pointer_diff(CapPtr base, CapPtr cursor) + SNMALLOC_FAST_PATH_INLINE size_t + pointer_diff(CapPtr base, CapPtr cursor) { return pointer_diff(base.unsafe_ptr(), cursor.unsafe_ptr()); } @@ -263,7 +268,8 @@ namespace snmalloc * Compute the difference in pointers in units of char. This can be used * across allocations. */ - inline ptrdiff_t pointer_diff_signed(void* base, void* cursor) + SNMALLOC_FAST_PATH_INLINE ptrdiff_t + pointer_diff_signed(void* base, void* cursor) { return static_cast( static_cast(cursor) - static_cast(base)); @@ -274,7 +280,7 @@ namespace snmalloc typename U = void, SNMALLOC_CONCEPT(capptr::IsBound) Tbounds, SNMALLOC_CONCEPT(capptr::IsBound) Ubounds> - inline ptrdiff_t + SNMALLOC_FAST_PATH_INLINE ptrdiff_t pointer_diff_signed(CapPtr base, CapPtr cursor) { return pointer_diff_signed(base.unsafe_ptr(), cursor.unsafe_ptr()); @@ -285,7 +291,7 @@ namespace snmalloc * putative alignment. */ template - inline size_t address_misalignment(address_t a) + SNMALLOC_FAST_PATH_INLINE size_t address_misalignment(address_t a) { return static_cast(a - pointer_align_down(a)); } diff --git a/src/snmalloc/backend_helpers/range_helpers.h b/src/snmalloc/backend_helpers/range_helpers.h index 9d97f0e..076b9fd 100644 --- a/src/snmalloc/backend_helpers/range_helpers.h +++ b/src/snmalloc/backend_helpers/range_helpers.h @@ -16,7 +16,7 @@ namespace snmalloc // Find the minimum set of maximally aligned blocks in this range. // Each block's alignment and size are equal. - while (length >= sizeof(void*)) + while (length >= bits::one_at_bit(MIN_BITS)) { size_t base_align_bits = bits::ctz(address_cast(base)); size_t length_align_bits = (bits::BITS - 1) - bits::clz(length); diff --git a/src/snmalloc/mem/freelist.h b/src/snmalloc/mem/freelist.h index de0134a..49348d1 100644 --- a/src/snmalloc/mem/freelist.h +++ b/src/snmalloc/mem/freelist.h @@ -187,6 +187,7 @@ namespace snmalloc signed_prev(address_cast(this), address_cast(n_tame), key)); } } + Aal::prefetch(&(n_tame->next_object)); return n_tame; }