diff --git a/src/pal/pal.h b/src/pal/pal.h index 10f9cd8..e5343bd 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -117,13 +117,18 @@ namespace snmalloc "The smallest architectural page size must divide OS_PAGE_SIZE"); // Some system headers (e.g. Linux' sys/user.h, FreeBSD's machine/param.h) - // define `PAGE_SIZE` as a macro. We don't use `PAGE_SIZE` as our variable - // name, to avoid conflicts, but if we do see a macro definition then check - // that our value matches the platform's expected value. + // define `PAGE_SIZE` as a macro, while others (e.g. macOS 11's + // mach/machine/vm_param.h) define `PAGE_SIZE` as an extern. We don't use + // `PAGE_SIZE` as our variable name, to avoid conflicts, but if we do see a + // macro definition evaluates to a constant then check that our value matches + // the platform's expected value. #ifdef PAGE_SIZE static_assert( - PAGE_SIZE == OS_PAGE_SIZE, +# if __has_builtin(__builtin_constant_p) + !__builtin_constant_p(PAGE_SIZE) || (PAGE_SIZE == OS_PAGE_SIZE), +# else + true, +# endif "Page size from system header does not match snmalloc config page size."); #endif - } // namespace snmalloc diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index ad5b473..685ae93 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -1,92 +1,286 @@ #pragma once #ifdef __APPLE__ + # include "pal_bsd.h" +# include # include +# include +# include # include -# include +# include +# include +# include +# include +# include namespace snmalloc { /** * PAL implementation for Apple systems (macOS, iOS, watchOS, tvOS...). */ - template + template class PALApple : public PALBSD> { public: /** * The features exported by this PAL. + */ + static constexpr uint64_t pal_features = + AlignedAllocation | LazyCommit | Entropy; + + /* + * `page_size` * - * Currently, these are identical to the generic BSD PAL. This field is - * declared explicitly to remind anyone who modifies this class that they - * should add any required features. - */ - static constexpr uint64_t pal_features = PALBSD::pal_features; - - /** - * Anonymous page tag ID. + * On 64-bit ARM platforms, the page size (for user-space) is 16KiB. + * Otherwise (e.g. x86_64) it is 4KiB. + * + * macOS on Apple Silicon ARM does support 4KiB pages, but they are reserved + * for "exotic" processes (i.e. Rosetta 2) and kernel-space. Using 4KiB + * pages from user-space in "native" (non-translated) processes is incorrect + * and will cause bugs. + * + * However, Apple's 64-bit embedded ARM-based platforms (phones, pads, tvs) + * do not support 4KiB pages. * - * Darwin platform allows to gives an ID to anonymous pages via - * the VM_MAKE_TAG's macro, from 240 up to 255 are guaranteed - * to be free of usage, however eventually a lower could be taken - * (e.g. LLVM sanitizers has 99) so we can monitor their states - * via vmmap for instance. This value is provided to `mmap` as the file - * descriptor for the mapping. */ - static constexpr int anonymous_memory_fd = VM_MAKE_TAG(PALAnonID); + static constexpr size_t page_size = Aal::aal_name == ARM ? 0x4000 : 0x1000; + + static constexpr size_t minimum_alloc_size = page_size; + + /* + * Memory Tag + * + * A memory tag is an 8-bit value that denotes auxillary "type information" + * of a vm region. This tag can be used for marking memory for profiling and + * debugging, or instructing the kernel to perform tag-specific behavior. + * (E.g. VM_MEMORY_MALLOC{_*} is reused by default, unless it is no longer + * in its "original state". See `vm_map_entry_is_reusable` in + * `osfmk/vm/vm_map.c` for more details of this behavior.) + * + * Memory tags are encoded using `VM_MAKE_TAG(tag_value)`, and can be passed + * to the kernel by either `mmap` or `mach_vm_map`: + * 1. `fd` argument of `mmap`. + * 2. `flags` argument of `mach_vm_map`. + * + * There are currently 4 categories of memory tags: + * + * 1. Reserved: [0, 39]. Typically used for Apple libraries and services. + * Use may trigger undocumented kernel-based behavior. + * + * 2. Defined "placeholders": [39, 98]. Typically used for Apple libraries + * and services. + * + * 3. Undefined "placeholders": [99, 239]. Unallocated by Apple. Typically + * used for libraries. (E.g. LLVM sanitizers use 99.) + * + * 4. Application specific: [240, 255] + * + * See for more details about memory tags and their + * uses. + * + * In the future, we may switch our default memory tag from "category 4" to + * "category 3", thereby affording us a "well-known" memory tag that can be + * easily identified in tools such as vmmap(1) or Instruments. + * + */ + + // Encoded memory tag passed to `mmap`. + static constexpr int anonymous_memory_fd = + int(VM_MAKE_TAG(uint32_t(PALAnonID))); + + // Encoded memory tag passed to `mach_vm_map`. + static constexpr int default_mach_vm_map_flags = + int(VM_MAKE_TAG(uint32_t(PALAnonID))); /** - * Note: The root's implementation works fine on Intel - * however mprotect/PROT_NONE fails on ARM - * especially since the 11.2 release (seems known issue - * spotted in various projects; might be a temporary fix). - */ - template - static void zero(void* p, size_t size) noexcept - { - if constexpr (Aal::aal_name != ARM) - PALBSD::zero(p, size); - else - ::bzero(p, size); - } - -# if defined(PLATFORM_IS_ARM) - /** - * Overriding here to mark the page as reusable - * rolling it as much as necessary. - * As above, the x86 h/w worked alright without this change - * however now large allocations work better and more reliably - * with on ARM, not to mention better RSS number accuracy - * for tools based on task_info API. + * Notify platform that we will not be using these pages. + * + * We deviate from `PALBSD::notify_not_using` b/c `MADV_FREE` does not + * behave as expected on Apple platforms. The pages are never marked as + * "reusable" by the kernel and this can be observed through profiling. E.g. + * at least ~75% to ~90% less dirty memory is used by `func-malloc-16` + * (observed on x86_64). + * + * Apple's own malloc implementation as well as many ports for Apple + * Operating Systems use MADV_FREE_REUS{E, ABLE} instead of MADV_FREE. See: + * https://opensource.apple.com/source/libmalloc/libmalloc-53.1.1/src/magazine_malloc.c.auto.html + * https://bugs.chromium.org/p/chromium/issues/detail?id=713892 + * */ static void notify_not_using(void* p, size_t size) noexcept { - SNMALLOC_ASSERT(is_aligned_block(p, size)); -# ifdef USE_POSIX_COMMIT_CHECKS + SNMALLOC_ASSERT(is_aligned_block(p, size)); + +# ifdef USE_POSIX_COMMIT_CHECKS memset(p, 0x5a, size); -# endif +# endif + + // `MADV_FREE_REUSABLE` can only be applied to writable pages, + // otherwise it's an error. + // + // `mach_vm_behavior_set` is observably slower in benchmarks. + // + // macOS 11 Big Sur may behave in an undocumented manner. while (madvise(p, size, MADV_FREE_REUSABLE) == -1 && errno == EAGAIN) ; + +# ifdef USE_POSIX_COMMIT_CHECKS + // This must occur after `MADV_FREE_REUSABLE`. + // + // `mach_vm_protect` is observably slower in benchmarks. + mprotect(p, size, PROT_NONE); +# endif } /** - * same remark as above but we need to mark the page as REUSE - * first + * Notify platform that we will be using these pages. + * + * We deviate from `PALPOSIX::notify_using` for three reasons: + * 1. `MADV_FREE_REUSABLE` must be paired with `MADV_FREE_REUSE`. + * 2. `MADV_FREE_REUSE` must only be applied to writable pages, otherwise it + * is an error. + * 3. `PALPOSIX::notify_using` will apply mprotect(PROT_READ|PROT_WRITE) to + * the pages, and then call `PALPOSIX::zero` (overwrite pages with + * mmap, and if mmap fails call bzero on the pages). This is very wasteful; + * if mmap succeeds we do not need to change the permissions of the pages + * since it is done during mmap. Instead `PALApple::notify_using` will try + * to overwrite with mmap, and if mmap fails, apply + * mprotect(PROT_READ|PROT_WRITE), madvise(MADV_REUSE), and finally call + * `bzero` to clear the pages. + * + * Currently, `PALPOSIX::zero` will call `bzero` on the pages that `mmap` + * failed to overwrite. In the future, we should duplicate the behavior of + * `PALWindows` and abort the process if a `mmap` call fails. But for now we + * are going to be consistent with the behavior of the other POSIX PAL + * implementations. + * */ template static void notify_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT( - is_aligned_block(p, size) || (zero_mem == NoZero)); + is_aligned_block(p, size) || (zero_mem == NoZero)); + + if constexpr (zero_mem == YesZero) + { + void* r = mmap( + p, + size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, + anonymous_memory_fd, + 0); + + if (likely(r != MAP_FAILED)) + { + return; + } + } + +# ifdef USE_POSIX_COMMIT_CHECKS + // Mark pages as writable for `madvise` below. + // + // `mach_vm_protect` is observably slower in benchmarks. + mprotect(p, size, PROT_READ | PROT_WRITE); +# endif + + // `MADV_FREE_REUSE` can only be applied to writable pages, + // otherwise it's an error. + // + // `mach_vm_behavior_set` is observably slower in benchmarks. + // + // macOS 11 Big Sur may behave in an undocumented manner. while (madvise(p, size, MADV_FREE_REUSE) == -1 && errno == EAGAIN) ; if constexpr (zero_mem == YesZero) - zero(p, size); + { + ::bzero(p, size); + } } + + // Apple's `mmap` doesn't support user-specified alignment and only + // guarantees mappings are aligned to the system page size, so we use + // `mach_vm_map` instead. + template + static void* reserve_aligned(size_t size) noexcept + { + SNMALLOC_ASSERT(bits::is_pow2(size)); + SNMALLOC_ASSERT(size >= minimum_alloc_size); + + // mask has least-significant bits set + mach_vm_offset_t mask = size - 1; + + int flags = VM_FLAGS_ANYWHERE | default_mach_vm_map_flags; + + // must be initialized to 0 or addr is interepreted as a lower-bound. + mach_vm_address_t addr = 0; + +# ifdef USE_POSIX_COMMIT_CHECKS + vm_prot_t prot = committed ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_NONE; +# else + vm_prot_t prot = VM_PROT_READ | VM_PROT_WRITE; # endif + + kern_return_t kr = mach_vm_map( + mach_task_self(), + &addr, + size, + mask, + flags, + MEMORY_OBJECT_NULL, + 0, + TRUE, + prot, + VM_PROT_READ | VM_PROT_WRITE, + VM_INHERIT_COPY); + + if (unlikely(kr != KERN_SUCCESS)) + { + error("Failed to allocate memory\n"); + } + + return reinterpret_cast(addr); + } + + /** + * Source of Entropy + * + * Apple platforms have a working `getentropy(2)` implementation. + * However, it is not allowed on the App Store and Apple actively + * discourages its use. The substitutes `arc4random_buf(3)`, + * `CCRandomGenerateBytes`, and `SecRandomCopyBytes` are recommended + * instead. + * + * `CCRandomGenerateBytes` was selected because: + * + * 1. The implementation of `arc4random_buf(3)` differs from its + * documentation. It is documented to never fail, yet its' + * implementation can fail silently: it calls the function + * `ccrng_generate`, but ignores the error case. + * `CCRandomGenerateBytes` is built on the same function, but can return an + * error code in case of failure. See: + * https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html + * https://opensource.apple.com/source/CommonCrypto/CommonCrypto-60061/include/CommonRandom.h.auto.html + * + * 2. `SecRandomCopyBytes` introduces a dependency on `Security.framework`. + * `CCRandomGenerateBytes` introduces no new dependencies. + * + */ + static uint64_t get_entropy64() + { + uint64_t result; + if ( + CCRandomGenerateBytes( + reinterpret_cast(&result), sizeof(result)) != kCCSuccess) + { + error("Failed to get system randomness"); + } + + return result; + } }; } // namespace snmalloc #endif diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 90ad321..84823fe 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -65,7 +65,7 @@ namespace snmalloc /** * Default Tag ID for the Apple class */ - static const int PALAnonDefaultID = 241; + static const uint8_t PALAnonDefaultID = 241; /** * This struct is used to represent callbacks for notification from the