From 631ff581afb36d132c9157dc176d5dd94eff9e81 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 21 Aug 2024 10:47:38 -0700 Subject: [PATCH 1/4] set compile as C++ in VS IDE --- ide/vs2022/mimalloc-override.vcxproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ide/vs2022/mimalloc-override.vcxproj b/ide/vs2022/mimalloc-override.vcxproj index e895fa3c..4383d886 100644 --- a/ide/vs2022/mimalloc-override.vcxproj +++ b/ide/vs2022/mimalloc-override.vcxproj @@ -98,7 +98,7 @@ MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false - Default + CompileAsCpp $(ProjectDir)\..\..\bin\mimalloc-redirect32.lib;%(AdditionalDependencies) @@ -126,7 +126,7 @@ MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false - Default + CompileAsCpp $(ProjectDir)\..\..\bin\mimalloc-redirect.lib;%(AdditionalDependencies) @@ -157,7 +157,7 @@ $(IntDir) false MultiThreadedDLL - Default + CompileAsCpp false @@ -189,7 +189,7 @@ $(IntDir) false MultiThreadedDLL - Default + CompileAsCpp false From 96f7cc4ef2c1f03a1775c9a3fa3a08a1e572b7b2 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 21 Aug 2024 11:21:25 -0700 Subject: [PATCH 2/4] clean up guarded allocation --- src/alloc.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 7c1284fb..34adc861 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -118,8 +118,9 @@ extern void* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t siz } #if MI_DEBUG_GUARDED -// forward declaration -static mi_decl_restrict void* mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept; +static mi_decl_restrict void* mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; +static inline bool mi_heap_malloc_use_guarded(size_t size, bool huge_alignment); +static inline bool mi_heap_malloc_small_use_guarded(size_t size); #endif static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { @@ -133,9 +134,7 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, if (size == 0) { size = sizeof(void*); } #endif #if MI_DEBUG_GUARDED - if (size <= (size_t)_mi_option_get_fast(mi_option_debug_guarded_max) && size >= (size_t)_mi_option_get_fast(mi_option_debug_guarded_min)) { - return mi_heap_malloc_guarded(heap, size, zero, 0); - } + if (mi_heap_malloc_small_use_guarded(size)) { return mi_heap_malloc_guarded(heap, size, zero); } #endif mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE); @@ -172,13 +171,7 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z return mi_heap_malloc_small_zero(heap, size, zero); } #if MI_DEBUG_GUARDED - else if ( huge_alignment == 0 && // guarded pages do not work with huge aligments at the moment - _mi_option_get_fast(mi_option_debug_guarded_max) > 0 && // guarded must be enabled - ((size >= (size_t)_mi_option_get_fast(mi_option_debug_guarded_min) && size <= (size_t)_mi_option_get_fast(mi_option_debug_guarded_max)) - || ((mi_good_size(size) & (_mi_os_page_size()-1)) == 0)) ) // page-size multiple are always guarded so we can have a correct `mi_usable_size`. - { - return mi_heap_malloc_guarded(heap, size, zero, 0); - } + else if (mi_use_guarded(size,huge_alignment)) { return mi_heap_malloc_guarded(heap, size, zero); } #endif else { mi_assert(heap!=NULL); @@ -601,7 +594,20 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { } #if MI_DEBUG_GUARDED -static mi_decl_restrict void* mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept +static inline bool mi_heap_malloc_small_use_guarded(size_t size) { + return (size <= (size_t)_mi_option_get_fast(mi_option_debug_guarded_max) + && size >= (size_t)_mi_option_get_fast(mi_option_debug_guarded_min)); +} + +static inline bool mi_heap_malloc_use_guarded(size_t size, bool huge_alignment) { + return (huge_alignment == 0 // guarded pages do not work with huge aligments at the moment + && _mi_option_get_fast(mi_option_debug_guarded_max) > 0 // guarded must be enabled + && (mi_heap_malloc_small_use_guarded(size) + || ((mi_good_size(size) & (_mi_os_page_size() - 1)) == 0)) // page-size multiple are always guarded so we can have a correct `mi_usable_size`. + ); +} + +static mi_decl_restrict void* mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { #if defined(MI_PADDING_SIZE) mi_assert(MI_PADDING_SIZE==0); @@ -610,7 +616,7 @@ static mi_decl_restrict void* mi_heap_malloc_guarded(mi_heap_t* heap, size_t siz const size_t obj_size = _mi_align_up(size, MI_MAX_ALIGN_SIZE); // ensure minimal alignment requirement const size_t os_page_size = _mi_os_page_size(); const size_t req_size = _mi_align_up(obj_size + os_page_size, os_page_size); - void* const block = _mi_malloc_generic(heap, req_size, zero, huge_alignment); + void* const block = _mi_malloc_generic(heap, req_size, zero, 0 /* huge_alignment */); if (block==NULL) return NULL; mi_page_t* page = _mi_ptr_page(block); mi_segment_t* segment = _mi_page_segment(page); From cc4dc1bb53017da9862e1b4d3e74853fc9b6ec27 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 21 Aug 2024 11:26:12 -0700 Subject: [PATCH 3/4] add comments --- src/alloc.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 34adc861..c91de62d 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -31,18 +31,22 @@ terms of the MIT license. A copy of the license can be found in the file extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { mi_assert_internal(page->block_size == 0 /* empty heap */ || mi_page_block_size(page) >= size); + + // check the free list mi_block_t* const block = page->free; if mi_unlikely(block == NULL) { return _mi_malloc_generic(heap, size, zero, 0); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); + // pop from the free list page->free = mi_block_next(page, block); page->used++; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); + #if MI_DEBUG>3 - if (page->free_is_zero && size > sizeof(*block)) { + if (page->free_is_zero && size > sizeof(*block)) { mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block))); } #endif @@ -125,11 +129,11 @@ static inline bool mi_heap_malloc_small_use_guarded(size_t size); static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { mi_assert(heap != NULL); + mi_assert(size <= MI_SMALL_SIZE_MAX); #if MI_DEBUG const uintptr_t tid = _mi_thread_id(); mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local #endif - mi_assert(size <= MI_SMALL_SIZE_MAX); #if (MI_PADDING || MI_DEBUG_GUARDED) if (size == 0) { size = sizeof(void*); } #endif @@ -137,6 +141,7 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, if (mi_heap_malloc_small_use_guarded(size)) { return mi_heap_malloc_guarded(heap, size, zero); } #endif + // get page in constant time, and allocate from it mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE); void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero); mi_track_malloc(p,size,zero); @@ -166,6 +171,7 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t si // The main allocation function extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept { + // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); return mi_heap_malloc_small_zero(heap, size, zero); @@ -174,10 +180,12 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z else if (mi_use_guarded(size,huge_alignment)) { return mi_heap_malloc_guarded(heap, size, zero); } #endif else { + // regular allocation mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic mi_track_malloc(p,size,zero); + #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } From 58e743b83f0949ecfa4732eaec0288672c55123f Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 21 Aug 2024 11:30:00 -0700 Subject: [PATCH 4/4] fix use_guarded signature --- src/alloc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index c91de62d..ca60c11a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -123,7 +123,7 @@ extern void* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t siz #if MI_DEBUG_GUARDED static mi_decl_restrict void* mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; -static inline bool mi_heap_malloc_use_guarded(size_t size, bool huge_alignment); +static inline bool mi_heap_malloc_use_guarded(size_t size, bool has_huge_alignment); static inline bool mi_heap_malloc_small_use_guarded(size_t size); #endif @@ -177,7 +177,7 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z return mi_heap_malloc_small_zero(heap, size, zero); } #if MI_DEBUG_GUARDED - else if (mi_use_guarded(size,huge_alignment)) { return mi_heap_malloc_guarded(heap, size, zero); } + else if (mi_heap_malloc_use_guarded(size,huge_alignment>0)) { return mi_heap_malloc_guarded(heap, size, zero); } #endif else { // regular allocation @@ -607,8 +607,8 @@ static inline bool mi_heap_malloc_small_use_guarded(size_t size) { && size >= (size_t)_mi_option_get_fast(mi_option_debug_guarded_min)); } -static inline bool mi_heap_malloc_use_guarded(size_t size, bool huge_alignment) { - return (huge_alignment == 0 // guarded pages do not work with huge aligments at the moment +static inline bool mi_heap_malloc_use_guarded(size_t size, bool has_huge_alignment) { + return (!has_huge_alignment // guarded pages do not work with huge aligments at the moment && _mi_option_get_fast(mi_option_debug_guarded_max) > 0 // guarded must be enabled && (mi_heap_malloc_small_use_guarded(size) || ((mi_good_size(size) & (_mi_os_page_size() - 1)) == 0)) // page-size multiple are always guarded so we can have a correct `mi_usable_size`.