From d48b988c405601b5594be7db20177e9d7e223130 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 16 Apr 2023 11:21:45 -0700 Subject: [PATCH 1/2] clean up os api --- include/mimalloc/internal.h | 6 +++--- include/mimalloc/prim.h | 4 ++-- include/mimalloc/types.h | 5 ++++- src/arena.c | 22 ++++++++++------------ src/os.c | 26 +++++++++++--------------- src/segment.c | 4 ++-- 6 files changed, 32 insertions(+), 35 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index e346f120..fa932cda 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -103,9 +103,9 @@ bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats); bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats); -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, bool* is_zero, mi_stats_t* stats); -void* _mi_os_alloc_aligned_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool* large, bool* is_zero, mi_stats_t* tld_stats); -void _mi_os_free_aligned(void* p, size_t size, size_t alignment, size_t align_offset, bool was_committed, mi_stats_t* tld_stats); +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* stats); +void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* tld_stats); +void _mi_os_free_aligned_at_offset(void* p, size_t size, size_t alignment, size_t align_offset, bool was_committed, mi_stats_t* tld_stats); void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); bool _mi_os_use_large_page(size_t size, size_t alignment); size_t _mi_os_large_page_size(void); diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 094d7ab9..40f5d2d7 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -26,8 +26,8 @@ typedef struct mi_os_mem_config_s { size_t large_page_size; // 2MiB size_t alloc_granularity; // smallest allocation size (on Windows 64KiB) bool has_overcommit; // can we reserve more memory than can be actually committed? - bool must_free_whole; // must allocated blocks free as a whole (false for mmap, true for VirtualAlloc) - bool has_virtual_reserve; // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory) + bool must_free_whole; // must allocated blocks be freed as a whole (false for mmap, true for VirtualAlloc) + bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory) } mi_os_mem_config_t; // Initialize diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 18201ccf..2a5d172d 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -312,6 +312,10 @@ typedef struct mi_page_s { +// ------------------------------------------------------ +// Mimalloc segments contain mimalloc pages +// ------------------------------------------------------ + typedef enum mi_page_kind_e { MI_PAGE_SMALL, // small blocks go into 64KiB pages inside a segment MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages inside a segment @@ -346,7 +350,6 @@ typedef struct mi_memid_s { mi_memid_arena_info_t arena;// only used for MI_MEM_ARENA } mem; bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large OS pages) - bool is_large; // `true` if the memory is in OS large (2MiB) or huge (1GiB) pages. (`is_pinned` will be true) bool was_committed; // `true` if the memory was originally allocated as committed bool was_zero; // `true` if the memory was originally zero initialized mi_memkind_t memkind; diff --git a/src/arena.c b/src/arena.c index 33c6e4a4..473bdc55 100644 --- a/src/arena.c +++ b/src/arena.c @@ -50,9 +50,9 @@ typedef struct mi_arena_s { mi_memid_t meta_memid; // memid of the arena structure itself (OS or static allocation) int numa_node; // associated NUMA node bool is_zero_init; // is the arena zero initialized? - bool allow_decommit; // is decommit allowed? if true, is_large should be false and blocks_committed != NULL bool is_large; // large- or huge OS pages (always committed) bool is_huge_alloc; // huge OS pages allocated by `_mi_os_alloc_huge_pages` + bool allow_decommit; // is decommit allowed? if true, is_large should be false and blocks_committed != NULL _Atomic(size_t) search_idx; // optimization to start the search for free blocks _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be decommitted from `blocks_decommit`. mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? @@ -256,7 +256,6 @@ static mi_decl_noinline void* mi_arena_alloc_at(mi_arena_t* arena, size_t arena_ // claimed it! void* p = arena->start + mi_arena_block_size(mi_bitmap_index_bit(bitmap_index)); *memid = mi_memid_create_arena(arena->id, arena->exclusive, bitmap_index); - memid->is_large = arena->is_large; memid->is_pinned = (arena->is_large || !arena->allow_decommit); // none of the claimed blocks should be scheduled for a decommit @@ -434,13 +433,12 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset } // finally, fall back to the OS - bool os_large = allow_large; + bool os_is_large = false; bool os_is_zero = false; - void* p = _mi_os_alloc_aligned_offset(size, alignment, align_offset, commit, &os_large, &os_is_zero, tld->stats); + void* p = _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, &os_is_large, &os_is_zero, tld->stats); if (p != NULL) { *memid = mi_memid_create_os(commit); - memid->is_large = os_large; - memid->is_pinned = os_large; + memid->is_pinned = os_is_large; memid->was_zero = os_is_zero; memid->mem.os.alignment = alignment; memid->mem.os.align_offset = align_offset; @@ -644,7 +642,7 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi _mi_stat_decrease(&stats->committed, committed_size); } if (memid.mem.os.align_offset != 0) { - _mi_os_free_aligned(p, size, memid.mem.os.alignment, memid.mem.os.align_offset, all_committed, stats); + _mi_os_free_aligned_at_offset(p, size, memid.mem.os.alignment, memid.mem.os.align_offset, all_committed, stats); } else { _mi_os_free(p, size, stats); @@ -857,16 +855,16 @@ static int mi_reserve_os_memory_ex2(size_t size, bool commit, bool allow_large, { if (arena_id != NULL) *arena_id = _mi_arena_id_none(); size = _mi_align_up(size, MI_ARENA_BLOCK_SIZE); // at least one block - bool large = allow_large; - bool is_zero; - void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, &large, &is_zero, &_mi_stats_main); + bool is_large = false; + bool is_zero = false; + void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, allow_large, &is_large, &is_zero, &_mi_stats_main); if (start==NULL) return ENOMEM; - if (!mi_manage_os_memory_ex2(start, size, (large || commit), large, false, is_zero, -1, exclusive, owned, arena_id)) { + if (!mi_manage_os_memory_ex2(start, size, (is_large || commit), is_large, false, is_zero, -1, exclusive, owned, arena_id)) { _mi_os_free_ex(start, size, commit, &_mi_stats_main); _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size,1024)); return ENOMEM; } - _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : ""); + _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size,1024), is_large ? " (in large os pages)" : ""); return 0; } diff --git a/src/os.c b/src/os.c index d657d5d5..cffeed29 100644 --- a/src/os.c +++ b/src/os.c @@ -275,23 +275,19 @@ void* _mi_os_alloc(size_t size, bool* is_zero, mi_stats_t* tld_stats) { return p; } -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, bool* is_zero, mi_stats_t* tld_stats) +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* tld_stats) { MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings MI_UNUSED(tld_stats); if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); alignment = _mi_align_up(alignment, _mi_os_page_size()); - bool allow_large = false; - if (large != NULL) { - allow_large = *large; - *large = false; - } - bool is_largex = false; - bool is_zerox = false; - void* p = mi_os_mem_alloc_aligned(size, alignment, commit, allow_large, &is_largex, &is_zerox, &_mi_stats_main /*tld->stats*/ ); - if (large != NULL) { *large = is_largex; } - if (is_zero != NULL) { *is_zero = is_zerox; } + + bool os_is_large = false; + bool os_is_zero = false; + void* p = mi_os_mem_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &_mi_stats_main /*tld->stats*/ ); + if (is_large != NULL) { *is_large = os_is_large; } + if (is_zero != NULL) { *is_zero = os_is_zero; } return p; } @@ -303,20 +299,20 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* lar to use the actual start of the memory region. ----------------------------------------------------------- */ -void* _mi_os_alloc_aligned_offset(size_t size, size_t alignment, size_t offset, bool commit, bool* large, bool* is_zero, mi_stats_t* tld_stats) { +void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* tld_stats) { mi_assert(offset <= MI_SEGMENT_SIZE); mi_assert(offset <= size); mi_assert((alignment % _mi_os_page_size()) == 0); if (offset > MI_SEGMENT_SIZE) return NULL; if (offset == 0) { // regular aligned allocation - return _mi_os_alloc_aligned(size, alignment, commit, large, is_zero, tld_stats); + return _mi_os_alloc_aligned(size, alignment, commit, allow_large, is_large, is_zero, tld_stats); } else { // overallocate to align at an offset const size_t extra = _mi_align_up(offset, alignment) - offset; const size_t oversize = size + extra; - void* start = _mi_os_alloc_aligned(oversize, alignment, commit, large, is_zero, tld_stats); + void* start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, is_large, is_zero, tld_stats); if (start == NULL) return NULL; void* p = (uint8_t*)start + extra; mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment)); @@ -328,7 +324,7 @@ void* _mi_os_alloc_aligned_offset(size_t size, size_t alignment, size_t offset, } } -void _mi_os_free_aligned(void* p, size_t size, size_t alignment, size_t align_offset, bool was_committed, mi_stats_t* tld_stats) { +void _mi_os_free_aligned_at_offset(void* p, size_t size, size_t alignment, size_t align_offset, bool was_committed, mi_stats_t* tld_stats) { mi_assert(align_offset <= MI_SEGMENT_SIZE); const size_t extra = _mi_align_up(align_offset, alignment) - align_offset; void* start = (uint8_t*)p - extra; diff --git a/src/segment.c b/src/segment.c index 9485a55a..41c28065 100644 --- a/src/segment.c +++ b/src/segment.c @@ -530,7 +530,7 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme if (!memid.was_committed) { // ensure the initial info is committed - mi_assert_internal(!memid.is_large && !memid.is_pinned); + mi_assert_internal(!memid.is_pinned); bool ok = _mi_os_commit(segment, pre_size, NULL, tld_os->stats); if (!ok) { // commit failed; we cannot touch the memory: free the segment directly and return `NULL` @@ -541,7 +541,7 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme mi_track_mem_undefined(segment, info_size); MI_UNUSED(info_size); segment->memid = memid; - segment->allow_decommit = !memid.is_pinned && !memid.is_large; + segment->allow_decommit = !memid.is_pinned; segment->allow_purge = segment->allow_decommit && mi_option_is_enabled(mi_option_allow_purge); segment->segment_size = segment_size; mi_segments_track_size((long)(segment_size), tld); From a655c28b6664b059cee3d30b89771d5b53004800 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 16 Apr 2023 12:29:21 -0700 Subject: [PATCH 2/2] cleanup --- include/mimalloc/internal.h | 12 ++++++++---- src/arena.c | 28 ++++++++++++---------------- src/init.c | 2 +- src/os.c | 2 +- src/stats.c | 2 +- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index fa932cda..e0bcfaea 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -88,8 +88,10 @@ void _mi_thread_data_collect(void); // os.c void _mi_os_init(void); // called from process init -void* _mi_os_alloc(size_t size, bool* is_zero, mi_stats_t* stats); // to allocate thread local data -void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data +void* _mi_os_alloc(size_t size, bool* is_zero, mi_stats_t* stats); +void _mi_os_free(void* p, size_t size, mi_stats_t* stats); +void _mi_os_free_ex(void* p, size_t size, bool is_committed, mi_stats_t* stats); + size_t _mi_os_page_size(void); size_t _mi_os_good_alloc_size(size_t size); bool _mi_os_has_overcommit(void); @@ -106,13 +108,13 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* s void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* stats); void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* tld_stats); void _mi_os_free_aligned_at_offset(void* p, size_t size, size_t alignment, size_t align_offset, bool was_committed, mi_stats_t* tld_stats); + void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); bool _mi_os_use_large_page(size_t size, size_t alignment); size_t _mi_os_large_page_size(void); -void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* stats); void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, bool* is_zero); -void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats); +void _mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats); // arena.c mi_arena_id_t _mi_arena_id_none(void); @@ -838,6 +840,8 @@ static inline void _mi_memzero(void* dst, size_t n) { } #endif +// initialize a local variable to zero; use memset as compilers optimize constant sized memset's +#define _mi_memzero_var(x) memset(&x,0,sizeof(x)) // ------------------------------------------------------------------------------- // The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned diff --git a/src/arena.c b/src/arena.c index 473bdc55..f9b844bc 100644 --- a/src/arena.c +++ b/src/arena.c @@ -96,22 +96,19 @@ static bool mi_arena_id_is_suitable(mi_arena_id_t arena_id, bool arena_is_exclus memory id's ----------------------------------------------------------- */ -static mi_memid_t mi_memid_none(void) { - mi_memid_t memid; - _mi_memzero(&memid, sizeof(memid)); - memid.memkind = MI_MEM_NONE; - return memid; -} - static mi_memid_t mi_memid_create(mi_memkind_t memkind) { - mi_memid_t memid = mi_memid_none(); + mi_memid_t memid; + _mi_memzero_var(memid); memid.memkind = memkind; return memid; } +static mi_memid_t mi_memid_none(void) { + return mi_memid_create(MI_MEM_NONE); +} + static mi_memid_t mi_memid_create_os(bool committed) { - mi_memid_t memid = mi_memid_none(); - memid.memkind = MI_MEM_OS; + mi_memid_t memid = mi_memid_create(MI_MEM_OS); memid.was_committed = committed; return memid; } @@ -163,11 +160,10 @@ static bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bit /* ----------------------------------------------------------- Special static area for mimalloc internal structures - to avoid OS calls (for example, for the arena and thread - metadata) + to avoid OS calls (for example, for the arena metadata) ----------------------------------------------------------- */ -#define MI_ARENA_STATIC_MAX (MI_INTPTR_SIZE*8*MI_KiB) // 64 KiB on 64-bit +#define MI_ARENA_STATIC_MAX (MI_INTPTR_SIZE*MI_KiB) // 8 KiB on 64-bit static uint8_t mi_arena_static[MI_ARENA_STATIC_MAX]; static _Atomic(size_t) mi_arena_static_top; @@ -209,7 +205,7 @@ static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* st p = _mi_os_alloc(size, &is_zero, stats); if (p != NULL) { *memid = mi_memid_create_os(true); - if (!is_zero) { _mi_memzero(p, size); } + if (!is_zero) { _mi_memzero_aligned(p, size); } return p; } @@ -724,7 +720,7 @@ static void mi_arenas_unsafe_destroy(void) { if (arena->owned && arena->start != NULL) { mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], NULL); if (arena->is_huge_alloc) { - _mi_os_free_huge_pages(arena->start, mi_arena_size(arena), &_mi_stats_main); + _mi_os_free_huge_os_pages(arena->start, mi_arena_size(arena), &_mi_stats_main); } else { _mi_os_free(arena->start, mi_arena_size(arena), &_mi_stats_main); @@ -938,7 +934,7 @@ int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_m _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); if (!mi_manage_os_memory_ex2(p, hsize, true, true, true, is_zero, numa_node, exclusive, true /* owned */, arena_id)) { - _mi_os_free_huge_pages(p, hsize, &_mi_stats_main); + _mi_os_free_huge_os_pages(p, hsize, &_mi_stats_main); return ENOMEM; } return 0; diff --git a/src/init.c b/src/init.c index 8ab0f943..6dc61c48 100644 --- a/src/init.c +++ b/src/init.c @@ -217,7 +217,7 @@ static mi_thread_data_t* mi_thread_data_zalloc(void) { } if (td != NULL && !is_zero) { - _mi_memzero(td, sizeof(*td)); + _mi_memzero_aligned(td, sizeof(*td)); } return td; } diff --git a/src/os.c b/src/os.c index cffeed29..cf53472f 100644 --- a/src/os.c +++ b/src/os.c @@ -599,7 +599,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse // free every huge page in a range individually (as we allocated per page) // note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems. -void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) { +void _mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats) { if (p==NULL || size==0) return; uint8_t* base = (uint8_t*)p; while (size >= MI_HUGE_OS_PAGE_SIZE) { diff --git a/src/stats.c b/src/stats.c index 448248c7..a8eac648 100644 --- a/src/stats.c +++ b/src/stats.c @@ -444,7 +444,7 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) { mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept { mi_process_info_t pinfo; - _mi_memzero(&pinfo,sizeof(pinfo)); + _mi_memzero_var(pinfo); pinfo.elapsed = _mi_clock_end(mi_process_start); pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak));