From 1e4b6b734e06b9f4723826e0db375987a44d5aac Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 31 Jan 2023 16:02:35 -0800 Subject: [PATCH 001/102] fix assertion that was too strict (issue #691) --- src/segment.c | 3 ++- test/main-override.cpp | 45 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/segment.c b/src/segment.c index dc98e3e7..683e413c 100644 --- a/src/segment.c +++ b/src/segment.c @@ -632,7 +632,8 @@ static mi_slice_t* mi_segment_span_free_coalesce(mi_slice_t* slice, mi_segments_ // for huge pages, just mark as free but don't add to the queues if (segment->kind == MI_SEGMENT_HUGE) { - mi_assert_internal(segment->used == 1); // decreased right after this call in `mi_segment_page_clear` + // issue #691: segment->used can be 0 if the huge page block was freed while abandoned (reclaim will get here in that case) + mi_assert_internal((segment->used==0 && slice->xblock_size==0) || segment->used == 1); // decreased right after this call in `mi_segment_page_clear` slice->xblock_size = 0; // mark as free anyways // we should mark the last slice `xblock_size=0` now to maintain invariants but we skip it to // avoid a possible cache miss (and the segment is about to be freed) diff --git a/test/main-override.cpp b/test/main-override.cpp index 7242eb29..40787831 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -37,15 +37,16 @@ static void fail_aslr(); // issue #372 static void tsan_numa_test(); // issue #414 static void strdup_test(); // issue #445 static void bench_alloc_large(void); // issue #xxx +static void test_large_migrate(void); // issue #691 static void heap_thread_free_huge(); static void test_stl_allocators(); int main() { - mi_stats_reset(); // ignore earlier allocations - heap_thread_free_huge(); + mi_stats_reset(); // ignore earlier allocations /* + heap_thread_free_huge(); heap_thread_free_large(); heap_no_delete(); heap_late_free(); @@ -55,8 +56,9 @@ int main() { tsan_numa_test(); strdup_test(); */ - test_stl_allocators(); - test_mt_shutdown(); + // test_stl_allocators(); + // test_mt_shutdown(); + test_large_migrate(); //fail_aslr(); bench_alloc_large(); @@ -171,6 +173,41 @@ static void test_stl_allocators() { test_stl_allocator6(); } + +// issue #691 +static char* cptr; + +static void* thread1_allocate() +{ + cptr = mi_calloc_tp(char,22085632); + return NULL; +} + +static void* thread2_free() +{ + assert(cptr); + mi_free(cptr); + cptr = NULL; + return NULL; +} + +static void test_large_migrate(void) { + auto t1 = std::thread(thread1_allocate); + t1.join(); + auto t2 = std::thread(thread2_free); + t2.join(); + /* + pthread_t thread1, thread2; + + pthread_create(&thread1, NULL, &thread1_allocate, NULL); + pthread_join(thread1, NULL); + + pthread_create(&thread2, NULL, &thread2_free, NULL); + pthread_join(thread2, NULL); + */ + return; +} + // issue 445 static void strdup_test() { #ifdef _MSC_VER From fca492aacc451b1baded724563a3c1a2daa87d92 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 31 Jan 2023 21:08:43 -0800 Subject: [PATCH 002/102] update mimalloc-redirect for win11; potential fix for issue #657 --- bin/mimalloc-redirect.dll | Bin 56832 -> 60416 bytes bin/mimalloc-redirect.lib | Bin 2874 -> 2874 bytes bin/mimalloc-redirect32.dll | Bin 40448 -> 41984 bytes bin/mimalloc-redirect32.lib | Bin 2928 -> 2928 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index 83b6bd4f379ba4d9c0fcdf6b846c3b60867a31d1..5a2c459ae172a54c87ba5fece7ddb86ee6ef86f6 100644 GIT binary patch delta 6954 zcmZ`-3v^Rey58qBvEh+6EUBbe=xIqItnx@7KzMWvOof9Mh&-e`90SNu6qo?5cCL(# z7VoqZdMCJN-PPstV){ znXL7n??2!F|9k)Y-;Z;0Jg=vDhSPi=`Sf$1(RV-mG_1+_hR@Eur`&%Qejm82`M=M_ z6#Vqu4F#u#RlKd-Z$7xMip8GJGoKzY%rt)f$9;tV5+q-mB4*?f7l?c+3O^QX^q_EW zwnwMA0D}UF3P3j@b}FPTpqGVtlS>{2V}iKo*TU?j3n!}%WGe76$aj7YLL7@8+f%&< z-0r{O;(`1a;-(wLnHlE07=;cD!1w5r;ymF`^bD>a(2KanL!XLX;hudf2)T8{g3Ejn z7ajV$_2{b5FLkBDitg)#$X;X5W`SJP8Hik8b_Ob`Hiuc-@)HpvHyKaDDf11T@O-33 zR}vXbmz(rQuZOcMn}qn!CP6tLmGz@Xe)>tS>^5d;FsK%^RV; z*=q%{VSA`&W}P60_S3?gvPrLRLpSwhuWh48b7qSh_S4fj%fv0)=#`uzF>f3FHfP5C zs-1+y>pw{*GtcV3*(#9dBh@nyG6t1Y8@;D2V^ES=x3-4LX5A2UQ?>|%3hVEsw%l~F zV=rBvJ6n8yFWr%QM9kVtFX!GCNfG_E^s@MrjaKH3h({OG75R(Ab&Kia`9BorE~bO| z&xj3+LW}48Ruq3%NN*Puip_;Id+txg8HJ%=%@swlXnv^KI$aPPq5ks?uTX=t7 z&IVKfGv^qcin`6z?%2#Yu%faz(B})^n0Ksz)v4T4>2*{=HL%fSz<)=D8tqb|chSce zVUaS=>V&el5m7yJ0_kI<9?gv2HTo)Rh5jhr`oiMuVu@5wWf|J!`dTxo5l83n zyNH!FX{+$AmSk^CCB3pazVroK0ZnAgsB@C)>2m!Ei-Wh|Knn&^qrdec`eOc!kjRtz zTaAo>r_=PLv^vZV2395~%5+}FWEbB^Cgn@zFsB01vr}%AB^D*~_GMOXX*5G^+G}?h z6oTJk^awEXe%0h1YVsAx#QP9T(fHbTl1ZA#TtjWPIfXx9&bE3xshP&|u`A<^{}4%x z&RHF1Cr-(Am3x^!Z7ZgUDaN{5x&C#F2PRUTDSJ;Usy6POX>4Ge^Q!5gh$yyPEnbNwcR#h@0}_$S{alb~`-QIOJ%6O&P{ zRoDku8cc&iU|2=y=VptcW*T!coS+M{i|a7L%(JK)7G6196QxRfPq)SeV^5e)g zK1)jO9<(3N{x+E$UC$b`p)o$N`nWO+*41G(SDZ_1uu4E2IoFMv|B>k$Oj=9Z#!q54 zRN4D0wN{PO1;ChN!a%3M^W<^mgAnTrZi!zRgzyE=82 z*niVKA=>4IXX zaFq(SIn-aApLP|S>Pqo0YAabX+l-!Ax<1xnJ=SRE$-CI1K8SIf1zKOSK(Nzb$s=O! zWSUtzV}`Qpu=@o&0tj1-4m_0Yw5+r&y?70l-L?Ou&z8;4 zv&`rJI~cXP`TgUL3jgAS)Gece4SroLq7f>Y%IxzAD&(njRExNz4_K zX@ytSa@nMiQ$V7%DpHfHxl5$x?v!7nY(nTqgb*x5yQG7n`mwXAK5k+i$?V{DNHA42 z;RszFb|_x#A%?Ex_~DE6{PMfkx@4~jH8{2sgY3PSOd@*^9M#_O{9s2^(_gSQ*D^|| z!J81yFWKubvrcU4q3xnGSIjMZ3;XlCDJuPz!(C#f>;iqG%q^NPP}>i3>Rijz<>-t%&=lT3=V3WM z!AUv(U+0xY`+!13GyOTywz-?lUkW$(vp*jTaDmY^XM4{^i`5u$90npqWWPSojaGX8BTM!f^Z zi~5fj2?R$Fo9p^F?Z1EfZTi4{&$0=)dfyBNAK$l7EQ!!8$DH)nSk$Hnz0WaU{L}A3 zk2}7eoP8-DE56#yT6bU))dzMm>y>=^vj-1kZ~Dj9<@xm62j3H)3(B9;_P)oO{zWYufb0ptq!RasARfqUyhI#+A-$wXDz+7ugxU^Y5loWx$)ZI?^U+Sqv080Ks=(vi_cq+Ta*sPo`r9)?K8tt5Qrkq_0@8Y~C#oH9Vne)LE?^F0F=SNa}MB&Fc zZ~lXtOesCza?@yIcgV%b*%HB_@Ku~|Nbw$p_j5jy;v)+GKIh{MvAm`T#{cCMiD^`x ziK*z#%+bnGooGT({PsUnQ<^z%6}QM7ofABLnrDHFTdFwLPVgk#&?lz+i##8<_&EkA zc#fpne3M&Z97iU2nzS~J3C%L34bo_4Dx2PM%E2k;IOXD$J4NMrIOXH$=lJ*433E%7 zV{C%wcy&g&CBe~fT}@8fs-j05NrA*IHjXk!C-)4dJYC%4=IG%_xTj0=T++-f0nLLC z#j#86T^m`L+a1pc%3OW8B+8lL36a4Gk#TO9Q;}nwF}?HsJ{-Hhe{bb>rxt0nb4F&U ztl{#W%r2N_qIXB{;#uuX*G}xFyDMgrjM@nt6mC?fo)ujI#!>%1Rt#53Bbc z=2c+ltjw{Jqm#EqSh3j@&vJ8114pfrF79!sz|SpVj>8=Ne8M7HCG*C(#dJ!|#mrG_ zb9}j2xy8Y;lH=bkf&w?UG;j=X9O1o>XuZ#ha!Z_})~juYI@UV{h9)&HD@U25n|UyI zwhviV3f$b{;~3@`;hqxnWJ^+PbM&W&z6?*w44DpgiLXZeLY=u9P2pAkFvI}e2mBC1 z7>|z$??A?m;}dQfAx}Z9%myRmwOOR$33`{`$EO;}% zbwdIy7`Pb{WWm7aAYte};IAPO=n>#;NEABRLP!xL2HgQX1Q~|z0loqmVgA6s)Zr3m z0bB7*h9scl$w1aX#-QWuCnq4J2Au)VLk!UI{3GiiCg?crNiD<-9ZwU|3X!1Wc|qQX zSfS&5Aoo0sfk2ml6%af0Iv12v5C?Ajz}F!%bZk)~Y)1mnv8l-_NF{V^N^%I|g6;vn z0;z%?0p5nVp<~0697rv6Y!b2=;(?BBN=`u$LGIasDPRG>JrF;1 ztaZ`}2|&kcCS)f@1s%sODTaih<4HzdghZg@*+gdU#uP9fxcw0fBy{v`yyV7eg{-~0 z&DDq2A30dP`pEHvTdI#8JaT~ikv0wG3jas@hl+%Y^v^@tLMi=fXo-*)T73Q8^t$Qj zbfiDtFAa9@AnTB=$ct#YfUwX)6C*4*Z9uWb*uH+1+qq8hx8hH9lhp0sjsrn)^G23_sav$0oP#FAR6|gY=Bz7 z$KT*@uJij^np^xW!InU4ur=HoZH=@=+hT1aZNu#&?eX@p_C&{6hoQ^VB?YVjd$*(8 z*;Co$>T&l}^}2g&dmDN^eGPrSzUDrEUvs~|KhPiS5BCQL!UK_k=s;{BIv5)q8B7e0 z4U!mjM*VPd41^nHC?qGMMJKPiLiS`Wl#CnH&rHFH0`_;Eeqe@U{yaGO|ALUzNTr=svP!9W{? zc}4E*v9l`VW;m<{%F#Ev@LT(&1;c=&k zM>fX0vE3SeHTxc6^}mf0BKfqx-zt!iYz>i`tFJ;OZheHMEjQmghYIL$bpExZ(M z%w8Ft$dy`i+DF6R_mUiRKr-H^R#4Um-xr(@bJu!vhEPhDcy;y)3ZA4fo~~|7Jdg ziZ%;kWLvmzZj&H>T1U%@tEbaZKiTP_;`!opb@XI$rPyqzmx`B*wRZYl@tivy zKPDv6{O43^#@jjNaDOT4o`VoGRN~h9UNKEVNfj2^!__4p2-$1y69}F8z&vUx%@u#Q zhgOx&7k|Bn?kYVfZrVdfOK*zFa(dSAD{+50br|DfeHmT7V3}B1Mjv0WQJh^yhZa09 zdhQC}z3?AJacBv>xu{HRSwf2zKO^QY3IAfTD2m2K;WqOuK`?{|?>r<36|}T$lkrjmhD&ITx{AjA{ztU&a0Fl;}0|@$zC} z8GX6@3$fA=KDP9@K>c?Yt<)2DK8v?Vudw_Y####EzE(j-Pc zpfesYQtk4>CM$8zlA4d1EK~c~pYBmlj*1hzn2)vKs9}AifSm`VZ$ z7i&N(jIvAZpif8`RmtiHc_1OlSBq^GOUzXBh-Md1-SUOzHHv$)l{C&&2Vlu3Hlo)F z!#Q@cm1+|IMSrq<1^w(c-QyC=O6Hl%YV|oc{=n0aC7(^h=HATu#lvoLeD5f+)Ieg2 zSdP%F(q7iY;j+5x&#A@OO>$!E|D;mXzjEvBorpg%OPHuSm&22h6N&HW`FrLR8k!S1 z?_k5>O))C4>+iMkN!Z)ZtgdV9Z7c4YTxkpS*r>V+0b1;c^6Q*6TB7m^eZFoyr z=u&HSuDBV?a{gcGA?sZ6!acOrY89TQBi6Ow3sQ2;1Qk`?pKINOpy8|ZiKeWmCt_G&FSCDRNVH0$9{?EbC!e*+q@Y#&mZ*vAzit>~9K*1%Q>KiFsG(!8KmN68_7Y1ws>B<2rIImqGTi z+!fIqRwT~~u44tWoMfpL^YFc;R?b0gNVuV8u59($;#Wz!MrLGFjouuju~gj|%^p2z zyd&dxLejzOH}Jx)ZZc#;vA5by*!wo9d}k;-3TMfEDV4%~fOkDCcDCc7VC(5n`mS}T z?F<}{^l^l$gW{~(P3A=Qm#NgmW@Zu^bv($XXGK^h3es(ZL>J#{y!$46l>t~8NKZ+p z1_K$UWOHdWl?vglkXKLR_ioC4_+BL78Kvdx_7{G#R3J+EUm+8g9 zXk3c1#3(ZY_ z=1eSJ)`e!(tqJdharp6p%QC@7)Zp8Ba9$>ueJpHXwHip{Ds)76h%K=OA=i>;t^{jUr7SYPq!ICQcYYA=LzQ5?zlK+!^ri5zc55?Ec z(*KmVh>sT2Wwmp~nqqoi?YC{n;{D! ze%Mbz-h#v+e+WN!pg9X^>(01d7jl;X)ZM!8y(QnOriQf!VOAwhU+4p3cR=IZuXjWjH_Q zLK!ZU;VyG-oO2V}KXFTvTW*?$)nPRp^&F9j@)woWB!0W7oHJIAwke+8jHk>Q2S?`= z&t%%O(90P=#{fgRwCPjhTig=mIL0xZ7oo=IKhnmzHOUZ*JE7HFQ0vh%R4f*5v1KeW zx74LAi(K5|<>=>_9_`jF1UVDo7@gvonFZq*XW|?uIqF#T=-9<7FzQCsmVu*{qm6q7 z6fc=u>o~eNCfQP>FPV9*YT?#4j_K?z53643b}-#bqoB;u`>P_{5}Ohkm=Zb8c~2%X zk&d|d!!az5AGY(&ayI1}eM zImJ_E{TcN!HB&Q38%M)VUIFvcI=I!vP|3E1TLK&-9Mjpll-}cM>!{kBfn(as&%BBx zZgosy<7{0|S~NvYPI#s`$n?$Qii4jMfhi6;?w}2EBFd0{_VTCb);PwEagIrjSjYX?|4BXe|e{K489z1-TyF~V_-k2;Z@y%Hc(L{Ap0k>|!2O0);0e>#x ze_^o4fn)d`orGO?9|pP~*Ba~rU=x1-JD}rxfcyu<1s(T4nezZ49_YC5$rgwgI=&~! zF-Qw^+~?#h#P4oGFqAl?jRga5--gavFmMAT$bx}~AtC6vokljT=sMt9hy}U~JO;5sZ}LMKfY{K$5hQ0pfu!1CK$x&~fm{yO0*> zI1XgmL*!xNX8{kPI7l0G>~Zo-NB}x^GD$*$(8-SoSpW$^#~noe9uk3$TZQB~kUX;k zx9q|aG=^)3f1Y*bj~=0Ld-e|ZfvpD{-0Ke>X?W0mxZ&V_@*CQEy;QhN2d^&|qV)Rp zB4Gjj>iQ~STKMkx?{e`^A52)UzE|$m^y&JfKGLu0xAjl$wQEjAPsL7=jz~weW3nUE8R<-R2D?ICiLO9* zushz}))VOQ2it-zz5d>|-e9k{ucgo57ijA9^n3eT`rGq^e4ynV@Vd=DX+B&69xzpTb>9Tg&x}+{c zx4GNWZSA&o>w646mL6M=)T0aPgXW+$XbWn3b-jjOORu$;^fhVvCY74?{Yj;2-9SRA zTQe9}DxVxAr^iwKm{NO^RUg9todGeW`a~d#`p4TNsDJEK2=&K0f}NqxXlI~{)gS4K zcC~f0IS6$}y8S(D9)dlgo|d3L7zldT^{NvR==JomDQW9-^{X@D?{^NUbJ8+UH|WHi ZI8L)Uah>*@mWSBPIEP$A_&sD3{}(`gI>G<| diff --git a/bin/mimalloc-redirect.lib b/bin/mimalloc-redirect.lib index 059fb8702a71277ea4eb9995e0826eb53f88f6e5..de128bb9483df71a613fbfd3099fbba2f75f878d 100644 GIT binary patch delta 100 zcmdlbwo7b-1PjaM`aORp%dqTVQDo?=o}9&@Joz_^15;o1W?NQWMwavsuD>Sdu}wh8 lc5?6lWhdLRN=^1)7GiwV{%mH*$0`0RTALBn$um delta 100 zcmdlbwo7b-1PjZx_CHrA%dqTVk!9FfIys9&dGc=-2d15+n{8Qj8Cg2T_q>^$$2I{W l+sVNLl$~tLDmB@cJ(h(_eck`b>)7GiwV{%mH*$0`0RSkuBMJZj diff --git a/bin/mimalloc-redirect32.dll b/bin/mimalloc-redirect32.dll index 2892d459fc5302190558bb941046049b87488271..522723e5017b71b458afb4152bd6c7eec55e9e62 100644 GIT binary patch delta 5079 zcmZu#4NzOxm45G8n1?OH!xld_1Pg;>FvQU>=wC|AbJ-Ma1-2Tx_D*n(Q(&kx4_{nz&<*63#Nw>LhJ`QJ_~;eGh> zXE)~L_q`hn@;iSb{a%szmX2dSYJ0J~_~J6ZL-k)D9%C|p%M4e(nawR{ngHA*9Q%Au zg2S?nVN7J6t0gzsJ)2(oIuHTWL_T9{sy$fU_vd05*S_|n{-eMi`v8w_kOTBh;^6kM zm&lXc9`ZA8D|wYGWPcbL=X%+~r)nAI?zeLAm@2^IlPo+o?Ta)hx>?OoEi2p_9K4Pl z19r+V{%to{`0Jc!^4@x$3eHA*6#8f^Yl1T5Vt(mAZ0u((vhEl8 zguKGrO=(f`TyOwBgYK`w1v27IhQ;^>5jb@iJV&KrZ+wF{c~1N)h(i-0=eUuoGC1KW zE>`xYEl7RRMBhj4iD7fD1q&jyPhm5?L`!2VcdRx;euXut) z6{jL?TSX3fksqt}u$<;VWUgQr%Z`%I3-?u29YFkc`egj7ym5Ex?7Mzt*vq#_h5xy_ z8kkb}MoB}Fk*hpFzE`BBufH!UtTH{z2-4JwtdKXJRSEO6QGel@qb*^#vU#Of?r zD9Gkh@n8OYR{rybSK#sN_FdTj+V&^#`;Hmo+kc8`WYdXJuR*nun2osT6_(}wrfb=Qo=A{_j0(|aOOW4hG+aa)8bNVMtP~7DvL{= z3m2xoT3u}pdlM}_KlyyuPq;HXBCl5LW<4wXw3tHSF|nMmm_qG@P*m=e)vfH;2Hsn( z@uoub-cf7Gym9`_A7-KIFF;Ch?Rn#zeVFoDFpXu!-U^FKlvyc>R_4~?26Dn~KJc89 z>EU7zz#{yp;GxJ(6%GPU>5=w-LX)nf*H1jz8wX&N7wD%^V9P+Sf|@=fdl8t^T*$Pio*(23sm%tPsD>La=c3yw@CF4y@*ldsol?tN+2Um<&i+Au)+I4cyG4MhHKdKFxmID)JZG3P-Quwma1K9Y}w@r9(N0`EZb_`2QQ zMI^%)m#Y%jf9fNm+0#_B^F9+=gSj5*5E^IAMY`hP5$zaTewq! zn>o$WU#;th$?|qJ=EOk~*DDH85h@g&fYjh19kL;vrbQ(B(8jW9F^=;vnGU!hraTwi z@n)%6R^!QZRTEKH7wbCQiSzUly+XjYiOCykN8BBvjBOAFG|aS^q<)YUMgR6rce0eU zS34{A^R!fyA#o+pzry(UQvp&k-LR>YYIlmzyH#D8Z99!RW^M-gp4QP*taUb#(-HG6BvsLD&X zsS59lg26?4q+@9yD6aZuDTaEHw6{sP=CPvNsa4e59Lnvlsj6?0I#AvwD>YX0FFufO zB5!;Kv$UF4o=>9UIT`0qu$dLwLAsUJ)kS06D2uXrf_zfJv!!IF_RE1{^e)r45l=;5 zwk~=%7g2SJG)DfyA+{8KO6L!4`Rq~lF1;5-A8zKYUZsruN=`6_$#z*!V0sV&8d@1@ z=3($Q#JC0pwZjB!R0Qs3hujhdW-g!$-hpJx30GKLK`}_p(%&r5py)TY3^x*fQ~{U| z=ePJ2JqT??mMRg(NjgV*a^cYFo9<=!eyt$kZj;C17J2e$QrOyM(JoS;*Jf zqo5(iejfYIEpe626fyOclzk;T#63y5@H5W1*l%d#pMEIqRF-WJ;nYQ%H6`pnlarc* zh(bZ48k@H3yNKc!^YU&eD)eAjb*4p{ml4d6MC^?~e}d`B_YG0av3c^4w#+za!upN0 z6)H77p?XU2DOaKV@_g5*?uE3E82x>;0fP;d<|0w;7S1_O-q#ilbRmKC3n5*clefn; zSNi$@`hrwKPrM<$g-3cUA*-N?W2b$Ws2IaSYQ4{Jgc@F>5;{WOOG`Xb{vQqPzERn) z;E{Gh{3P-#ZyBYQGgL%}SLQGtY4P**EolVyK!1wqG5F@>Eik|p=`Hfcqws?Mpu>xA z@hw9x*=3pXRu%@uKj}^ub-2?vpXCZjpbqx=I>kl^yqwVfr0iEoe9pDX@K*FW1XGcOhBkj_B5YwbNhc zogfc>!+Q?W3TK!#nUbzSlaNgwjLdUoY~;{41din*$D4~3*?O&7r_t#n0q;*(UR`xs zyH8uqkcS>`;P+)X{N{g^C|idkBdsA$<@X&|pX@k!w5#jsW9qJsZ=X7P>N(;%ypyyY zHnB|P`-eBPS-#7R&_{0iO4vBL`0c}FgcY)az%EmaD32YHk*h$)WF(kDqCn8mrO$HIjBPTlfMsV#JL?a`8;I3p4kBm%!i-M5E zA$}H%g$$FwjD-YBfary3$nT!*q>0JrUtt6RR1Bh$I63po(-$?${3-w%kS!zdtebd% z90B<<64$z<56BScQby9YCgHCF83V~jo55mnjak7WwZ`yQa1szz%df=30CIu;^w5WZ zw1IpX$@-yt0R%x;K*RfyjkN>hj{%tiErOzu#MdObaUdy>@^2ZTdq27N>{B!`rIlbc zAiA^An0~v!)1Jfex=WkDv}P2#GYUTNwDDvGKbT-fVL78P0-m-VRp76KnFP^jRdZO} z0VP3n&*87hg5G@94^eWUS`Zbi1*@_F@POJgf{~1cY7oFM=qjk~0HVqW_(=c@AbGNV&h2XmPq@(lnQK^_oQUr%Qn06!=Qx<6~G5dagQ1yB^Ome=x;03f`+=7#R7 z0oMfT0trxyXH->}0E~eaK`wv{>LD`EDcHzFZ({)ltmmZ{=GZ1O|H3x*WAd99cCw2k z>qTwJEc@qv V$B+5H*>TF>bF|BS{N&T8o?{p_X@1ejDYGKM7aMa1T!)`S~iL1m(A~4KepA|o9yQux;TJ~BrmO6{gLjQy12=qryW2QNi z&FnHC88FY7XU&rNkfqh)w=7%kSQu-UwcC2$nzW{@#kL_^(00`(*j09o{fd3WK54IV z)H)g+V~z>Og2Ux(aJD&Topa6=C;jh?=XJxn z%Q}_5SYN8I)Q{+|>aXcX^#ipAjlp2B872&qh6ZD!an874JW|(IC)KUg#p{xF{`%hf zzWSkh#-uRuCZ$O*oi`1eE}1Tyu9%9=rDnCc(p+V}YQAP3HBXo)%^HgVkv3QwEt8h0 zWyUgRS+F!(o2(1gMeCCFjF&TG!=&M_y#8b*EMWzd*3E=^Q3rJ2<Rx`ZyNOX(PWpMFpu)bj?VK`^Kc#fD3U%Z4k4s|K~P(pY7zH9n{@UN?>!$BYxk lNu#OGR@YEBRX0;NTQ^s?Q0J*XRNq>Eq`s{_(j3U$@P7kX_lGCJ$iP$NL%w{^=kfGczXpzL*w@+Nr!ha6!=)Ff1zL$+*v)@~ zJcsMcq_ z9UjrT=eV4oy+jokBK=}zB$_cn1#%@<`{T8Pd@ZvLaNNmK#UZV6B$aH+T$G2dP_yq- zAsFHRnckzXr<8Zkv31D%Yxj`FOuL{hBRZ)opPH0AV#AKO;*8fh3s3GmhfK(I$!1BZ z5*LRG5Oc`+P}$5FPeO6T);Pe^fI}ClR`JBvcoG*K4~LL6k+Zr?)RaSpeL{2(%jcp~ z8e$%)Fu`B&!z>E&8Jof< zHyQ=lhJTQ^n-^vp!?T-r@O+4ToWHL$(uf||=#}&5<=l2AFTCm3C?2Ur&Hrd+B{(H| zL!_y|BuqAvuN4^R?MDUqW%`qxtWGU6gPhwLlW@Nf@#n7w%2u2j_sZr-V|ck>h|i_! z!@s=8eja5R5ie}nf%V6>9KluE`kL@s3HkZfpwL`Gnu`8O2OZcyCL@90+zP>H|ezBPsd|I+q_YOZm;<~@^8WJtBZz}Y2 zax`<=v8=;`;#qF->WOv7`N|(};W)*)sJN@neZX-N=g*pUEHtH!7hKfXvEaIVdFtWH zid*r-TfFV$;~lRE?-z!@SGtR@9)!zAoTuP`2EK|@3Z53Zej&OIT~qdJT$Fi3h9@M^ zrH0?5UDD7!BMnbvJO`(oAdQj5VlY=Xi=yF93DM662Q?X|wjR$=Jen472-&D*sR+{! z4Tge)7rHy#OgAaLoZ=Q+Xt0RvQjL$6+lA#e?Wv@pYq@}v3d5eH;?R^klEaRK;!F%X<1|N{lef_)_)oBboZ%Et z@~*gUNpZx3Y6hFqH{Pe@u(%FGr6aN3#X9?^?BpO)a4^B;TzCq;-g&l7nijS7U9zvN zX_GWHEIf8`B!iI+F4g(SCEDR# zruI0RYWD&u*kdWK_$zy7GWbT8TmA5p)g=_NxJ8E%dYUZm*_ipxe^Q>lS_(*Due3Xc z^|@(BlHM*mlCI0nm|J~=8A-#A<#U6{mYbqheDmbMXX=c*n46e$dj>C@J&GsM;+>~z zeJb2>KK&#f=hP<{_p;H&GFmLn6D{w)el3&y!0=_fHL!)^%BMfN>M4ye68#E`zX5)3^~t|bR*Y7IA=H&sTp-8uV@w z(%UjC`jJ5PZ^;Aw(;fHl;;nN?tk_$ud}%nFl5EEJPdnluIjrxA(?1$doE}Il614K1 zzl2VA>V#s#{*BF%#P+LSL}kxM-;8#U?;7$;aSR_h2~m}f^r^im=LC*7oPFe7LlHkq z{@d_0aT%?K_P5EPHO1(~WheodG#qju{ zVWD96iDM@^e7>HepXoW()z^2dvumI4*q3(`=i%+7`*2No?6Aa36IVI;9)M3`(ba&@ z$T{JlG!H^eld<|V83!R=OIgaRmNE!kn$)k76(Fo>a%7ck0@0c#dsoSJ5dJhdzDk}4 zF`6c0tK=AnsWdseN-ltir$`=|f9Y9C`X=%LP!6wWOcf+onsG6v6H*|}^fG1?(ybIj z+K#kI_wc-&mQ~2oKXS4RpgDd?i-(RHG1VD1Vxi|4J4om=#vzvi54=e+*M)Z$#bam&7-{RyvpbXIMM-Lg>0OUBOs9z& zWF#SI$XpleK(6lFoLmJo19Z_CX*|`P+X=}JoJq4TVmRuPA%uWiX_6!S>F(TFNQ*!m zh%})V8bH!q$JPRLpL4BvXa#%#T|;J$BamJNX40&VvShYf%u1x00C}OWhqU=Le1tw_ zSKPo60cea6W11ngrkO>`p1p*8X9_!8>g)|0C0vygt2*iK}up!E3 zi{IhoLV&Ka2&N}j52+dGq^yIJ^(=pU9Lb|Vh_d*l4(K?fDPSIW)Q;>JB{+RYbN0nF+6mw zIqU2JEWY0mHtwptQTd^1ef77hkC=VtfcbakTFb%O!*xHYyH}UAK4JS?Tet1^woh!F zeY?HNUSmIKKVk2)U$?(*f7AX_J%TiFTp4bo)jzCWYc4YXk9m{DXX&#HSYEW8w_LQ0 zSiWhwVSzy}KDu!WS#DY)7VX&@U5&M7wB|<5L``08VQp3INbS|y@mfutTvt&yRClRv ztWLB_)}7XX^}ID?O^Q2j4*=eb;R9P&jIb<2Lj9YG53Tw1AS6QhyYvfv5tgiObSyu0jTB(j! zoL5&^cOF%9R$8-cJ;SQLWKE!M(dI|p0b2}pCv85|Jz!T^-En)u-pOk3t@qb2vig_n z@lP{?1BoWI>D%>Q{d~F$s=n1g`_O5aMIZLkMl2c*7+pr%i8kX5dU4;_R7pGGs%)){ QuzuXDoUIJ=!|T@kFGx(BZ2$lO diff --git a/bin/mimalloc-redirect32.lib b/bin/mimalloc-redirect32.lib index 7dadab3d657e82efecf90d288b13df7d1ffab7a2..87f19b8ec0f7ae1024ff508a738b517c71f11aad 100644 GIT binary patch delta 118 zcmew$_Caie84F7UkK4D&RxG<%6dC%eCu^~*PnKu3W9qBk9M5XV$Qe;K{r^!01_q|d z_AJtqx3Cq#l<5PNOpa%jnq0~r%ejdy_Sr+IDznLg9Q80&(v#P7a6naVzRWR~2>^ga BBijG~ delta 118 zcmew$_Caie84JtnFDi#7Te0k7k!9FfI$4WdeX=~O9n;Rz&GD>;jGSgqmpneoz`(#X z*`7su@)outm@<8!lF9L`Qj<&BV>!E@9lG)ms>*D#AV)n+mGtEG92`)Un=f Date: Wed, 1 Feb 2023 11:23:20 -0800 Subject: [PATCH 003/102] update mimalloc-redirect for win11; potential fix for issue #657, attempt 2 --- bin/mimalloc-redirect.dll | Bin 60416 -> 76288 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index 5a2c459ae172a54c87ba5fece7ddb86ee6ef86f6..b3ccf78c906560e8c753d83102f88fed62f3c02d 100644 GIT binary patch literal 76288 zcmeHw3w&F}mG>yNiSlr)ggi_M^ycBlg(R35+B`u9iz=;6OcFz&DVAa@v5Ft69*Nqbl;E58nXIfvpYAN=+d z_Ky>OIH%iG_QN^rYu!H0<89pHb=GUu&W46YzqZMxc>@iayFn{mvremTtZ^;Q&7C-3 zivGKYnqxox)tlWaywv)e=YPiYzv+G*@h5lx*YmwRe)IXC@OWysh|lx2c>38~ZPK_Yb671X8%&(sWI=4<$!1QW^f<;Ys}!Pnp=C!F{lQ94jOuP+v)MBI8)=Rh%F$mMqo_zQ~tNY-s{{*d*j0 z9D-@F-__*blvgRyg*4+YlExDY0Uk?b#0{G&1ESnREF`kjpZ_dJjYF&fZ1 zUaViN5VKgYu`&~Qx}>~zorNdB7g_8Pj0fj{1Wedm#20w6ezB4~x8Q3fqeY*&ZXbZ$ zgpZZ@ygn}tILDYQoCfmhbbPM5Mle=VAAvpRn?H!}7L@)?e$LRRZ!^&Dn==WQ*B9XO zG2qbmjrF)30TO-o*|=<@?3-8P^4gWS{Df$C5zRxF61r%xfxbx-aVe$jk4fHQN-ek+ zm$xS4awHd*LaJMRIWBL)^7~#f*p$()5jWxU^w@^?i97E?@j0Ex)NHTwZH?GWnUxb=k0j9i@*`}6AAiFpc8;^R0R4?qZ z4vmCpn4^$56v6WjYMsCZ5E6NeKua^V*=DOy$$=hwn;-^vR4{)5SixRy&O1J#l}3!& z2O?60StQM|ZZ0?Ma*RYZu3#Gdpuu@qiGu}ZBdTSA38!tPP>#jMBQbza$W|KLEXu}n ztYRCG^RTUJ(7q$@IBVrSa|pH2Ut29FnkXfRwOmu`qs^0d+{W^bmz!3+m=9-h$pg&$(;Bz@;CbF1X;iUEXf^>jkA0bq`DSursN7-C>amAjf-1GG=Gae1 z)U=#nvKfxrxY%cv5vR6C1E^X#$yy@*TkJbm{o78M8%E!EBW&vo)=4lSV0L8v*dF?U zJv8G*aL}+%w%9{C$B^L|<{U&S_6=0}DzB8g8+mTX7lB!6EBTAoLH>f4Xfb7jC!iZ< z#dLe{L@Q5Clqt*Bd3h&Wh!Z`*$b~5x{S$ig&7$c4LDY92E>L*@8w75uQr6bHieFV` z{>N{TPFyX<;JP|2sv(>~Viu6a3#=z7>zXp+mYKw^ydoqnWLItwN z+$K`vS{h;~11MWMsO4ogi|<1ihC68&(~M!b0zKJ{l4udv9-MrlSzumW6Xp@JiXHlGakoz@S=B^3qDJb-JnV)0mgASkH)Oe)7DzR5_Rv z)*IUCaicVfboH^Ni<~~9#%+$BM<8V3U z=4iQKG-JgyGyhd`D@68Sj`sYH+XD@N8SCGPk>ecKPfkD>wOqw%+p<`#NRUjEAZZcG zB9As}h&v%ibh+CNxwBum@$@Y-gdo;kGVsk>PHw)zSwZmPgCW|A%3#PnX z_YrC5DtD7ymvZ{f@q&o;GY8fGDjz%V|`OD zPRg(0h)&riXwc3=%AT_2yHDA!K>PuiE=j-EKS3n4z(+7f=2qJk7op`Y?&wI)qChc( z(#{*b98nMLdfg(bJE7Q2_{UccQY^k4SiN}h-o*>}UjEf*o}BnC+cc|f62r_iAF zd!#+us>|^Ou2cKIV8m>1Yv};^iW0J3fPxy_qZXYHPu#amujld46pA$Yb44PM*Yc%= zx|yUE3T{u}gV^PL@m@@aAN(J((H#~Rm`tNj!%rY9@zZ1lKQ;wTI7{Kne-m-pt8tee zD(Cm~8o9O!xR6!!f?60|LK=S#d_sU*AyyNpQym`Mn0XvQ&wqpuXi$b=d zKb{3OmIpysz%7oaR$6U0pnaG^YWp{Qo0PPNdZOP%$6^!sl=uVIxW3Q7gN}xC{`5O4 z&VAdjc60lhQQ`mbHdXg8?{#8mf11XHzP%AP^M^kT48Vmxi6K`0YzUv7*eC8klTL#A zVbuK24!#&k7CaF}WXfwh^5-y>#xViQH0vU$fj zxwS<@4jGEzC08bjd6~p~W)x!l#rio+-pSG&3rul_RP75S^AjKK|rzb*;|{~W38RFd$G{~8Nr z@61kC_T(qgRL*FW{a`ZI!3)Kzw*=NGaLswEKUufsq?;QsTr`lG?cm{iAoJF~O9Jg& zbTo0`kCE)>A^Q^Q`?I?t^Gnf7p>NLSHdB(mb4lL`msZ~ok-oqB-BHd^68dBGBJFRq$3xe3_r$CU&3K3jQ)}s+gg?)T^0&cNm+H(bi(+B{@4q&MtCa61fVI`?|<&5V@lw=McH)MXr`|r&~OT=z}#kOkF7p zOZhWic^C|#@Hr5p_X$Ots7~+%MJ#Vu7p~}e8_5fWi5Q*C`EtJUIUINdSA78y^Us18 zA|oW1rsUQACv|AXW;F3U35)e|sujk7eO_$Q$nDTHcTJUJPV6rsWBTc~g`q~d^5-~VltDZJuaBe2&S3n zMGunLi-*Ed!~-+FjVx`^!uxuWf4{ExiEHs+LptI4NZJP+`gylQo(|KqiOJD>ehoVf zl~>r?Ot75icF{Rr+WK6+tQ(&3g&+=dUy074+C%Wm6Nd1Nd%-W}!JEJ0jv1R|E3Aue zlC6lbkHz^^(gO=B#tyc0WE&=~pWfb^51~EW(y-`f$EdwD_qQF;Sb!U9X{U5BV&_K# zr)_5HVp+J{Os7W0uNJ>DFk{`zCWgBzx}n%!jg9_ps?lc4FOF<(S-7TLf?ZUYX=(CMjzioxxG4P4{_c_w|;vydr(oU4198iRjtX{Jb=rp0FEJ!jK44`AfvBZ6S0kcx>+cV2nVt-frM!vat`1ONexOM)haAQs}RaTh=Fx=3^e zd$hR!I31x!g{jkebDC$<4tne>4|MF*E39eHwNxMl!=t0YI{;HRjUj&~8wwNCDVQ}@ ze=DzxL%;j~oeu3_=Njf~w;30t=OV&A6>gbZ3Xvq64|AI*rf3zaU}d*r6HOnqN#x>J z^=CQZFqM(ZE3}_3ov+bMD#L^US6PTeSzwZHzd&k&bk|03Wb}%GQ6({Esb^}4<7mvQTPi1>6{)Qkt@oHgGe*%J3=j#va6XT z9?E^?-!N!2y@ES%#z$!WmY711b?^w5&?Pv;0XygT5fuef<>J8I-iGPPpI|~HolL<} z=*b;|n6Vc5gicII4XzmPRoDu<%_Q-@2xe4yVW`yPq1mak;*`Wz6kN*u8+f8=GV|X~ zsgM;-B=9W-ajBgc=kOFMw9L#OzbuViEeh$7%C-~_qfsBNPe-mO_9WTyzGmn!=A);t z_i|0t4{$M!@EBoW#@pYa^~X#PGj`CEhHPkxk<4^VE#>Vze~wI-zY`LAte2-TbpQCT ziIZ3^TtiRd$5BPi$*`Qjv@Wh7y;_z+a_|IQg?8gH+0C4_Rj?F*&emp`hFm z%HEk&P>b5NvGIL(V|n0)O!r3@KN2HyH)j&ZOTC2z8>TbM(>8cXifA&!uVa}GS7u}5 z!=ewrh9RT}TKrn|K4 zC6U;2K*U1U+5*PRbPt9tjvb2!-^|!a+fhFt*5_XbqzKcc6;V7t8JMvMOM{xtIeow4 zJh{zy3?04tELRc-gO&L-oC7mVNxaVyFF~R9E^w2`YcRXQyj_1UNpFYl1y7IC*t;#FNFWnC^ua3qP*xEN7d zDAPh4ECFNaOU3iJo#*KuSi+rK^aA#PhTclAv(z+aN|Cfvk94dd>@kMa4Xx!NWjFfXCuB6Fk;w zn}$(JXgn9YC1Rovo^OkchD^LNFr|Sh&$p`=ZHmNB2i4S_$Xc!;9y?-_LrJl;+4&tw zuN}J=5Tj526!>r-r_TcF`^tDDBhD}>_ z4^~p%^JTUsA8p$CK143PsjtcqK26epEVG$?7%})QSh~K$w(z41tU}T4I2Yh26n?A> zn9KZy2^1q4jEuJj&(R}YD)(|y$X*nExEJ#}{UKNa;Dhj1n;HS~G(_07&`Rk* zt7du&f}RP>{8F}cT7=rnfG)1d`)&7A^29K8cH%=UR4EWU*_Q&qfgpHQ*qp9cnp zbp~l5O4M)X`3}-Uo~$$@E$*r*rHtcP7HQhTNn${eB@ZRejMO4-w(s0w{>{L^gAY!j z6bz&}FWioqgl~WGuCnN+w`u92gUBg)k9?I=AtxA}0+RtQ($+{aH5M(#+xyBbgIG=n zIy!+IIuQyK2l;Y#nr;Z=s|h)C)O?I@CEKRcb`+}I7r~;=E5{T8Hoyt}eb0gg3YxbE zcqfBV+HgWS(Fwi}ZibNPSMchBs?C*4(+qi}B&8=vg03_s3|IOsIO4uyG_8ORHWjkK z#&`}pk=rrv_`Ahr@;p(ENrk5PLiYS<7%u`q<$TOYRFZE5>6cl&*>U??hCTlU0Q&0yT_9g$eo@JI$# zbp$%dZ>Dg=#AJa}M}NY(<4Q1Kd;SR!|0g<$l>)oFZ{uJ~M>}LUN56l8jB|c)$NZe_ zpO#~vBgTGw3N()cVx=Ku@G?!!1mIU@kRFWY zq&9&4ESWaCio0*#)tjy8YWEMgAAQ#+wP~EkUm+>c-z6-C4#vN&LleY7!%Slo)zq7} zw4KPX#YulYNnhF_Pj4%J1y5dTF&qv^+z|b44<8F_=B)$H!{cARqSyT}qA*Rl^TD z1W=c(x=P|0fL@)9PN!~(M;EwhD&^k?!y~5_DugF&=XG;SZU*Z$#42B6yaG7G+if>l`B{v(d5gVH zJb7WdS;GjU9BwYJh6bxDO6_AAhk|W6JxTnos zdJ};B#VL5Of0rtY4(;ki7LQZ6;|Ktoh`e~;b{&_DWg#p>{HYzJXfs-Y)mz*dZN+GlW!_6Vd<+dp{y~k+JlW4o{=-9L-%dv^r^E>Agx zs|k$nGV!k`;KpZMfM$yW6Gy}Irqi}sni>3t=p)jb$M3yy?vlOZD%#1@7iZ}P;0Z8` zC+KvC|MA#QW2L8T*(UgZY`huOG!OL16W8rj)2wQ4iq}L97W*m2ARA*was*QE@>YsZ zlv2#v`~*-Ep2-Kr+x6Qgh(z}G@k=6obfRRh7Ix0PGq-Yk7%mg$OEaD}ov?I5aF+1T zOxVIQyMf`LY=UMA z;MsXgGjEt2Hy&5u9zqc`B0zjCVh!$9V}49)oH&x0jlU(^Pv{7#NW-ct_$HoqY=V!2Tmh?r+gisPupYNv8|_UX45cR86|gI>>LK( z!nWQR-^`#av4Jne8-QT4VC;-~$arJl1Ag>nlK4y>dJmTZ*^t>x`@Lph$hWWxImBkr z#G^v}dusfl`p0xz#{=o$${yywF_99Q-UFM+qN%j-Bw4$h7+?-f?uq{X1ZZ#tu>*hz zQ=I1AAIO021~VQIuZD>=@wviaO*|9M-G}?L=zpVTs|_z4(Q7?{tFYZ6CxNAdUP1g7 zi70@GSmIOWcMaU2^*mvgz)V;=~g6Dgp>*#AA1j|+sZ&MZTjaR%yRqTI(S6m`1en3@Rg^C5LVnMv(998i- zQE{rQ_zRpn@D>TMRbnufK!!>1JE9MPUrh2Vv4o?y{EX^PO91o>C_0rh?o)VclscU0 zZfQtU98n>k|N0@SSdf*8W}4i!@@67TqY$6qwfgZe3=s}D*p$x#TX-^50TNu^W8VQJ z{JoD7tQLxgRvwgtC!9MQ;6zVyHePIDe%yWYB$j`=pCnFf(;)3*acNrL9sK#xkZ1QJ z0Yrzw!Dp*@+3_^ar^so&h`2a1jc=Ngq6A9Y(1_DNh@LycBzbAE}5o{h1to+SjF1-d_<$0!O`pSNzdC zgDPfJ6&~K9SG;p@McL<6{rK}NxIL&j6>5sxLo9;G|6N8Q3HT-3rxnjflZP#hzJr-A zLS9<^@lyuS_YUYQ^v8A`c@y$@`X`7lkE4Q_0v5ikR zYH`Ge?!WG!3NpYa#w()|N#j2!lZ?1F(*B2#S6-VWGj}+dL$5zcvZf9vi}V~$UW$Hy z9&P|K$>TfkB5ZH*z+W8rCwKgJ!wHt4W`vU4$GsV5b$mq9)E{y^$kd-ql??v)?=vN@ z#9|CM{^^~9BbtW?Xz>-)5q%M_qQ&OL=HSbJzq%aitI;Bt2|Fryc=VHeo=(DPKr z_W94tT-IfZ-4est;_mAqG%l2(=lhQ6IOI_*)Y+Iuoz+sEVRq!n3dQ|Z5-9RUmH%GM zu5Z$PC>;a*2Dj>5rMz8V2MtGr!E^Ktr7HZsBE9JYmf7-dUYsV}#943_M3;is8=Fe_gK61ZJ0tNyG0tNyG0tNyG0tNyG0tNyG z0tNyG0tNyG0+$>C9CI0;zhVT6nuPN8?s{h({t0Q3*Hz>8x~l!lHNV$c?OLv7EO&Xm zjb2sW;IHEOs(P2t=iEZ?5Ax5g%Qsf~TwdSC8mF^iV~uO;#z8G@#6LA{tgdr!T3oGa z;m3QNbv_a@hQf8}KYb#y7)^;oZ%0;^JHFw(9u3cSflVYx4 zv3_l(ZN=TTbt~4~p2%;zJ)JmKwBojsvURocrf6JP6N_m0#3|nqD{l z1Wp#nN_x4^RUPogOH=5JYFwKFTN3K(`KF zwMK7^%e!2a==?4rBG9lXmBd*KxY3j6Sg37+{VdmtnhK<^BnB7^bqFKwb8dCvPY&;I zY^Wi-K;N3&^?`cr(}8-A=5s%Qs+b?zO>|XS#N$ANa})H0oGmWDw$1DIyEKo>TkrPK z^wKsrdNpTFjTcT;^Z8+ES|R*efoITAh??Ax(Vhp2yBCtm4>Te9fLm{HD82@#&c@<; zajPL93(@QfEnbjL;kW>qinDtgVXT{+)%Rt#?K$!4n4HyoL8>D3)6gZUs-7T@Gl@K7 zRjwuvELty+d0kMT+UqCDx*^Gu1btn;#ySiw7LhqvN63g>G*~q3`YccKvNVbwuWKtt zLF2k}o0NVe!~7I>N)A?x)^e>H|B#8X#{5^~Cya;>rmnwAz%rDUb7{#VWj&;ba)6XA zEO4h9jH>6Z_rRCvoy5O%LKALcBxG=1`KO+$I@BaHDDbK7$$3XBFIQ3Ftn~HOy6WnR zm*^f`2qH}NDvFiIsvG*gglcL&m#f}K3T}c2RvrL;%DLI^@8-69z`V$B0k}g1E9h6fI8Y*4*@+iohM0{dts7m0XbzPEWMdBMQ zjq@c)!~En8h#8(JFisW%iI0{E)vC-YHMtx))f2^#UV@yUSHMTUwu(Cp#SFd#vM!d( zZE$U)wXD|LxJ@-raudkO{B9rEs%l+M54_hFxNp4{@Bg6Ucx{1)#22gCmIhZ%LS-MIy0G?_Cm=+7pAVR!ub-^}GUK5=19dsnj=R z?d0tx^pCi!lC)0kXNo+%|AT8L;1I7w{iF8cYkO*QY^}d;9Tt8&CL5{8`BP~`2kkZt zTPj}?P4caEmCzlq(;w&7<)$s8dipphgCZh6Iz7d(c)!#jtFcJHQm-eCP0uR~g?NEa zT#j_PaY1@Lj-jkXdJ(RwUP9SmdNSmC9XXp)x*PAWKHHKFGf93jPf|mv?WA-&S$|#5 zpzecD%!8UuTX^Sr`zbUk(7mH%iRBVMk6sS>rM-h(t6L{E_<=a}9!=NI!%r|J6W^tcdxyi%~`a&$XO9t$o{ zm8HR2Z*K^mB>4$4;+l~Z;ICvPj6o7z?-OT_^QYh|O67PenQ3@Z^ooy1iekE6N%Kh# zCh46xPc!f&^+TTvgR$m(Y2+oqrn;B5-2}drUL@eFw;%7aE)X zuRZe4kMFNNT8Ivf(J3tjizGQ}sppqk8$(Sp`?myW-Uus(iTogshwi zS(kaoDgHTGH_j=Z{gz&Ta#qgdtO<|isB)3_>iJZB=42PlnK#=sjR^^++1WSFE}mtY z$%KZcNm&Jx=3RD7Z%^c3&6v;2D#)8R@tBT(s=ild7hE}Srs*ao5HU^5E|@ki*R+v| z8m3v<1+(T&Gkt)G5mEVRDlhe&uJldq|5e#HUR9iL>SR+n0!sdLC12?Gs;=ME?1HKD zCYzQsfskp?cnW>=_EY;aJG)@^yji9j6ke(FQ|)O^7U3NquT=X=#V^%fg*`v2+iR-6 zv$IH_w{-iOm3`x^;%TPo3cu9xPpvQV$8`B}yr!w~O6||gtb&>IroNSNz6g6hrrYP7 zY}=e$XPffX_@%Z#C97b{ya{jV_9gM1rtlT|d-d^4#Vb|+)V#nuK0i~-Q{`Q3K2=_7 zd#QK~SDwnBs?TukrShlxL*b9Sy1k^9r`Au+57+)xSp`?k%Re?<`J>eSq~?Wv9nM@(U5H2umy`_BDig2tI_5A~-~;{1}Isx^A6Y8o3{$rO5cLlpI_xvIv^59yO={IMIasIv)D zx|>00WPC=BfcQz}1pd#PnrT$?Wt>nlqGZLLqW*}O4JI9dLW#F^l(@H78o29VUPPio z;$1sR+ygeU!Lc3_lYG^CD6m67Q~2;@)fEei3NLStAnfv!lel-M~E# z=0&8U3h&S;*T=g?S??lr{99zrmv|SA68B~U_feUJUE*CkO5Cjm?xVuJRpNbcl( z+((6bMB;sJl(@4EhJPOo@E0WB`HzeaKdXWJXkceW67Pyp;@)oHJ}TTj67Tj=;@)fE zJ}TV1B;IF7iMw|9=={^Dl>Hm3bV1@h+VjZ2At>J4u7riXj`lqAy;b&ar!}+0U7bhf z%e?BL;~vA0C@d~zyinq-&Lfwm;BGN+zm#z6vZ4_7vQ^4BU0BE@h%i;;x>r zI#O_NHgLa`aO<*qCGJe#w>G8V-fQ5lV|6JL?GopX6ud8G8-}dG2y`2`4+iBup!7=I zjr*qe0PKV&&Pm*j`=$g?-h*Vn#ND`WdJn)(Y~l-I|8CqjC4w^c>9cYlX_5FB%D6+G zGgf@kI42m3QP?eUw@bVoGVYbQHygN*1?(&$ac`D*@0PJg;;tFEj|uMG66YQnt41uF zlKsoUT_Ruf&2jv_ZN+EAqeI}F#JfKQ_x239FEwx<9ndMY%wH6CtjSmvYt`xP-D=>j zAbMA^RpLF`_mrS!vh+*%;VmKOxvl8ci8JpYXe(_!rR~WdD2G(Kg6iS>&`<~vg4Hyh*5wJ_#)q8ri zGOw8X4SQEmT++B*;;i1&>q^00a~k`ZOA5FYc2?pXO~E^2oXd;_+;N$IC+yiGV^v_? zXxO>JVl3i9iTBbJ+?j#Voc&D ziT7?9cgwg-ozEEeO=E)lIfb+QeF)=w2V(;BAfXo|?#6lKAZW&vVEK|*SBqp^A!Al0 z=*@`1=Txk?u!))8!_%5NQYE*AUL^HZeBc#P%bcN4gjBoEaweI?`6eKS!toKbvV{ z|7(_sZ6}%eCUyv6BhnGXg;yXA{tCp$5jv6XMSRl-OzaV)9f-$WiMmJ^A>NPh0`VZ8 zJ=?^}h>lph%EX=^e#GBGs3RW4ldcBO-=iGyc7%68??C*(T*wFAGtb0cM3{~AImDm7 z27N)=ve3jni10enMTn~q&JaK1FCsjFbO+*ogihjLgtif0K)M<6^2JC$4t~VHLwEvd z?K%^iR)n^Yu0XsQp_6zJXDxxAlt%ngF>ogS8-NYM8KNU*H$s2NFGBqJk6>ItKZm&N zqv#9qApRGGI;8s%=PpBiq#cN7u0a2hZb$rkgq28JR+^Y;75Kk~@k0DngoQ|VA-?h^ z=t*?Mxi_N?N+bR?6*Q6$mv2nvjjSTUW9lr!b+rj5x-!A4oF9CGqJxQY)9Ji zF%z>RypFU3@%IrXKY_j=e(>YKAM~P6nAl@w&;j&*#4nZuE6~~PCidAiz!T|ni0`a` z9O%%Dxa|(K2YN5!m)AlL=xiPOv>q6O-i&x%C3Hr*9r5!BD6ZU6X|BehY(&s8vo_W-b6S< z{2NTH7~yr|N8E&P9%=j!Bl|AGJ4oZd0h#%3VDfbn!~fN>4G6Q5#&7JhFCZ*L8ow{i zP9v;D8oznW*geo6Y5YGDvmtCm8vjwn9zdu=8vi51ok}UW}6UJB8~6Pvwa9HW#4|=_14% z5Vj+Y?;x|M5FSAq--~7cf$#*<`0g58hR}&LzE8zk5MDqU-|=CuA)G-P-#cMT9MGS5 z5H}&5M;gDc$c`esgEW47j{O>8@{{04JlzTXk;ZQpu~LMENLL{CBdkOk-<4x~5z3Ip z_lVd}5H=!>@3ye)P0$}{{7x^cL)eZqemj=EjqnK4_&rlLzXtjv?LquGgoRI_FUH3} zz(Bx2;5|m5>9^x}i;u{-Q^sdx{DzD#$avaYf`74$%Va#(YWMbwcB!4s)Mi3HvF*Q> z{%p!#>#tkg;P#g^_}uIKUU$P5_S!hm;`#NBl@0Fd#u}F_X1~utc@k}XW1O9_CRSGC z{g}&NT32UxHq_L)*eTQP{t9oS-&O6u)9v*KoOR`{`bO{ljNL%(gZFm-oi4A>-Pmw@ zpnellj5U%JTSK7U<#qa9WsS}nS50|iO`y)@V{=G@I|456{kDd!Zf|2ly{o~0r_<|p zZbECHp_;e38)|F~e((LaHF`^GYP>EVWH$I~>gpCFBo394{{IdE6U&EA2+hf=;hqeH zqBQgg*^Jpy<}qcl9mX~$8MZXJl{Yul*J)eDoLjZv`o%>HG*<&=IUK;M1(oY>TeNh6 z=JR9vI_nx6T&ot`@A56U`KH{7E1f={tA116{Tc)`_*N|lcpH}cs%u^KPT!(>ceS_C z*SOigsJgL!xzks_c;_~T*iN&@iRQ1Evl)1LL>a;q#u3F&qt!~)b zc%RF=Knu7_s;l8IRxQ}9q1Xy>qC(@;ZeZGHJ%cAQ=d} zuMz0NeJlX5bmVtbbhJLz{#4IXY>#=5bx+gY=Dpo}`}bYgSG2$Ofak!$1LqD#4{C=t z9CjSueYp3Tv(H#M?MEt(v>u5(+w&|t3Lz2PCnH+gVeM$@=z6OAss5)5_Z01^-P5u6 z;NEk4^Y?4}+YjtMU_N9ylz*u3Q1ju|!|jJV4qteNb(%Z#J3U95jx-->KN5Ym|Je)A znvb$x+*d=MqocOt>{I8Ss@St(PtV@oz1IEq{T2Hi`@0WB4)h#2d!Xph(nHom_Cpni z4j%40+83r+ds_GG-qXML!d|w|vafc(XMfZF*8S%WL=W^IxNv|S+;GTosQ2*M z!{-k7AGUW^bZ+RZ?IhV}kC5DoqpaV=Y9N;+_H-0JUG%i|>5e@I_jK*)-V@oAzfap& zxUXp6(tYjwckl1me{g@-e)B=gLG57kq1Hp~hjt(8ICSAKd&c}s{xhD=rq1Th*3S0M z=#l;-7ml!J&CfcH)*fww_T$)Wv~jlMTt~E{zvDtj#nT&}c067CwC8D%jgNtVfq;R) Hh!FU{B>c_G literal 60416 zcmeHw34B}CneUOYi?TQtkbqfQZW4!x2?rV>YFtQUFPybrL^Y%?2-f<-KE~!xMgnADQ{igiEI&8YV}T%JoBn`|7Pu{NL~r@I z8H+6l2*N@F5^ZI6_}{Y-A?PN4^@6UAlo4&=vi&FxT`&}+j^x+@L*7BJr=#;XG=s2) z_yJSS&lp9wGC5`9#lA(q!Dnw~Z1Dd~!lhvjE^h*_!K-h?<*PT~vYSHtX5#WA^xoie zP>sRdTX7kgg3FhnA%l5D(tkBB|9lfJ|NBZ@u9}6*?`Pw39hIGho)~quq{7Bt%(a7Z#itQu;lf^Anf`E$f$p)P zJPaoB=cup?Epv_%BTY+1(rEb#saB0e0PFT5CyPu5WZbaD?Gzfp57p$xLo)wL4oke2 zqAs@e0TMzhc$T$^25M3~bV*r}lhVtIY&2hi-Zp9>Z(keKMZsUFpUK52VJto+1xS~^B)2G3sKD0rEBf^I(}*65h&&jwvf`KuT%7@s^}Ek84KT~Ke^&I4-^f>LU%x`ZtN#*oQnRhgj!$ms^0!h zy?sVLNJtr`LSgb4A{@-T->kQ>_&=i5Prps2^8W*Ip~IJpn^KsvHVm*b^jTq&66@64 z9u{_xU&X?9?mM>LTyUza$cuJNCfvLRP0`P|jNiZ;d;}LrTLGDLXm^QVv9+@JEUAM> ze=}TErMEwix-{aVwGK~uVwwoWTqsZ!pDJVTqZn0CWc(wTCNV&kC%!*LjIF$Cs^jsxkGhtma8ca>eF;LLTY!BJayj&SPG#}@Bs~^lxSoUg zW%`@HB%aRD6N02Y|G`AH9r>$Yf%SN{%r=6?&Xg48kdA8{tjjIPrw(_Px9815!SZ>} zuW9cu@5syN1#@ksZDo#;GzEYDGBK_!n%OqpHu#qqs?b{f@Q;PF6|?i*Ge`C8WZ-aeXc$ zqYD`@rJ!L2r;rNsdc8<^)^5Csr9{E0n9PV=$ur`!Q~@+a<6#MsTS$A7+vAG}mWq!M z0S3ZL9Cgu0l{`!X#vQ&*)C#G5LMl-^M25XH)`~QssV(ah{peOw2_`f#f^q!gCZFp~ zK7118eDX{_Mf3V#2`(LA3)U1&Z6ELd-^l*wB9J(mFcnF2A>@GZMnjDIB|V>UH5Q(k ztT&qR+8bPNc)w_v2|sr!$O}PJ=ObFI+^(i0y*-yhk~9Oss3BFXFXBJskqgYaqt_4I zuDe-eHraPwtRE~&E!Pi4gU>^|6$xKL4ED$r4~tw`92?w2QLeLm(U7cyytBn>f*Vj> zY5~`Bk1kdd9zI`?TF->OFQ(z38Iv)rpzP=stVWPy{Fa8sNVDubgsffE zm6Rw3fY!Rah(*p&66WU2vp=`bt>+7QJ6nfKw_hhi4?K}CLNN`jreBd zYDq};lus-r67CX$$A#AA4StN97Q|YI=Y$JL#jyT?;U5f-g%YN_m_@q}n4rf7W5gJ>l}G@KDi z(P*#BVO*Xh+RWV1qRk-oUjTb4n%iXJR!rPh(X>%ewhkBbjZm2>R~hl(rhsSYCQCZs zP--_)vA&;)B49KP$*U=O5t2Dk9mPl>5d9wG z1Vq$s6C#LfuSMk~6mqupnudy)< zYYpgYYmAUvKD^6rDqDyEl>Y<{5I$%65W!4@X5rp;7mjV`}UK+w%4pEfQNGDPX+? zObBwoTEe#9pd=S}L#F6YVnkEnARQWpg>+s;L;`Lmb))uB$JJ41y7X5ED96*Q8d0Hl zU>l68iKdG54vp|oBfLi20<1?QTsX1YZG}f*_T&qMu8Dq<>3K^c{DyMsplm%~XU@vu z0%|m$sBxZ!AKAL7aoYe*%jgEOgE;-9uAc_%sFzzLmD?zhRQ!w->u;hmC|4a#;f9*x z4z?xwE`GijAkKVo&fIlS(lroTi2S3)8~7q^7%x*lFwru`t~*Y))ai2S1cnm=dzDsO8ddve?rn z&-3*wgzV5+vV z$Ae@=q8cmM@jB5`wB3!GaBP%N&nnYkiQ6IOJyl6x$PG_H>t`ejeo*FW&uvTLPxzFXrlGot%;s_W_eg17t{%34`38`(#RHlnO=T zR}Pknga0RAAh<8Bm;Q97%F^Gs-$*)6$6jRS;gMfNK24uRr1+XpzlC_ZNEoK@RJ4M3 zj2r7SO4C{R8l)A?p~n?5J)reibJ&oCHC?7*VKyi+-cZhM_`Wa9AYTsR;gQ@$u{b|d zb+)_f=Frn>W1O}xa+`6KJayE1*h6&X$c#e|SVvmk;FV;UJM!J@vAzran~+$i0$k(t zkr`W43eW>~)ct(%#I>i2#tGMZCuwa-!H&;L8YI%pyTo3wLn$&!@S*1-9*AI(#fKlz z$C7y(r6LRX>AQ5h>?i(gftylc^X)_e#tANnC@zAhq)TP#> zYLUhL|MT4f@q5<-aocMGaWx6%4esO{_;MpBTJ@CcuGVdmE z;Q-R^GVcZzBdY*&n}0o3@`ZD>-873$6&RCki%Yy)W17X*A*YUL>L|^LaipO;trbrp zBXr{v zlQb=)X|X{qrUz9Db~<3zF!fMR{ke^rIeO`NExOYH8L@;hPl;R4>qdx9m(f&o5K)Lm z)6Eny&ijnJVfj#QF`BlRZKth)3r&%St_aZC0zDje;m2W=J+nOQhpIH(^|Y;sgGpqO zTFmoR70+mm<xtKi5>h6@?)O3#+D7{&{XX?c68rnmQt#CE;VxZ}G}z}Ach8B&Kw zZymUq^RV!AY1nzKZb)D!^Y$q;jOw5Sjcv^v+(ik{VUJ_0R@&>}*?hy4e_F?&UPMxm zB@#J&5yx-P>3mRQGUxntG8CFHb#66PpgocvmQUwMpjy#hw`ecTLG2$zR-du@JG{M@ zI`LY@ur-)8id83=zypGu`1?7=*jGfZ5@!ztzi}5@9-m#x*Pirb9i@E+rvb7<~ z^BK)x&dlnYRDYEASuJ49#4e74sv{oYcjuK^7IP~!fm96(%{HP;73=cfe!^wO&8g^>6v$R((FNl9rh9{xv- z*4}vF|BBuavRHH6$WbjY@fzgpj68n&sz3Y^U_+T*xwyeDYByoPRyqX+fgM2qEe2^c z^B}bvjCXiA$=-0&kHa83Xm}5GlAD`l!bvke_9l!O4H?;c>9<-L&ZC^GieI8rU|lSF z!LY>m@XOp_U%(gwTbys}G=I2Arzqg$0W6|$3{N)pQ^W=4%{jCYCx4Scak(d~7hGWR z=gCJ;l2JaVktpJ508jA62po@Ik4quco~&>u$xKU`f*tQ7TRST6iaz;RH?eaQJ6J_` z?&Gt_xVdqJfo&6QZ(Fz9+?Xk;NBDx+9sgHOSkQ5pS4F_KMx9JNena~R+UwJO`j5k> z4%5rAs8&V+qO~C1xEXtKlG;lXxR-_f$<>}Iv+*A+OaiSVI< zEG+ILb^yUwWR%6iD-h-A$0ws#X@i6-^+fLD*xw?Img|Iv+c*e*;x-%|s8z^dI#%P#yL1g`oO^n4!B1)lQ=jN9BNGtc~p|cUs2sd90i1bwB@JIivfZ0!*izg8P8D?#*Zt0kl+F{8lHbe)KkAs|ILW6CH2F6 zOEcm%d~;4w=pP=(H};}9sjR~3Mheu4Bj)^##HAk;aq}kO#m^2e1-;Sh0X~YLL1cFd zWs@{&kjBE|e;$LsgBD>X_r8xz8BTi)H|Q|4&PC$GGEq0ae;Rmn%=iFL=re-!^f%zq z;@gp0X_n1Rl-*#K{fsEPRF=KNEbB(u60>X{$Yj0C%(6A2>?~RKcVxPFy#(D#(UQn} z!N(zsxQ%%zBFw?NX%T?di=+EL+yU6{N9K!p|FHlMpph;Bi}YKgOM&~MG~Ax4L&C&O z<5Xp}j#6E2LH>byB{_qSPu_Sbc@#&BkBD%)$4;H+Hj42?3575Xlond(W+do3@;^^V ztml**f~bT44_t(?mcXq`3b%VO%MjBaeJ#OM8@v-|AAcIy6f{op^wn3Vk_z@2Ux3d~ zj#GD=NqEXaZL&~HnnLO5t0cWA4gKixBP*FL3*DHekVOm9)n`l*jT(TT+Njj_OVTfp z-enoE^J{~<}g5&oiBni^8*=S%uInfY7pqDZKQX`6~p9D<+P-i-Ku zT^9OsnnD(QGo4om_4#YbWNSty8SB4G7G0LHD3{{G_5Wd#X!&p8r?y-);=<)m$@0%; zEN}J?4NI5tbFo29jFM#iBfAItPn0#OluDk|TVecP#coTMj3oEsoI+E2V!#qNs{r6YUdOsr6&YPOZ%|>*t53F5?9{BhHn# zqY`jgIE!F{ALsK=!q6oOBQfpP;0HO7!nf()Usd{V4mIuDw>GcYZ$RyHSJ4ZG0Dh;1&}W zpD^6U-yyLA~uwF3pzXK_`|~<;~q)jq+Ti#Y1>{y!fo~)aSqxBwD;S{yzAqm-yX4rAm9z6-?E>9?$H=?~GqfIVx=mrOk`q zihjF!CiUC9=~*5zpiq+-Pa`vOk!N<^=wX$F+2usU`T+vjv6>l~vla(IBXidC>j(LD z1HW$M*ZcVOetvzBUq6KF2O zWD)q*w{T!IzJBW481!({LvUq}z|j*>_^CYHrDoINjv?=u%%lA5kFkmkWg8Yg`jg3r z@BQ7~AK5$jzV0dW=>-z(uBDqchWx?Mrdpq`X;ZC#>!!x~MqdNIOEEv_udNUIYr>nF zBjHVrp{+IZqjxOcRN-&%`$GQFWcZ54W`T5G7Vt|#z&6RoCe6aT`SNWnpM1RBVj3{y zaq_RRv1`az>B^_{l}NwZ#;%?v>Bz@3Z-qpk>f3G1PWh@n%703hdmTRhos+v#`kO62 z(NjA9gf+vqg4w3#u&K7S%yv}{yUKQrsONd)D}LAIuZj6OXJgl?_E~+x!5UZfT`WD5 zoT_{}f3w7!>ZAU!`XnFnv%f3?ZxsS~zJ&hD&Hg3|{ew?)61ghG8{k^t-s51b3GRBh z)o`=nx{&@^xEJ8g!NuW-Lk_+`mdi%XjSFfU8jxDeXG~wXIapoW+~iMAsc)(eQ;M2T zt842+zReAs%1rY&Wk{+H`-6=nL-vCq$#!q6Z>nwH7Sgt2e^KAugq+7B3#20w)t}smTH^DY zQ6gB6k1&#TP0-B``D-FUl{zYYeyx9Vx0U>^+EV`h0&^92G8^+@Iw`a|0M`I>KYGjdTkFwz>D zeYFS+c1iq95d~vjUnuNrs`UkHHL=mt@I6c=0!h7f;R1TMRpOV1z8YW6thSVzNg_o; zBu~))P^7^Zl+17667>5Qq+yz1|F#V^g@LV94nkN7N)tNzrT#Hr7+nTEY_5$o_@Vb& zbFkJQ#NS-R&!qEEXb>!#- z{aaxUG`Yy5VJhr^Io?dJvov2AIL@3Cpn}TKP}Di6UJWWGMSWucW7rf1U!O$}P(DZN-gB~ zH-@Mc_&BabZ8hH(tVmJ&WKDCz_=RZ_ew64M>YE-GhKsA7JVrwD=AaK0eqRtaET{oJ z%CBi|Y(ytgDgT`UnVLF(&BG+tmgb;lw$iGnTQH-j95Lz$1F%bPpA!xJ;|#O{@*U#S zJcbL@$FT1uu|PZ*rNW zU5r`Eq!H#p!3+LM`m_}3>yakLvRzWGs@+T{#$-c)i)eH;rJ1de1Uz50OckPXlC?_W zbe6Pgn&UV}UX7^7lBVocQh8~^D2?+=#7(SByC``}`7AYC>N9ODI7$w(?^U(3{oJw_MzUmK!oz&_Sc$4u>Lu1NijtViWst!)VI}Oc6@)hif z{x^9ft6X|M$uy(zNW)82=y_Ru&%93hqIWL+>iwYn9{Qyhc}Gx2B_3^K#HHUeK5^MC z@#r2SE^M6YORvPEZ;ZIO#wjl6Bp${Xap@nYxC|*gUKyL6=o+WEI3ymtIK^eR#G_}7xO9$FT>2y)17pM`HcoLFmUt9?dyMvT z!x+oG%?+93racrBMW=+gJ3;LZBh{!G&#{nEKKwap;nM zPYNyr6I5SbkT~>9e;@@H#{|`vA%(-M6LY*|ny(5a9?mQ-Nf5au9-b^NNw|0=9)T<_ zNw~B~JUXS{m4eSiUDx$W9Qvf+pMuLoUDpjr9EPNCKP%=XO|Dy7Cg?n^NgQ0#*QM`~ zxb#d=T)Yy8fb?6W&m=D135v^Zi9@%-Lt16`gURc>UYYOyYg-HbcW1o+`eb|0rQp(& zf{P*Z^$CK@uxzj6JIVUt-jp17%6!iR!KFmDS5LuZypNNMwa+W@h)TaR1t0eV$$E1! ze&Z#-N8%8ZeqRbM6Lnp8PU0{mefxKl^}{>Z;0PU-8?4=7wx?w2Us#=9SoICM(CSNbuDi#9=V8Sni**7ni=_s!Q6;EgJZ zk0qHW5uJQT0jDyfvH<1FfBma`5w+y2*E$dZ=PzC%V)EzjZ;&z|-+Zrwd>w}3E20*@ z5&jCrYW^|{S)BM=lEr%;n*7`7T~&3#=55s$I!m&ks}CW&y3X$l0GfyJiXW4jzi(nx zHi=@vN4?oW$tz)#$ria49@YX0T+|yLL;>Qj=rZ(IYBn#JMlN4Cxjo79RrA-k@b;w2 zXDv8dPEpV*XXo~#@2yFtH`CJ4SNKcmUF^PaqLgR{c#XeXxYXdM^eCxUd4H*H zO)9|gNGW52Lj9!jsQ(d1y*xzaD4=LfI@Kpq9*`*xNjWL?3LRCz(Jaipn4-_IF;THJ+4L1}7uj?86 zGMs_%5d6j)7_)<>6aH&(=O`Wi8#gjGjIjGA#y$x*fN&4|pTW67&)yCGa885+@K55~ z2UQ5i@LK--*^HGStiitrue)zSc<3(}yI~$~+zL1iVGsNra2|xa;OE^191zyvZ^M^w zst|VIn>c@UH`;;l0Q^%rutk`y0nG=16Q#qy{~q)y>TQAl>2k=9^da~IYvChZTL*sY zp(jZ1f&X_N;DIpSo6LR%7eyHF3TAWRS`fzjaoK%vZ3uheKMmK3Fy4d8&cN+P81IT@ zLvURP<2|5k7F;*Nc*i5Ffa^gR@4;kG!Sy1HcO0@W!o?8Ad;QpXxITp027H$Xt{-8% zzl}A(okJM!EMw2W4G=y2U&0wgUx|Kz8$uZWOJ*zKh7rbpaoKh_hVOmg`5Sf`&W2CfcaJd?@J!vzp#RnQH% zD8hI~hE>3|AdKhE*wb)r2;*5R_9|Q_!gyYUU3)+9CwllkxGse8TmU-=*Nrglg+sZ~ zCWP_-6}AYj7h(MOlzj>=hA{qb$!^#L{3#v&eIG(Q5YGOx2xJlX|A#>IqvBuSPfP!p z^uHzjA4z{$`d5EU&@YgFx%A)a7W8+Dda0f*RHr3wt^1wQp3S+q%eV#~yDDu8)o%#X zC(qbxIY>*yE1Neq)z>uF`einIBMiH#eHStlv27K*y#oR~zp}BXyKY#LaUnt~n+>8$v zfY1e-(6Tf+14>93`}P~(EF3+LhsbxrkK{2?{MMQvcE zY0JaGNGQCfX-l)@HtQnXif8e1D9OJQSs#8>#*q^AKN3ML{#sA4ek;DP_>ezjW?HjX zMe&JA`l??!KKX|4D$;euTwiES)7IvP{lU3fq`tJK2KHjb+%3L_kbkaryUAzy?U`#? zetVKk%WpRuhvLg`SGXZ!_Chw0MIeg+N1%lt>9B$0#&PY$@CkOZk>~A~Jd7$q=*TL?C0|$E##SRT0>hC_+?Ko^4 z9y;ti!j9UHx{nqf(~eaga~&@^9yqR_@SO0TtUC$DU9?t#>uTw0>)PGbxo7vDu01__ zy7%_%?cLkAH@2^DU;n;=edqQM>^Jrg?;koae87IlaY*ZSc9$G>ANCxnI^sPNI8t{s za5Q?f?P$xfwqu>gb|33Hw)=S3@$Tb2$9s?Woaj9fJJENd|3u%({*wbIhfWTkWIdRt zQ13vO(KXy<-{aUbyoc>|?A7)<_d50!?sM*U?JwD{?{^>24|ooE4^$oW9;`bUJs3C? zJ=Aij?NH~Tw(idEuEX7jdk)7A_a2EI={s`nNdM7uM~$P#v7uw^I6GlKQFx;8q;}Fp z?TleR4$sw9(&g^bfu+93v!`m07kGO2*6j`KjRMo?zLtHR`#SgU-ru#q8@P5K=sC~} zWMc>W4)z~B2XxOJ8aQMe8Un&Y-NW7X!}cSNBZWt_BhDk*QRh+D(GnnyYO=p90+)@z F{{taEVBr7& From 389b004cd05404e35df70543ddd90b5ce1ad817a Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 1 Feb 2023 11:28:01 -0800 Subject: [PATCH 004/102] update mimalloc-redirect for win11; potential fix for issue #657, attempt 3 --- bin/mimalloc-redirect.dll | Bin 76288 -> 68096 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index b3ccf78c906560e8c753d83102f88fed62f3c02d..a3a3591ff9b07edada16ad83ca1391963a126b2d 100644 GIT binary patch literal 68096 zcmeHw3v^u7b@mw{;qgOAU<@LJ8v-dvxd}p719{0|j4fpi0OBiE?2;<|74IrkEpMOvOHH_lR z-uv`r>~E+4{oDap)!)ypZVbn?NVIi(G}x@w2U}WN2tHXt0Qqja~krU`->^Gq9OSzbrn1@_!4awdZKAe9>sto7njW2|mTv>_M|GPYp`GTISlAv7XbK7N?T5<$k+Tmxl$ zP}q(@6<9ug#!^cng0Px^L|d5~@y{(pNcu@$SkQHnF`_M8xBt#T7mY=!2^>3MC^`tS z<>M!L(NI$>2zQe_V9MoPI)UyM3)>%M@)i1wTsoJrkvFE}>#H~5YZ?rHWZt#-diFYe z4ZaIskJ9DO=HY7uT{!Y0>|mh16#`+R(D zCEms8j*(81w;6pg^4pv6^%Rw^q$*2D=&#UyBX1G!5PD?fnFaW|frRdQ558W39dVt1 zPrvlNRUWs#x7t%$wYSn!R)tGV?>JMgcUO9J9a&mc7rykaYL8#UyoV%(-9VQ<)^-mmp_QCsouYx7R1aNJ>G4v2rN_n7vOwY?n(Qk% zRp==>RpoK5em-7oUMze*U6}r>6)Ci>?A+T%E#&R%gt;jA%k-BEJ!AptnKmGO`i+I2 zSeXJ_(=Qp+yPhXyyq)g!tLIjCrW%1_(MUTPP3}5Hud0fkV|!BZTlHOQ|NB808rWhV zfK{b`MbPaWDZgK(=eXv+SQ&-ewRk^Aul zZRe4;ulmB<@MR*E@`1sVo+Yn7NLN&W$M^z- zW(h!1AbrTDMjNl1>g*~$xC8Ysq^>0`d{lQtzY(d}4Zyw~r5yWfNPeS1%Nru`No)0` zzfC^Gi%w|sIDRlG1$ZFliizl8TVRp}Z>W9LeWDrm6n6aE2axJ6 zdi`yJp?DNmWW~AKVX;c|lDmWtqBVp}sYvqPMXl{Klmyz=S~_RRbQ3dsGbxCY%Ko2;W4m z)hj8ksoZLBIvo=eGs z#57)zo~Ibl6pg1PNPeO1N$!u&qL>{Yp#mI49ZA$hA64@-4H$oXk*F0~1%+0kcBl-0 zXKWC8KvP>iDEiT_v=TySVuaxIXG}TQm~!|e%HcjEei`*lA{Tu=U87wZR$QOos%XXFJ~w;~xJQy=M< zIRTcKwjwohfG)Ys@MLCx35vfMm{qP^6Wlqm*)*0ILJ5-(F0=H~pfKljfaXUcdxJH{$@ zUM*AiJX|bNDGj32@SqTm6s^f75h(0!;`+SgRMF*cGOTbC1!-TN#ri-Li}i(QE*9%A z{t{yd&DVJI^AFF7_LtNw zqP$&e|CH7#?U<^HMqZ{IAPB#{kF%;JA^B52v6M@=%LyJIT30mkDehWu)iE|dUP2~@ z^$#5XNPH^ND#7$prDwTD+6G8lTvdeewTx1Jm;|qCAy!->r6q2nNy7&fmz}rr|A57d zl(RUoiYnl1_45MYlNlL4E@Qcb>O4T)kdd)$=N(d`8y~I|DZ{7lC*u#Ccl+MrgP$Yz zpov&h2C<6smiTB!Vl1&z@7nF5(;U4cgfbs6qRoDV{n zO;R`^q!J~24v{n^VG^lz5~N6C9u|CE4#C4V(O4wrrT&(FbSbAL_K zgQ#fN83Q8#rWZ%V;GoA4yGr)FinE$+6$6DXz56qsA?n!f;>)S75B$_zLR%IU#LQR9 zun~|6d*QnZq)8o0=yOh@p8T_p#z_i=XMv87O=%@EE;`{1QtS| zjshA$i%1S1K#PzdyIv#~Kt8(U1wK_3Iq12hD^?XHd;Sl~?fOPY{JM<3D@`FQm5`NI zeXqBuR!zL&bVjvuQ#C&4LA7$B+8R?TKguVK)?}$Rw{W6rMI`{{b?kk_{$jgW_yBFD18=z#khKB^ym8 z8%Z`gyHO|!bCTnuNmbQ^sxXz!Sq>o>{i{jV_ej>uyFKkhjt|XhNuKDg_9XT0eV%q) z8Q&r@Ir~;&no3q-0;LkY>oa0{?k@gy7=*itu)C;)ssR0dBIzzFm6yHZ@)~*BCobp7 z%YJdWLS7Dt%Qf;cMVHWY5HWrDL3=%X5MS_ei7_JUp7TT-V^0yH#t5J{}j(XcTj#^*eRYs~&}ebZPC$KyWI^5!L!7Ba*_fD{1%y&xy_intf(;}GEif!`BjugMC znH_nOae8{)r)fL|ypYxedyBLPG2J|(yvw;|s#E3laiqtkleypM}XCieA zE#k12$EtoEPA+gRiJdr|OZL2mmdcT)!6VNpYN```KGf9nnJlbKlm*US-xpZgMujYqLp5@b2j-bKU@vX zz~^fOHRq?dO=Uhl_2Gz|oCtWS-jf^xmco!v!R z`r4AOpeftNd5>vCg#nfFk}xkXB0=RPUs-?Y>gONgx7fTD16U)|^zcTAlo&L|{+(Ou z2Wcci$x<$bcPMUTU@*#SD4(ShKQYtO(?)iv1FiFfD9az%FZ7pFbmC`%zO52XvffMD zb*2yNOPcfs5Chwiuk7}ukc7?ADd3HO%3M6sEJ$Q8!K7&}rzbziR3pao%jq0P*t;78 zSK7Or_E9G{d8`s5!sI^-dKy%GVNMA9SG;Q8xq(FVBaYgLLZjhR)xFQt1F=+9>D@R+K-EN3L*o{W2v8&HjMK}oo|ACl#B;wD9)a1e7YSVxeS7a*_-lSc-tN>o z&NP~iWxQl%SUhZ8BQF@l#2~&zP})Au5AP z)zKbVy_8M^M&Ao}M)&t1nY$Q1JcY&Q7@h$l4q)KHKmqoEn?&E2!pOmaO;@EWVjDVM zqW5-`vp5_trINE){G&YC<%cF&<4mJ`O(*IbM3QTGqg&i>(Zu~05YWjHDAV8Jf^dB$ zU3=4?;-}4x7q>Ijd|pnb??wADzHlFs!#_Ufj&93r^AMSJkyJ9@4Jx(GZzKywUq&c~ z@%oLDrE0`_%^B)X%o)ljE!|JVUc*?TranvzUNQB3ZeLDi%4vQ=E$4=l#lP=5!`J^% zvX3`gZYH*)YBH)`%A}4Forjd4Ql^`iH&KX08kn>rjAdfQF#@3&P8LL7DLeKEH9*KMOa=m(zIorf_*G&CjI0UH-=T^z`2p zmpx~>O*=uKMru9$A-ZyW&PVoIM_SRyG;++{#bftkeeC@wp|MU3+~V}{IXBoCUIV7o z{Q>UDC{;r#ZuefY+VYa!elAndH3q=~;R@ zPZ9y~a+`=y3SR`O`D{*&STZHe>ES`WTi#N^VNJW_ufYr{_A^5?Sd+#Jn6^k;qnDm{qdPUw5la~Jl)dAOZp7$x9!*6D5rt?pjYBcc2aV6- zK~eCnMAKHXowT*`p(*mv8VNd^p@$16f1F79Gt0x0s0wHDg|4+}2uUngi+R4P;sqV4 z0(w{wk5gLbo{>W!ANeA_6bE`$iNYx`ODs|aLR43wt0Pr}CKS?LqC!qACeQSibd935 z2!X$5iJMh{MW3|zCT?M&CmUVf ztGvQ@WgzF7G?wB;5R<+%{Wm1#tcpk*p15_;`Ksp%2&7SmXAY$=!l(FTA_glPC` z)%HvtjbA=B)JV5NV>Q+!hc%sXTrw!11k}j?aazr2!?6QD_K0H#Pvc!aR3R=pQq%N~ z;W6@boT-kRgK;Mooq|a|s55OOB9=O58#I>jYfq~2#1{mZ?8-!oI+EB%NFX|raP$S~ z>t#vGPwM!px+*Zo4T;Py-Dhn3E*wBd-h;Tj`15H7uwN-BR$i-L??}37dZAmtR0n00 zv$mprNC68f`&u!v&N(91QU2bP>{ZfV~l-W;&L(} zpbq$rAED*x>nr%$pPme*yuXH7abn&gWMMr)D~NoDqtGF%ep;_;*w6D*K%nbK@@c-g zfGodp{5yOhKSxdE1Gqb7`~sQq2AISo)_~Scr-+yXAkD(w3`*`X&IhtA{Tj(cBC-o3 z%0#{a@enb~ZU6Pr}K0wDMf!D{!AAY!mCt)D%CEcIGpu#Vt$jGRP8-C62V)#&x z@3NA8U*HAY42;WYJo+;xhg#@hXyI?h$Ccj}LU4Fw=5VL)N6vgj!~MPgWjeRX%skxT zLm=H`+Iyu-(PAY8rNEt5IXK)9Iin*-A%WC%?@WsJb{KA{qG%7Twz!I&cs@Z15Gqcn z$*ts?ggB2CHJO^@p)lk?^EcsjMi!!as+YO}8;OsobQz0(mUB^@8w>d-!!E00Z$?p#eaJz#?oD{ z0N7Eq5hVe@v-0U8-eA6T)!oKPJm!ULejZq~6POCiERx+mEgPLr0%WD4=ysqp0-lu$ z1X!m4L6PCK#b$8AGQs%E^v&`_ID`Mvyft(fMH4D98x0R88vSnpG`j9v+>c(=T}~e7 z`#YGOsEY|2B$8%0s_m8 z|MyMa5tB?7G+$t?&lgOvwaluB7Wuf%Msv_fk^)!j9mRAUok25-xy;vy2_s$;H|6l+g;```nYr{Maeh!! z+x>+P&7;Mv9|v{;;qf|ly8M{bXf*TfpFjPNoUo+(2fQloC*fpE_MC|D7VO>T=7b!>QWaUW8eh>c-}sem_*Q`_qaI(-oGvSBl!Y z{BXcHd)K?B(@H6;wS*0sWp*xMQ^T>5ZG`~pAK6xDLY0bEi++$w%Jy{yFiwge+klpD ze+e!BA}PA-MY7)uw#WP5bZhpZ*>MKo$~z?b-)teY>x{;4h~DA;4eMII-sMBnF9w`? z?AcGeeu5J))1dAyex?f0Eg}KkbH0W&N%R>@kwK3RA@gZ7^UW8{TkG7x2v-B7X&mC| zb#hK#y5t33>TL2s^o$;Va885m;b&Un3A){XhGOG4nDN9O>_Jk<2Zin-g+7m}h>}=H ztnEQP{&dbF)8RG>rF~duP^!_`&)HKrKf^9dh@DMhKZpD;xs1;u!tKBwL|AGU@I`dZ z|BSg<_wVHcZigY3 zzCflo(hdni6%1LrZ}u!URO#C&%atf4ZZHY2?-QGAy4K_Xrcn%KHQpMf zp&%QM24yyo7A+S+%l}E+r#^?P_mtv96HwC6?_iS65ApO7@W_~@v`(I00TD`{d*1063LqAhewpP zx9Ho*f~q)W*-C2>9}48sCUcK3jC`@Qp$U=JHE|!UX)Kw1L4y_s==|e{@Lr;cWaQl> z@x;h(M0})DHNv;@6_i)5pu8l}YpbRp{e7CIy{v7utvIW~`^^f)kmmLxe~INyKdbs6 zGrlKXNah>RXUWgG{>-^1wx1NDg*ZoBCkF}Li0D`|$l~}(2Ol+*(RJk@tmOb%%ZHv5 z0|MiRm&_V8-a}a!Kfd&11Q&$S@a!V$`%#2F!Ui%w%(pZn&ZZ?Pzw9pFj(wvs4K!r3 z3a4u+(I{@&6o*L4`$3U5Z@QoTEdDG|@#2Z=0X~XgLF9KHdR8<^qXua#E`E15f*+S- z5kE)2m70LlJL53uFtaW|=4P3x8-8NO^SeA_(0BpI5(JNy-ih2bCT}>yyTarxl)Sf4 z4THv?nY@1RmYKYZGrZTDyn|om^%n4qLE}RvFF|*UXi4JzkTc4q(|pzg7BNh=`eq)&up6s&l;gVM zIp|XJWEo4`FVbBBH+7!h*#8XhLJHGBMVW;T=ZMr=qU$bxk&yUXPRSuimh$&fh&0v` zcoaYoW0oPIw_1d1BRAtj>cYULpz-qK3**^Y!a%-^nBR%9?~b$ZluVtHsXYf%F8Uft zAIL#Jas0?iu9r;L=3uf|L9Y6YnW9m{i0zHCw_lQenc|Kd2gx@2Zb`o;-rl_UeqAy>lY_~EZ@%*tVLpE;LU^(ehs)wzpg~VsifHB>$<-&)3cl=grb&Ji4E=dcvPpV5a;k@0IqZF^7~cfd5oQ1w^luHJG#WJQX0 zShoHVnW8HGooSyave%C#J5@*L5u$#6rtdReL?dtuRovdqz-5svk{Nki&KHHF%P?f> z#{v~Ahz!1VN+HkI9%8cNJ3Tk2;IE{+`&@{1(}Rksm0zP{{cJD6?Tc@c@@o3jM6L`B z=fHntGX5M|UW0X{P0LJunf7JtYX_+zPo?LeZ#WMtS0abLpPP*ST=iX<$KKCD-)r)) z=F|6=la0?D_9txTN%}#4w(U>_`Pyf(^y!n~!`qjmQhR^?Ja_ftd6%60SijnZyq-r0 zM@oF1t3E%@BPu_C4t@SDkC3zL11C+p5?W5v>1_?Ql&@NK>Mg(V4Iz2r2}%yAMF0LM zcQ4uKC-~P~?X?p9{+HX%kGL!K8{fkVB`gz$-}p2#(l?}WIIwowdYlNw%eeKK4u5>k z8#rq_>y*Or&UP}GC~=H;X}>Fddm7tQ>fXD}+U0WlC-|K^2t@t=mk#wN*1tp6pK~2& zF}bW{W_(+f;zvXQ9NWJ{dk&}+Fuo2AI3Ojvzl2QKBc72d#rCxs$r^ePg{X=_CH5m? zLwUQPbAt{~+7QKuC54yR_@i~Pc|5rg2NR&EAP#^*nAuZxY(Kgqwp#J`{9-(Tk6U*X?R@$YB&_p|supXwZ1 z2LT5G2LT5G2LT5G2LT5G2LT5G2LT5G2LT5G2LT6xTnOL@X|j*{Z=3=UZAYlwg%B1T zstk95*_8;#QFdIG2@2!b^D(m9Qk11%*8#I0G3-?wwZueE`Ev@`9M@XrnpMDNxi&D@RQp|ixr*XQ}x@gRlbsA=TrIi*iLt~i(O6aQ}R?yI7}r!pPou?u6V1&o9d(fu*Rex z%AJpcz{NrU&v($P3D{Bm>V$4z(VRsgiR*m`y$IKr;{SmVu0i-Pf)^o${8tfvj4*~U zb1#@6p#c9-SjZ;Kol6>;nvh$|XVhT4HCo%y+7imn3Acpfl%wX`+JnR%g> zJXy8zP_&tJa6S$Kf0zjPlA>J`M%OAW9u3xqR%v<5L(yn!bdvH1g3*?6%XUSP8@~J@za0)nNYs*?sc0st*=}s%a^UL-gMWx)gN8A zdG$v8+>e>`>+Z-Uj`>#KR#CNi-Re6wuHJYX9!fXqA@-Jyo0Zs1eqhu3jhoh2ufJzq zZPmt#%G!;aD%WjdEN{Jmit1bS+VvX(8#h&No`mkUjhk+@wAL=~w$N5hTc$1dEnmS~ zx-}7QYFMSs$9rA*=YeobL+b-EtquE&aBB-n9!xBefe2K8awloIUfc?fXc&LOO5QU= zw>TE6PefJjg#5(~p{s#r)1f+)Pbr4^1_^EM9vErc zg5f5xJ`j#KLVPqH#>}xp3q~SMVPx|PLosdfVlB9}6{V;f7->zd!3Ly-IA)b+swf2W z`eN~5OG7Z)poxv1hX2D=ETnbYvL(yv-^VG%R}T7G{A+Hlg=;RUl!%c&(a`opQ!pxp zFWnXmg_h(HYLWlDnpFQs+Y$%H(ntttr7um~=%4U|!8p1O_So8xXbQppwbp1uD7tC_ zT}ZGdS{9qElr^6Znnhq-q;19CdX?r&&WF#l)~jUYieab(JS;61YzyJfjCZuQG>|=_ z50c^LM6>pZL~}%og&%}yj6dxr`erP_*A&FJJrvg-h=$`KErQiVI7Y)&+twP@f(;GP zP%Ng!uyhG&-e6ptAK|7^+f3qwHiDB67|4&HA>l>Tga?yIJ{V?`mA3?~dT?tc0KKSY zwKlne7X72G7zA5`_4nl;Gjkck3ag$k#A*p;%e4g6ELYdOje^+P-r7(yf)OO8K#sUJ z<3xQlPBkb)$ut94wl<-Q=FN}IogbS=gN85csQXMWk>#DKcOubH8_?C{B9i(>_#rvo zh24?k6{xS$>hTZO=ri~a)eoZcd8fcXm|D!!s7s+tgep>wkuJUS!^#IJ`fzguKF3rZ z|CWs!xk(UnhDZLXwqk=CWMA`RQe8Dp0Kzie2>#65GQ70lHms0Q`wUI27DG6C6#6O`#2?MWLCVgENy~Y1c z%ojpT{?={VVxf4gm~OfLB54u?@oIiceUd&J;v-({4Ta-U?cfYq^=9)}T9HexaDx&a zlCR{NQmhoY>yfV1vi+0|tZmpzC-kV7@Rcp4Jos193Q54rZF!teiq4W_$rGKfN@YFV z5JaBx&x(TABYigI*{)5jO#d%iAoa7<%$p_nl_e3alpy8^v-z_9F08_37OX7nXlaDvEO++pGL+Noo46 zLOs@U)gMZStnp{-pGjy;Jj_v%Wxk?+ZTuEk&sMHr&-A}pPqxWr^=lrQ33%k7Qx&%V zoVC-L*M1oj?|FUh7X0VM;}EkxiTA)1agW^P;64>I8kBgSo+9qO4(?Ndof#7Eu_@v{ zpKT#&*^A!?t{QLPr#_2VQw{MEL`yJe;gu7qjT{A`8dmY@TgnOgJyM2nd zvyV9InW+HZBk}H^BJM+uy-x*pHYo8Po+9o($KI!e`EfVoNSwzM-oMP=pE~=di-UWq#95PZxr|da0)OqE?0&d7TKw@z4M?2pWSo@o zsKh?XpqgnY7_PHE9mlxgyXb z@$Qpx%7*{&$Fu#-xyp6u&Vs;S3S3KNTqa}1>P`s6}K8f>y4ey?Nob{u^=wif!67SPC+}Xz*+%E>+O6I7-`wQ7PryP4% z5M7kGRN}4KaQ8d*eo^pNLVXf%zl;Mi?w7dNZOI-7iqDDP83K(G=cJ5#WE_#WYYy)3 z4B$N6eG=yZ84t>MRN~(2;GPG$ca}XYaUPSgDeYdzznEBDghc5V1-@Px>oWFB-0Pfu z(?vj?u||n=yNo+!tSWQPffW)LHr_9B?w9ePjGgny3yZT|+OWiVOvYt@CHzkNy)t(E zj2*HIa$PQQu92}4IwWyd*G_z45lBkB_se)d#-$$@_TKB*`-KHuN*k6qv%i+>8X4C~ z+*1zj7X^1m=>HA^^!`=z^|E-Qx6;Cr&9jJ3zNVK`nU|7*%H^9~Rk^sdl&km05&@Sl zUIA^&=db3KI-uNqFRy&Pv63s_NG}9ay$Sx8Lbd#5#FCxlTe8J_ds{*e(0@BLMq3}K zwa{6z1zk9X;@ZYgFal_{Fof zt(SOzzF^Aelfg^jt6t;H4N4T}fA6!jF36i;XX@SCN*;+a2Tw+>>K>j$_?tB9-OZNw zMx%IwoyvD@^J)}W3L4PcvqU|%+`M$Mx5d9iSl-tz^;d(~Ql5o3+QL;)e-MFsR2P`- zRU}zB@xiU6PV(MuIoK5?$Iq5W71{IE`=2vd3kliyE2)xS^%QT6kRY))2|2%b2RP5O zU8`!@rg+n4ybN83(=c(c=Gp)61ZB73_5OBxGcO1Irced_cWE%5;S%kDtO;dye$#5? zs6xL zFJC)R7JF_0xBHjNXbYB(9k#0x)@eSw~qIVk2L-(kiCr1i!}bfj=hc0hcvsJu|){|NaKIL z*mi^gr14*CtOp^5H2#N*y?`)?H2$N9y@@b{H2x=v%|keiH2y<|eFR|`Y5cDV+lOEv zjsMzU-$fWj8vp;mX4ODfr175w>`nx>2lR+PiQq=M7x4f>Dbjea75gDV8Pa&y5_|VY z!G|=Sk7PjvFVg*p{{z8?G~Nfq$`Hzt#=EfCEeL+3@qQ+@6G2BBzfEJO5CTZ!cRTD1 zLJiXR?G#&tP=_>rSHiX+G$M^>?%4r^2-5i71N%Ng5@|dW&Sq@^&Pd~Ve0C>7C(`&$ z1lxzugEXEOXJ19wk2Ib!WN#w$BF*kaT?l*gkjC$)*{cYrkshvPY*ig%qyvbbMldKJ@!LV@i?s7`5O5H&@Yj3m5eW(M!6*or9{0{x{d0zTw6JQU)r;^5O>Yj zw}j&rEwS+Ccr@Iyo&8e*@-o-et#`GA>suQ_vY5S9gz~Js>edW7d^)RYh~6HGS2i{2 z!Ip-m5PQjWM?4U1jfd*vcZZ|#M6hW?sJS(|gRzxVKWOiW-yMp^@Ql$NiRP_HG1f{_ z*0m&>Ls8u7uWAi8gc>%qHYA!tF*cVpxHA!o?pW8-7LK;IG>2N^cL$^4;8xUjKXKlM zr)<`>#G^ZIYmHVkG(e+Ou_Xu>xdZ={q5RdLh;%D2sg8 z#U9EG(QJ0-%0ytz&B^8_txb%%wF{Ol@h#9oEg0o+0BaZARejsy@&#Hfj^P_@YHbOv zU9ck*TX6GD#WU6fW3f>4)}|dA1hmA~E=WXMR>kTYL(Re1;^uIDv^CbcExx$EwRu%A z*1V)`*#fOO*b?3rim4GUYJ(_ETN95aV)6AY+ghcxd1sNfVjkxMXDn2oh=$`kTA(d8<*Zqnzm_#ivvgXs)NCBs*DO`I;lk_%C*dIAAixpm!QKpsz{#4E zJtvXqWenSRHrPAV>ptK=s2^-UIPhrd(XmH;hszH~4);CQ|CsTZ*5~c3>)U^%_sHoZ zrN_#S1&(zd?>RnroSkr=@SjMYY(F_bviczx(P-~jukS$P!N|eBgTs#+k7|c%4%ZzX zdhGOLdS9TgrmwLtb!70!(2?OI<;VQT^kX&0`i~DBPaPjR?mgi zSmUwCvE;G#W2cV~A0Iu=o^U@AI8k$=@kH`O`-#DmLnnt%j-DJl$%Yui(F5}z&<_k8 zOdT9Nc>194Q28PMp}?WO!~KT`4i6sI9``=(d%XN{|Kt1ndizR`mL1iO`i^!U>p8an zSl=;{T=oP>?K#0EV)H6=cK0?Oh#Y7?FnrKBIC^mGAbYgtP~D-%Ly<$tLqms8A09qz z93DNaKOT6z?(tOLVBb*R>AvB<@}vHv`q7%B{l^B5rH&0A8#?BF!uLe^6aFXkCwfox poftU5MscPyNiSlr)ggi_M^ycBlg(R35+B`u9iz=;6OcFz&DVAa@v5Ft69*Nqbl;E58nXIfvpYAN=+d z_Ky>OIH%iG_QN^rYu!H0<89pHb=GUu&W46YzqZMxc>@iayFn{mvremTtZ^;Q&7C-3 zivGKYnqxox)tlWaywv)e=YPiYzv+G*@h5lx*YmwRe)IXC@OWysh|lx2c>38~ZPK_Yb671X8%&(sWI=4<$!1QW^f<;Ys}!Pnp=C!F{lQ94jOuP+v)MBI8)=Rh%F$mMqo_zQ~tNY-s{{*d*j0 z9D-@F-__*blvgRyg*4+YlExDY0Uk?b#0{G&1ESnREF`kjpZ_dJjYF&fZ1 zUaViN5VKgYu`&~Qx}>~zorNdB7g_8Pj0fj{1Wedm#20w6ezB4~x8Q3fqeY*&ZXbZ$ zgpZZ@ygn}tILDYQoCfmhbbPM5Mle=VAAvpRn?H!}7L@)?e$LRRZ!^&Dn==WQ*B9XO zG2qbmjrF)30TO-o*|=<@?3-8P^4gWS{Df$C5zRxF61r%xfxbx-aVe$jk4fHQN-ek+ zm$xS4awHd*LaJMRIWBL)^7~#f*p$()5jWxU^w@^?i97E?@j0Ex)NHTwZH?GWnUxb=k0j9i@*`}6AAiFpc8;^R0R4?qZ z4vmCpn4^$56v6WjYMsCZ5E6NeKua^V*=DOy$$=hwn;-^vR4{)5SixRy&O1J#l}3!& z2O?60StQM|ZZ0?Ma*RYZu3#Gdpuu@qiGu}ZBdTSA38!tPP>#jMBQbza$W|KLEXu}n ztYRCG^RTUJ(7q$@IBVrSa|pH2Ut29FnkXfRwOmu`qs^0d+{W^bmz!3+m=9-h$pg&$(;Bz@;CbF1X;iUEXf^>jkA0bq`DSursN7-C>amAjf-1GG=Gae1 z)U=#nvKfxrxY%cv5vR6C1E^X#$yy@*TkJbm{o78M8%E!EBW&vo)=4lSV0L8v*dF?U zJv8G*aL}+%w%9{C$B^L|<{U&S_6=0}DzB8g8+mTX7lB!6EBTAoLH>f4Xfb7jC!iZ< z#dLe{L@Q5Clqt*Bd3h&Wh!Z`*$b~5x{S$ig&7$c4LDY92E>L*@8w75uQr6bHieFV` z{>N{TPFyX<;JP|2sv(>~Viu6a3#=z7>zXp+mYKw^ydoqnWLItwN z+$K`vS{h;~11MWMsO4ogi|<1ihC68&(~M!b0zKJ{l4udv9-MrlSzumW6Xp@JiXHlGakoz@S=B^3qDJb-JnV)0mgASkH)Oe)7DzR5_Rv z)*IUCaicVfboH^Ni<~~9#%+$BM<8V3U z=4iQKG-JgyGyhd`D@68Sj`sYH+XD@N8SCGPk>ecKPfkD>wOqw%+p<`#NRUjEAZZcG zB9As}h&v%ibh+CNxwBum@$@Y-gdo;kGVsk>PHw)zSwZmPgCW|A%3#PnX z_YrC5DtD7ymvZ{f@q&o;GY8fGDjz%V|`OD zPRg(0h)&riXwc3=%AT_2yHDA!K>PuiE=j-EKS3n4z(+7f=2qJk7op`Y?&wI)qChc( z(#{*b98nMLdfg(bJE7Q2_{UccQY^k4SiN}h-o*>}UjEf*o}BnC+cc|f62r_iAF zd!#+us>|^Ou2cKIV8m>1Yv};^iW0J3fPxy_qZXYHPu#amujld46pA$Yb44PM*Yc%= zx|yUE3T{u}gV^PL@m@@aAN(J((H#~Rm`tNj!%rY9@zZ1lKQ;wTI7{Kne-m-pt8tee zD(Cm~8o9O!xR6!!f?60|LK=S#d_sU*AyyNpQym`Mn0XvQ&wqpuXi$b=d zKb{3OmIpysz%7oaR$6U0pnaG^YWp{Qo0PPNdZOP%$6^!sl=uVIxW3Q7gN}xC{`5O4 z&VAdjc60lhQQ`mbHdXg8?{#8mf11XHzP%AP^M^kT48Vmxi6K`0YzUv7*eC8klTL#A zVbuK24!#&k7CaF}WXfwh^5-y>#xViQH0vU$fj zxwS<@4jGEzC08bjd6~p~W)x!l#rio+-pSG&3rul_RP75S^AjKK|rzb*;|{~W38RFd$G{~8Nr z@61kC_T(qgRL*FW{a`ZI!3)Kzw*=NGaLswEKUufsq?;QsTr`lG?cm{iAoJF~O9Jg& zbTo0`kCE)>A^Q^Q`?I?t^Gnf7p>NLSHdB(mb4lL`msZ~ok-oqB-BHd^68dBGBJFRq$3xe3_r$CU&3K3jQ)}s+gg?)T^0&cNm+H(bi(+B{@4q&MtCa61fVI`?|<&5V@lw=McH)MXr`|r&~OT=z}#kOkF7p zOZhWic^C|#@Hr5p_X$Ots7~+%MJ#Vu7p~}e8_5fWi5Q*C`EtJUIUINdSA78y^Us18 zA|oW1rsUQACv|AXW;F3U35)e|sujk7eO_$Q$nDTHcTJUJPV6rsWBTc~g`q~d^5-~VltDZJuaBe2&S3n zMGunLi-*Ed!~-+FjVx`^!uxuWf4{ExiEHs+LptI4NZJP+`gylQo(|KqiOJD>ehoVf zl~>r?Ot75icF{Rr+WK6+tQ(&3g&+=dUy074+C%Wm6Nd1Nd%-W}!JEJ0jv1R|E3Aue zlC6lbkHz^^(gO=B#tyc0WE&=~pWfb^51~EW(y-`f$EdwD_qQF;Sb!U9X{U5BV&_K# zr)_5HVp+J{Os7W0uNJ>DFk{`zCWgBzx}n%!jg9_ps?lc4FOF<(S-7TLf?ZUYX=(CMjzioxxG4P4{_c_w|;vydr(oU4198iRjtX{Jb=rp0FEJ!jK44`AfvBZ6S0kcx>+cV2nVt-frM!vat`1ONexOM)haAQs}RaTh=Fx=3^e zd$hR!I31x!g{jkebDC$<4tne>4|MF*E39eHwNxMl!=t0YI{;HRjUj&~8wwNCDVQ}@ ze=DzxL%;j~oeu3_=Njf~w;30t=OV&A6>gbZ3Xvq64|AI*rf3zaU}d*r6HOnqN#x>J z^=CQZFqM(ZE3}_3ov+bMD#L^US6PTeSzwZHzd&k&bk|03Wb}%GQ6({Esb^}4<7mvQTPi1>6{)Qkt@oHgGe*%J3=j#va6XT z9?E^?-!N!2y@ES%#z$!WmY711b?^w5&?Pv;0XygT5fuef<>J8I-iGPPpI|~HolL<} z=*b;|n6Vc5gicII4XzmPRoDu<%_Q-@2xe4yVW`yPq1mak;*`Wz6kN*u8+f8=GV|X~ zsgM;-B=9W-ajBgc=kOFMw9L#OzbuViEeh$7%C-~_qfsBNPe-mO_9WTyzGmn!=A);t z_i|0t4{$M!@EBoW#@pYa^~X#PGj`CEhHPkxk<4^VE#>Vze~wI-zY`LAte2-TbpQCT ziIZ3^TtiRd$5BPi$*`Qjv@Wh7y;_z+a_|IQg?8gH+0C4_Rj?F*&emp`hFm z%HEk&P>b5NvGIL(V|n0)O!r3@KN2HyH)j&ZOTC2z8>TbM(>8cXifA&!uVa}GS7u}5 z!=ewrh9RT}TKrn|K4 zC6U;2K*U1U+5*PRbPt9tjvb2!-^|!a+fhFt*5_XbqzKcc6;V7t8JMvMOM{xtIeow4 zJh{zy3?04tELRc-gO&L-oC7mVNxaVyFF~R9E^w2`YcRXQyj_1UNpFYl1y7IC*t;#FNFWnC^ua3qP*xEN7d zDAPh4ECFNaOU3iJo#*KuSi+rK^aA#PhTclAv(z+aN|Cfvk94dd>@kMa4Xx!NWjFfXCuB6Fk;w zn}$(JXgn9YC1Rovo^OkchD^LNFr|Sh&$p`=ZHmNB2i4S_$Xc!;9y?-_LrJl;+4&tw zuN}J=5Tj526!>r-r_TcF`^tDDBhD}>_ z4^~p%^JTUsA8p$CK143PsjtcqK26epEVG$?7%})QSh~K$w(z41tU}T4I2Yh26n?A> zn9KZy2^1q4jEuJj&(R}YD)(|y$X*nExEJ#}{UKNa;Dhj1n;HS~G(_07&`Rk* zt7du&f}RP>{8F}cT7=rnfG)1d`)&7A^29K8cH%=UR4EWU*_Q&qfgpHQ*qp9cnp zbp~l5O4M)X`3}-Uo~$$@E$*r*rHtcP7HQhTNn${eB@ZRejMO4-w(s0w{>{L^gAY!j z6bz&}FWioqgl~WGuCnN+w`u92gUBg)k9?I=AtxA}0+RtQ($+{aH5M(#+xyBbgIG=n zIy!+IIuQyK2l;Y#nr;Z=s|h)C)O?I@CEKRcb`+}I7r~;=E5{T8Hoyt}eb0gg3YxbE zcqfBV+HgWS(Fwi}ZibNPSMchBs?C*4(+qi}B&8=vg03_s3|IOsIO4uyG_8ORHWjkK z#&`}pk=rrv_`Ahr@;p(ENrk5PLiYS<7%u`q<$TOYRFZE5>6cl&*>U??hCTlU0Q&0yT_9g$eo@JI$# zbp$%dZ>Dg=#AJa}M}NY(<4Q1Kd;SR!|0g<$l>)oFZ{uJ~M>}LUN56l8jB|c)$NZe_ zpO#~vBgTGw3N()cVx=Ku@G?!!1mIU@kRFWY zq&9&4ESWaCio0*#)tjy8YWEMgAAQ#+wP~EkUm+>c-z6-C4#vN&LleY7!%Slo)zq7} zw4KPX#YulYNnhF_Pj4%J1y5dTF&qv^+z|b44<8F_=B)$H!{cARqSyT}qA*Rl^TD z1W=c(x=P|0fL@)9PN!~(M;EwhD&^k?!y~5_DugF&=XG;SZU*Z$#42B6yaG7G+if>l`B{v(d5gVH zJb7WdS;GjU9BwYJh6bxDO6_AAhk|W6JxTnos zdJ};B#VL5Of0rtY4(;ki7LQZ6;|Ktoh`e~;b{&_DWg#p>{HYzJXfs-Y)mz*dZN+GlW!_6Vd<+dp{y~k+JlW4o{=-9L-%dv^r^E>Agx zs|k$nGV!k`;KpZMfM$yW6Gy}Irqi}sni>3t=p)jb$M3yy?vlOZD%#1@7iZ}P;0Z8` zC+KvC|MA#QW2L8T*(UgZY`huOG!OL16W8rj)2wQ4iq}L97W*m2ARA*was*QE@>YsZ zlv2#v`~*-Ep2-Kr+x6Qgh(z}G@k=6obfRRh7Ix0PGq-Yk7%mg$OEaD}ov?I5aF+1T zOxVIQyMf`LY=UMA z;MsXgGjEt2Hy&5u9zqc`B0zjCVh!$9V}49)oH&x0jlU(^Pv{7#NW-ct_$HoqY=V!2Tmh?r+gisPupYNv8|_UX45cR86|gI>>LK( z!nWQR-^`#av4Jne8-QT4VC;-~$arJl1Ag>nlK4y>dJmTZ*^t>x`@Lph$hWWxImBkr z#G^v}dusfl`p0xz#{=o$${yywF_99Q-UFM+qN%j-Bw4$h7+?-f?uq{X1ZZ#tu>*hz zQ=I1AAIO021~VQIuZD>=@wviaO*|9M-G}?L=zpVTs|_z4(Q7?{tFYZ6CxNAdUP1g7 zi70@GSmIOWcMaU2^*mvgz)V;=~g6Dgp>*#AA1j|+sZ&MZTjaR%yRqTI(S6m`1en3@Rg^C5LVnMv(998i- zQE{rQ_zRpn@D>TMRbnufK!!>1JE9MPUrh2Vv4o?y{EX^PO91o>C_0rh?o)VclscU0 zZfQtU98n>k|N0@SSdf*8W}4i!@@67TqY$6qwfgZe3=s}D*p$x#TX-^50TNu^W8VQJ z{JoD7tQLxgRvwgtC!9MQ;6zVyHePIDe%yWYB$j`=pCnFf(;)3*acNrL9sK#xkZ1QJ z0Yrzw!Dp*@+3_^ar^so&h`2a1jc=Ngq6A9Y(1_DNh@LycBzbAE}5o{h1to+SjF1-d_<$0!O`pSNzdC zgDPfJ6&~K9SG;p@McL<6{rK}NxIL&j6>5sxLo9;G|6N8Q3HT-3rxnjflZP#hzJr-A zLS9<^@lyuS_YUYQ^v8A`c@y$@`X`7lkE4Q_0v5ikR zYH`Ge?!WG!3NpYa#w()|N#j2!lZ?1F(*B2#S6-VWGj}+dL$5zcvZf9vi}V~$UW$Hy z9&P|K$>TfkB5ZH*z+W8rCwKgJ!wHt4W`vU4$GsV5b$mq9)E{y^$kd-ql??v)?=vN@ z#9|CM{^^~9BbtW?Xz>-)5q%M_qQ&OL=HSbJzq%aitI;Bt2|Fryc=VHeo=(DPKr z_W94tT-IfZ-4est;_mAqG%l2(=lhQ6IOI_*)Y+Iuoz+sEVRq!n3dQ|Z5-9RUmH%GM zu5Z$PC>;a*2Dj>5rMz8V2MtGr!E^Ktr7HZsBE9JYmf7-dUYsV}#943_M3;is8=Fe_gK61ZJ0tNyG0tNyG0tNyG0tNyG0tNyG z0tNyG0tNyG0+$>C9CI0;zhVT6nuPN8?s{h({t0Q3*Hz>8x~l!lHNV$c?OLv7EO&Xm zjb2sW;IHEOs(P2t=iEZ?5Ax5g%Qsf~TwdSC8mF^iV~uO;#z8G@#6LA{tgdr!T3oGa z;m3QNbv_a@hQf8}KYb#y7)^;oZ%0;^JHFw(9u3cSflVYx4 zv3_l(ZN=TTbt~4~p2%;zJ)JmKwBojsvURocrf6JP6N_m0#3|nqD{l z1Wp#nN_x4^RUPogOH=5JYFwKFTN3K(`KF zwMK7^%e!2a==?4rBG9lXmBd*KxY3j6Sg37+{VdmtnhK<^BnB7^bqFKwb8dCvPY&;I zY^Wi-K;N3&^?`cr(}8-A=5s%Qs+b?zO>|XS#N$ANa})H0oGmWDw$1DIyEKo>TkrPK z^wKsrdNpTFjTcT;^Z8+ES|R*efoITAh??Ax(Vhp2yBCtm4>Te9fLm{HD82@#&c@<; zajPL93(@QfEnbjL;kW>qinDtgVXT{+)%Rt#?K$!4n4HyoL8>D3)6gZUs-7T@Gl@K7 zRjwuvELty+d0kMT+UqCDx*^Gu1btn;#ySiw7LhqvN63g>G*~q3`YccKvNVbwuWKtt zLF2k}o0NVe!~7I>N)A?x)^e>H|B#8X#{5^~Cya;>rmnwAz%rDUb7{#VWj&;ba)6XA zEO4h9jH>6Z_rRCvoy5O%LKALcBxG=1`KO+$I@BaHDDbK7$$3XBFIQ3Ftn~HOy6WnR zm*^f`2qH}NDvFiIsvG*gglcL&m#f}K3T}c2RvrL;%DLI^@8-69z`V$B0k}g1E9h6fI8Y*4*@+iohM0{dts7m0XbzPEWMdBMQ zjq@c)!~En8h#8(JFisW%iI0{E)vC-YHMtx))f2^#UV@yUSHMTUwu(Cp#SFd#vM!d( zZE$U)wXD|LxJ@-raudkO{B9rEs%l+M54_hFxNp4{@Bg6Ucx{1)#22gCmIhZ%LS-MIy0G?_Cm=+7pAVR!ub-^}GUK5=19dsnj=R z?d0tx^pCi!lC)0kXNo+%|AT8L;1I7w{iF8cYkO*QY^}d;9Tt8&CL5{8`BP~`2kkZt zTPj}?P4caEmCzlq(;w&7<)$s8dipphgCZh6Iz7d(c)!#jtFcJHQm-eCP0uR~g?NEa zT#j_PaY1@Lj-jkXdJ(RwUP9SmdNSmC9XXp)x*PAWKHHKFGf93jPf|mv?WA-&S$|#5 zpzecD%!8UuTX^Sr`zbUk(7mH%iRBVMk6sS>rM-h(t6L{E_<=a}9!=NI!%r|J6W^tcdxyi%~`a&$XO9t$o{ zm8HR2Z*K^mB>4$4;+l~Z;ICvPj6o7z?-OT_^QYh|O67PenQ3@Z^ooy1iekE6N%Kh# zCh46xPc!f&^+TTvgR$m(Y2+oqrn;B5-2}drUL@eFw;%7aE)X zuRZe4kMFNNT8Ivf(J3tjizGQ}sppqk8$(Sp`?myW-Uus(iTogshwi zS(kaoDgHTGH_j=Z{gz&Ta#qgdtO<|isB)3_>iJZB=42PlnK#=sjR^^++1WSFE}mtY z$%KZcNm&Jx=3RD7Z%^c3&6v;2D#)8R@tBT(s=ild7hE}Srs*ao5HU^5E|@ki*R+v| z8m3v<1+(T&Gkt)G5mEVRDlhe&uJldq|5e#HUR9iL>SR+n0!sdLC12?Gs;=ME?1HKD zCYzQsfskp?cnW>=_EY;aJG)@^yji9j6ke(FQ|)O^7U3NquT=X=#V^%fg*`v2+iR-6 zv$IH_w{-iOm3`x^;%TPo3cu9xPpvQV$8`B}yr!w~O6||gtb&>IroNSNz6g6hrrYP7 zY}=e$XPffX_@%Z#C97b{ya{jV_9gM1rtlT|d-d^4#Vb|+)V#nuK0i~-Q{`Q3K2=_7 zd#QK~SDwnBs?TukrShlxL*b9Sy1k^9r`Au+57+)xSp`?k%Re?<`J>eSq~?Wv9nM@(U5H2umy`_BDig2tI_5A~-~;{1}Isx^A6Y8o3{$rO5cLlpI_xvIv^59yO={IMIasIv)D zx|>00WPC=BfcQz}1pd#PnrT$?Wt>nlqGZLLqW*}O4JI9dLW#F^l(@H78o29VUPPio z;$1sR+ygeU!Lc3_lYG^CD6m67Q~2;@)fEei3NLStAnfv!lel-M~E# z=0&8U3h&S;*T=g?S??lr{99zrmv|SA68B~U_feUJUE*CkO5Cjm?xVuJRpNbcl( z+((6bMB;sJl(@4EhJPOo@E0WB`HzeaKdXWJXkceW67Pyp;@)oHJ}TTj67Tj=;@)fE zJ}TV1B;IF7iMw|9=={^Dl>Hm3bV1@h+VjZ2At>J4u7riXj`lqAy;b&ar!}+0U7bhf z%e?BL;~vA0C@d~zyinq-&Lfwm;BGN+zm#z6vZ4_7vQ^4BU0BE@h%i;;x>r zI#O_NHgLa`aO<*qCGJe#w>G8V-fQ5lV|6JL?GopX6ud8G8-}dG2y`2`4+iBup!7=I zjr*qe0PKV&&Pm*j`=$g?-h*Vn#ND`WdJn)(Y~l-I|8CqjC4w^c>9cYlX_5FB%D6+G zGgf@kI42m3QP?eUw@bVoGVYbQHygN*1?(&$ac`D*@0PJg;;tFEj|uMG66YQnt41uF zlKsoUT_Ruf&2jv_ZN+EAqeI}F#JfKQ_x239FEwx<9ndMY%wH6CtjSmvYt`xP-D=>j zAbMA^RpLF`_mrS!vh+*%;VmKOxvl8ci8JpYXe(_!rR~WdD2G(Kg6iS>&`<~vg4Hyh*5wJ_#)q8ri zGOw8X4SQEmT++B*;;i1&>q^00a~k`ZOA5FYc2?pXO~E^2oXd;_+;N$IC+yiGV^v_? zXxO>JVl3i9iTBbJ+?j#Voc&D ziT7?9cgwg-ozEEeO=E)lIfb+QeF)=w2V(;BAfXo|?#6lKAZW&vVEK|*SBqp^A!Al0 z=*@`1=Txk?u!))8!_%5NQYE*AUL^HZeBc#P%bcN4gjBoEaweI?`6eKS!toKbvV{ z|7(_sZ6}%eCUyv6BhnGXg;yXA{tCp$5jv6XMSRl-OzaV)9f-$WiMmJ^A>NPh0`VZ8 zJ=?^}h>lph%EX=^e#GBGs3RW4ldcBO-=iGyc7%68??C*(T*wFAGtb0cM3{~AImDm7 z27N)=ve3jni10enMTn~q&JaK1FCsjFbO+*ogihjLgtif0K)M<6^2JC$4t~VHLwEvd z?K%^iR)n^Yu0XsQp_6zJXDxxAlt%ngF>ogS8-NYM8KNU*H$s2NFGBqJk6>ItKZm&N zqv#9qApRGGI;8s%=PpBiq#cN7u0a2hZb$rkgq28JR+^Y;75Kk~@k0DngoQ|VA-?h^ z=t*?Mxi_N?N+bR?6*Q6$mv2nvjjSTUW9lr!b+rj5x-!A4oF9CGqJxQY)9Ji zF%z>RypFU3@%IrXKY_j=e(>YKAM~P6nAl@w&;j&*#4nZuE6~~PCidAiz!T|ni0`a` z9O%%Dxa|(K2YN5!m)AlL=xiPOv>q6O-i&x%C3Hr*9r5!BD6ZU6X|BehY(&s8vo_W-b6S< z{2NTH7~yr|N8E&P9%=j!Bl|AGJ4oZd0h#%3VDfbn!~fN>4G6Q5#&7JhFCZ*L8ow{i zP9v;D8oznW*geo6Y5YGDvmtCm8vjwn9zdu=8vi51ok}UW}6UJB8~6Pvwa9HW#4|=_14% z5Vj+Y?;x|M5FSAq--~7cf$#*<`0g58hR}&LzE8zk5MDqU-|=CuA)G-P-#cMT9MGS5 z5H}&5M;gDc$c`esgEW47j{O>8@{{04JlzTXk;ZQpu~LMENLL{CBdkOk-<4x~5z3Ip z_lVd}5H=!>@3ye)P0$}{{7x^cL)eZqemj=EjqnK4_&rlLzXtjv?LquGgoRI_FUH3} zz(Bx2;5|m5>9^x}i;u{-Q^sdx{DzD#$avaYf`74$%Va#(YWMbwcB!4s)Mi3HvF*Q> z{%p!#>#tkg;P#g^_}uIKUU$P5_S!hm;`#NBl@0Fd#u}F_X1~utc@k}XW1O9_CRSGC z{g}&NT32UxHq_L)*eTQP{t9oS-&O6u)9v*KoOR`{`bO{ljNL%(gZFm-oi4A>-Pmw@ zpnellj5U%JTSK7U<#qa9WsS}nS50|iO`y)@V{=G@I|456{kDd!Zf|2ly{o~0r_<|p zZbECHp_;e38)|F~e((LaHF`^GYP>EVWH$I~>gpCFBo394{{IdE6U&EA2+hf=;hqeH zqBQgg*^Jpy<}qcl9mX~$8MZXJl{Yul*J)eDoLjZv`o%>HG*<&=IUK;M1(oY>TeNh6 z=JR9vI_nx6T&ot`@A56U`KH{7E1f={tA116{Tc)`_*N|lcpH}cs%u^KPT!(>ceS_C z*SOigsJgL!xzks_c;_~T*iN&@iRQ1Evl)1LL>a;q#u3F&qt!~)b zc%RF=Knu7_s;l8IRxQ}9q1Xy>qC(@;ZeZGHJ%cAQ=d} zuMz0NeJlX5bmVtbbhJLz{#4IXY>#=5bx+gY=Dpo}`}bYgSG2$Ofak!$1LqD#4{C=t z9CjSueYp3Tv(H#M?MEt(v>u5(+w&|t3Lz2PCnH+gVeM$@=z6OAss5)5_Z01^-P5u6 z;NEk4^Y?4}+YjtMU_N9ylz*u3Q1ju|!|jJV4qteNb(%Z#J3U95jx-->KN5Ym|Je)A znvb$x+*d=MqocOt>{I8Ss@St(PtV@oz1IEq{T2Hi`@0WB4)h#2d!Xph(nHom_Cpni z4j%40+83r+ds_GG-qXML!d|w|vafc(XMfZF*8S%WL=W^IxNv|S+;GTosQ2*M z!{-k7AGUW^bZ+RZ?IhV}kC5DoqpaV=Y9N;+_H-0JUG%i|>5e@I_jK*)-V@oAzfap& zxUXp6(tYjwckl1me{g@-e)B=gLG57kq1Hp~hjt(8ICSAKd&c}s{xhD=rq1Th*3S0M z=#l;-7ml!J&CfcH)*fww_T$)Wv~jlMTt~E{zvDtj#nT&}c067CwC8D%jgNtVfq;R) Hh!FU{B>c_G From 507f75b56c96c8921216f2154a129ad425b49fec Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 7 Feb 2023 11:07:08 -0800 Subject: [PATCH 005/102] update vs2017 project files --- ide/vs2017/mimalloc-override-test.vcxproj | 4 +- ide/vs2017/mimalloc-override.vcxproj | 2 +- ide/vs2017/mimalloc-test-stress.vcxproj | 4 +- ide/vs2017/mimalloc-test.vcxproj | 6 +- ide/vs2017/mimalloc.sln | 142 +++++++++++----------- ide/vs2017/mimalloc.vcxproj | 4 +- 6 files changed, 81 insertions(+), 81 deletions(-) diff --git a/ide/vs2017/mimalloc-override-test.vcxproj b/ide/vs2017/mimalloc-override-test.vcxproj index faaa00e3..04c16a9f 100644 --- a/ide/vs2017/mimalloc-override-test.vcxproj +++ b/ide/vs2017/mimalloc-override-test.vcxproj @@ -1,4 +1,4 @@ - + @@ -22,8 +22,8 @@ 15.0 {FEF7868F-750E-4C21-A04D-22707CC66879} mimalloc-override-test - 10.0.17134.0 mimalloc-override-test + 10.0.19041.0 diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index 1c6a8fda..f3b7dd1e 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -22,8 +22,8 @@ 15.0 {ABB5EAE7-B3E6-432E-B636-333449892EA7} mimalloc-override - 10.0.17134.0 mimalloc-override + 10.0.19041.0 diff --git a/ide/vs2017/mimalloc-test-stress.vcxproj b/ide/vs2017/mimalloc-test-stress.vcxproj index b8267d0b..061b8605 100644 --- a/ide/vs2017/mimalloc-test-stress.vcxproj +++ b/ide/vs2017/mimalloc-test-stress.vcxproj @@ -1,4 +1,4 @@ - + @@ -22,8 +22,8 @@ 15.0 {FEF7958F-750E-4C21-A04D-22707CC66878} mimalloc-test-stress - 10.0.17134.0 mimalloc-test-stress + 10.0.19041.0 diff --git a/ide/vs2017/mimalloc-test.vcxproj b/ide/vs2017/mimalloc-test.vcxproj index 27c7bb6e..04bd6537 100644 --- a/ide/vs2017/mimalloc-test.vcxproj +++ b/ide/vs2017/mimalloc-test.vcxproj @@ -1,4 +1,4 @@ - + @@ -22,8 +22,8 @@ 15.0 {FEF7858F-750E-4C21-A04D-22707CC66878} mimalloctest - 10.0.17134.0 mimalloc-test + 10.0.19041.0 @@ -102,7 +102,7 @@ true true ..\..\include - stdcpp17 + stdcpp14 Console diff --git a/ide/vs2017/mimalloc.sln b/ide/vs2017/mimalloc.sln index 7dbf53e1..515c03f2 100644 --- a/ide/vs2017/mimalloc.sln +++ b/ide/vs2017/mimalloc.sln @@ -1,71 +1,71 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.28010.2016 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc", "mimalloc.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA6}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test", "mimalloc-test.vcxproj", "{FEF7858F-750E-4C21-A04D-22707CC66878}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override", "mimalloc-override.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA7}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-test", "mimalloc-override-test.vcxproj", "{FEF7868F-750E-4C21-A04D-22707CC66879}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-stress", "mimalloc-test-stress.vcxproj", "{FEF7958F-750E-4C21-A04D-22707CC66878}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.ActiveCfg = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.Build.0 = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.ActiveCfg = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.Build.0 = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.ActiveCfg = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.Build.0 = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.ActiveCfg = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.Build.0 = Release|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.ActiveCfg = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.Build.0 = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.ActiveCfg = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.Build.0 = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.ActiveCfg = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.Build.0 = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.ActiveCfg = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.Build.0 = Release|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.Build.0 = Debug|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.Build.0 = Debug|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.ActiveCfg = Release|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.Build.0 = Release|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.ActiveCfg = Release|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.Build.0 = Release|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {4297F93D-486A-4243-995F-7D32F59AE82A} - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26228.102 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc", "mimalloc.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA6}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test", "mimalloc-test.vcxproj", "{FEF7858F-750E-4C21-A04D-22707CC66878}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override", "mimalloc-override.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA7}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-test", "mimalloc-override-test.vcxproj", "{FEF7868F-750E-4C21-A04D-22707CC66879}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-stress", "mimalloc-test-stress.vcxproj", "{FEF7958F-750E-4C21-A04D-22707CC66878}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.ActiveCfg = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.Build.0 = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.ActiveCfg = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.Build.0 = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.ActiveCfg = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.Build.0 = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.ActiveCfg = Release|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.Build.0 = Release|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.ActiveCfg = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.Build.0 = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.ActiveCfg = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.Build.0 = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.ActiveCfg = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.Build.0 = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.ActiveCfg = Release|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.Build.0 = Release|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.ActiveCfg = Debug|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.Build.0 = Debug|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.ActiveCfg = Debug|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.Build.0 = Debug|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.ActiveCfg = Release|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.Build.0 = Release|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.ActiveCfg = Release|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.Build.0 = Release|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {4297F93D-486A-4243-995F-7D32F59AE82A} + EndGlobalSection +EndGlobal diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index 8102b9fe..d29c2c7f 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -22,7 +22,7 @@ 15.0 {ABB5EAE7-B3E6-432E-B636-333449892EA6} mimalloc - 10.0.17134.0 + 10.0.19041.0 mimalloc @@ -131,7 +131,7 @@ _CRT_SECURE_NO_WARNINGS;MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsC false - stdcpp17 + stdcpp14 From 6a230f8329260c52aae786c86d294d61f0862873 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 7 Feb 2023 11:07:52 -0800 Subject: [PATCH 006/102] fix compilation of heap specific STL allocators for vs2017 --- include/mimalloc.h | 4 +++- test/main-override.cpp | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index f5900336..c13bda23 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -470,7 +470,9 @@ template bool operator==(const mi_stl_allocator& , const template bool operator!=(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return false; } -#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 +#if (__cplusplus >= 201103L) || (_MSC_VER >= 1920) // C++11, at least vs2019 +#define MI_HAS_HEAP_STL_ALLOCATOR 1 + #include // std::shared_ptr // Common base class for STL allocators in a specific heap diff --git a/test/main-override.cpp b/test/main-override.cpp index e63d605a..db96efb1 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -130,6 +130,7 @@ static bool test_stl_allocator2() { return vec.size() == 0; } +#if MI_HAS_HEAP_STL_ALLOCATOR static bool test_stl_allocator3() { std::vector > vec; vec.push_back(1); @@ -157,14 +158,17 @@ static bool test_stl_allocator6() { vec.pop_back(); return vec.size() == 0; } +#endif static void test_stl_allocators() { test_stl_allocator1(); test_stl_allocator2(); +#if MI_HAS_HEAP_STL_ALLOCATOR test_stl_allocator3(); test_stl_allocator4(); test_stl_allocator5(); test_stl_allocator6(); +#endif } // issue 445 From 8be4cee4186120c32def9789c450a185c7213914 Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 16 Nov 2022 18:52:40 -0800 Subject: [PATCH 007/102] change max align size to 8 --- include/mimalloc-types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index f3af528e..a7c4d3c6 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -19,7 +19,7 @@ terms of the MIT license. A copy of the license can be found in the file // Minimal alignment necessary. On most platforms 16 bytes are needed // due to SSE registers for example. This must be at least `sizeof(void*)` #ifndef MI_MAX_ALIGN_SIZE -#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) +#define MI_MAX_ALIGN_SIZE 8 // sizeof(max_align_t) #endif // ------------------------------------------------------ From 5fe4a3480ffb2aa21bf12b1e92c220ab575a582f Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Feb 2023 12:21:06 -0800 Subject: [PATCH 008/102] revert default max align commit back to 16 --- include/mimalloc-types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index a7c4d3c6..f3af528e 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -19,7 +19,7 @@ terms of the MIT license. A copy of the license can be found in the file // Minimal alignment necessary. On most platforms 16 bytes are needed // due to SSE registers for example. This must be at least `sizeof(void*)` #ifndef MI_MAX_ALIGN_SIZE -#define MI_MAX_ALIGN_SIZE 8 // sizeof(max_align_t) +#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) #endif // ------------------------------------------------------ From cb4fc2c79265c6bbfb76ace676523de9da9a321a Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 4 Mar 2023 09:17:35 -0800 Subject: [PATCH 009/102] reset stats for stress test when using mimalloc --- test/test-stress.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test-stress.c b/test/test-stress.c index 298d48b0..87e41736 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -241,6 +241,9 @@ int main(int argc, char** argv) { //printf("(reserve huge: %i\n)", res); //bench_start_program(); +#ifndef USE_STD_MALLOC + mi_stats_reset(); +#endif // Run ITER full iterations where half the objects in the transfer buffer survive to the next round. srand(0x7feb352d); From 0d9e7ab61e9ff6b7fc1d5abcaa9531d0a746eff9 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 4 Mar 2023 09:27:04 -0800 Subject: [PATCH 010/102] remove extern inline from alloc_new functions to avoid link warnings --- src/alloc.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 18261fbf..810df842 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -922,7 +922,7 @@ static bool mi_try_new_handler(bool nothrow) { } #endif -static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow ) { +mi_decl_export mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow ) { void* p = NULL; while(p == NULL && mi_try_new_handler(nothrow)) { p = mi_heap_malloc(heap,size); @@ -935,7 +935,7 @@ static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) { } -mi_decl_nodiscard mi_decl_restrict extern inline void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) { +mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) { void* p = mi_heap_malloc(heap,size); if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false); return p; @@ -946,7 +946,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) { } -mi_decl_nodiscard mi_decl_restrict extern inline void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) { +mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(count, size, &total)) { mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc @@ -1019,8 +1019,8 @@ void* _mi_externs[] = { (void*)&mi_zalloc_small, (void*)&mi_heap_malloc, (void*)&mi_heap_zalloc, - (void*)&mi_heap_malloc_small, - (void*)&mi_heap_alloc_new, - (void*)&mi_heap_alloc_new_n + (void*)&mi_heap_malloc_small + // (void*)&mi_heap_alloc_new, + // (void*)&mi_heap_alloc_new_n }; #endif From 6cc0ad72fc28bde53783b2dd246a86c6efb9943d Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 4 Mar 2023 09:58:02 -0800 Subject: [PATCH 011/102] match declaration of mi_malloc_size_checked on macOS --- src/alloc-override.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc-override.c b/src/alloc-override.c index 84a0d19d..40098ac5 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -57,7 +57,7 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; // functions that are interposed (or the interposing does not work) #define MI_OSX_IS_INTERPOSED - mi_decl_externc static size_t mi_malloc_size_checked(void *p) { + mi_decl_externc size_t mi_malloc_size_checked(void *p) { if (!mi_is_in_heap_region(p)) return 0; return mi_usable_size(p); } From e24c7c9de6311f5268e8d6b29cdc1d02125ab77a Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 4 Mar 2023 09:58:17 -0800 Subject: [PATCH 012/102] fix asan compilation on macOSX --- CMakeLists.txt | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0011b874..dc4aeb71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,8 @@ set(mi_sources src/options.c src/init.c) +set(mi_cflags "") +set(mi_libraries "") # ----------------------------------------------------------------------------- # Convenience: set default build type depending on the build directory @@ -141,10 +143,15 @@ if(MI_VALGRIND) endif() if(MI_ASAN) + if (APPLE AND MI_OVERRIDE) + set(MI_ASAN OFF) + message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_ASAN=OFF)") + endif() if (MI_VALGRIND) set(MI_ASAN OFF) message(WARNING "Cannot enable address sanitizer support with also Valgrind support enabled (MI_ASAN=OFF)") - else() + endif() + if(MI_ASAN) CHECK_INCLUDE_FILES("sanitizer/asan_interface.h" MI_HAS_ASANH) if (NOT MI_HAS_ASANH) set(MI_ASAN OFF) @@ -154,7 +161,7 @@ if(MI_ASAN) message(STATUS "Compile with address sanitizer support (MI_ASAN=ON)") list(APPEND mi_defines MI_ASAN=1) list(APPEND mi_cflags -fsanitize=address) - list(APPEND CMAKE_EXE_LINKER_FLAGS -fsanitize=address) + list(APPEND mi_libraries -fsanitize=address) endif() endif() endif() @@ -199,7 +206,7 @@ if(MI_DEBUG_TSAN) message(STATUS "Build with thread sanitizer (MI_DEBUG_TSAN=ON)") list(APPEND mi_defines MI_TSAN=1) list(APPEND mi_cflags -fsanitize=thread -g -O1) - list(APPEND CMAKE_EXE_LINKER_FLAGS -fsanitize=thread) + list(APPEND mi_libraries -fsanitize=thread) else() message(WARNING "Can only use thread sanitizer with clang (MI_DEBUG_TSAN=ON but ignored)") endif() @@ -210,7 +217,7 @@ if(MI_DEBUG_UBSAN) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") message(STATUS "Build with undefined-behavior sanitizer (MI_DEBUG_UBSAN=ON)") list(APPEND mi_cflags -fsanitize=undefined -g -fno-sanitize-recover=undefined) - list(APPEND CMAKE_EXE_LINKER_FLAGS -fsanitize=undefined) + list(APPEND mi_libraries -fsanitize=undefined) if (NOT MI_USE_CXX) message(STATUS "(switch to use C++ due to MI_DEBUG_UBSAN)") set(MI_USE_CXX "ON") @@ -363,7 +370,7 @@ if(MI_BUILD_SHARED) set_target_properties(mimalloc PROPERTIES VERSION ${mi_version} SOVERSION ${mi_version_major} OUTPUT_NAME ${mi_basename} ) target_compile_definitions(mimalloc PRIVATE ${mi_defines} MI_SHARED_LIB MI_SHARED_LIB_EXPORT) target_compile_options(mimalloc PRIVATE ${mi_cflags}) - target_link_libraries(mimalloc PRIVATE ${mi_libraries}) + target_link_libraries(mimalloc PRIVATE ${mi_libraries} -fsanitize=address) target_include_directories(mimalloc PUBLIC $ $ From 6dcebdc3036a73256ec5c772f53ba314c9d425dc Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 4 Mar 2023 14:49:02 -0800 Subject: [PATCH 013/102] fix sizes in memory tracking and padding for huge alignments --- CMakeLists.txt | 8 +-- include/mimalloc-internal.h | 1 + include/mimalloc-track.h | 15 ++++-- include/mimalloc-types.h | 13 +++-- src/alloc-aligned.c | 8 +-- src/alloc.c | 103 +++++++++++++++++++++--------------- src/init.c | 2 +- src/page.c | 4 +- test/test-api.c | 5 +- 9 files changed, 96 insertions(+), 63 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dc4aeb71..46bbc109 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ set(CMAKE_CXX_STANDARD 17) option(MI_SECURE "Use full security mitigations (like guard pages, allocation randomization, double-free mitigation, and free-list corruption detection)" OFF) option(MI_DEBUG_FULL "Use full internal heap invariant checking in DEBUG mode (expensive)" OFF) -option(MI_PADDING "Enable padding to detect heap block overflow (used only in DEBUG mode or with Valgrind)" ON) +option(MI_PADDING "Enable padding to detect heap block overflow (always on in DEBUG mode or with Valgrind)" OFF) option(MI_OVERRIDE "Override the standard malloc interface (e.g. define entry points for malloc() etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) @@ -186,9 +186,9 @@ if(MI_DEBUG_FULL) list(APPEND mi_defines MI_DEBUG=3) # full invariant checking endif() -if(NOT MI_PADDING) - message(STATUS "Disable padding of heap blocks in debug mode (MI_PADDING=OFF)") - list(APPEND mi_defines MI_PADDING=0) +if(MI_PADDING) + message(STATUS "Enable padding of heap blocks explicitly (MI_PADDING=ON)") + list(APPEND mi_defines MI_PADDING=1) endif() if(MI_XMALLOC) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index fc18a8f2..b3cdf716 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -162,6 +162,7 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); bool _mi_free_delayed_block(mi_block_t* block); void _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept; // for runtime integration +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); #if MI_DEBUG>1 bool _mi_page_is_valid(mi_page_t* page); diff --git a/include/mimalloc-track.h b/include/mimalloc-track.h index f60d7acd..35fb786a 100644 --- a/include/mimalloc-track.h +++ b/include/mimalloc-track.h @@ -13,6 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file // address sanitizer, or other memory checkers. // ------------------------------------------------------ + #if MI_VALGRIND #define MI_TRACK_ENABLED 1 @@ -23,8 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_track_malloc(p,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero) #define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/) -#define mi_track_free(p) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/) -#define mi_track_free_size(p,_size) mi_track_free(p) +#define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/) #define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size) #define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size) #define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) @@ -38,7 +38,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_track_malloc(p,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size) #define mi_track_resize(p,oldsize,newsize) ASAN_POISON_MEMORY_REGION(p,oldsize); ASAN_UNPOISON_MEMORY_REGION(p,newsize) -#define mi_track_free(p) ASAN_POISON_MEMORY_REGION(p,mi_usable_size(p)) #define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size) #define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) #define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) @@ -51,7 +50,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_track_malloc(p,size,zero) #define mi_track_resize(p,oldsize,newsize) -#define mi_track_free(p) #define mi_track_free_size(p,_size) #define mi_track_mem_defined(p,size) #define mi_track_mem_undefined(p,size) @@ -59,4 +57,13 @@ terms of the MIT license. A copy of the license can be found in the file #endif +#ifndef mi_track_free +#define mi_track_free(p) mi_track_free_size(p,mi_usable_size(p)); +#endif + +#ifndef mi_track_resize +#define mi_track_resize(p,oldsize,newsize) mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false) +#endif + + #endif diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 3ffa7fa2..e365a8f5 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -58,11 +58,16 @@ terms of the MIT license. A copy of the license can be found in the file #endif // Reserve extra padding at the end of each block to be more resilient against heap block overflows. -// The padding can detect byte-precise buffer overflow on free. -#if !defined(MI_PADDING) && (MI_DEBUG>=1 || MI_VALGRIND) +// The padding can detect buffer overflow on free. +#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || MI_VALGRIND || MI_ASAN) #define MI_PADDING 1 #endif +// Check padding bytes; allows byte-precise buffer overflow detection +#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1) +#define MI_PADDING_CHECK 1 +#endif + // Encoded free lists allow detection of corrupted free lists // and can detect buffer overflows, modify after free, and double `free`s. @@ -282,8 +287,8 @@ typedef struct mi_page_s { uint32_t xblock_size; // size available in each block (always `>0`) mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) - #ifdef MI_ENCODE_FREELIST - uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`) + #if (MI_ENCODE_FREELIST || MI_PADDING) + uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`) or padding canary #endif _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 8de3412b..8059f4b5 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -46,7 +46,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment); // the page block size should be large enough to align in the single huge page block // zero afterwards as only the area from the aligned_p may be committed! - if (p == NULL) return NULL; + if (p == NULL) return NULL; } else { // otherwise over-allocate @@ -61,7 +61,9 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* mi_assert_internal(adjust < alignment); void* aligned_p = (void*)((uintptr_t)p + adjust); if (aligned_p != p) { - mi_page_set_has_aligned(_mi_ptr_page(p), true); + mi_page_t* page = _mi_ptr_page(p); + mi_page_set_has_aligned(page, true); + _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); } mi_assert_internal(mi_page_usable_block_size(_mi_ptr_page(p)) >= adjust + size); @@ -80,7 +82,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* #if MI_TRACK_ENABLED if (p != aligned_p) { - mi_track_free_size(p, oversize); + mi_track_free(p); mi_track_malloc(aligned_p, size, zero); } else { diff --git a/src/alloc.c b/src/alloc.c index 810df842..9a91cc14 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -70,20 +70,22 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz } #endif -#if (MI_PADDING > 0) && defined(MI_ENCODE_FREELIST) && !MI_TRACK_ENABLED +#if MI_PADDING // && !MI_TRACK_ENABLED mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page)); ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); - #if (MI_DEBUG>1) + #if (MI_DEBUG>=2) mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta)); mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess #endif padding->canary = (uint32_t)(mi_ptr_encode(page,block,page->keys)); padding->delta = (uint32_t)(delta); + #if MI_PADDING_CHECK if (!mi_page_is_huge(page)) { uint8_t* fill = (uint8_t*)padding - delta; const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } } + #endif #endif return block; @@ -96,21 +98,23 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local #endif mi_assert(size <= MI_SMALL_SIZE_MAX); -#if (MI_PADDING) - if (size == 0) { - size = sizeof(void*); - } -#endif + #if (MI_PADDING) + if (size == 0) { size = sizeof(void*); } + #endif mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE); void* p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE, zero); + #if MI_PADDING + mi_assert_internal(p == NULL || mi_usable_size(p) == size); + #else mi_assert_internal(p == NULL || mi_usable_size(p) >= size); -#if MI_STAT>1 + #endif + #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_get_default_heap(); } mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); } -#endif - mi_track_malloc(p,size,zero); + #endif + if (p!=NULL) { mi_track_malloc(p,size,zero); } return p; } @@ -133,14 +137,18 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic + #if MI_PADDING + mi_assert_internal(p == NULL || mi_usable_size(p) == size); + #else mi_assert_internal(p == NULL || mi_usable_size(p) >= size); + #endif #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_get_default_heap(); } mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); } #endif - mi_track_malloc(p,size,zero); + if (p!=NULL) { mi_track_malloc(p,size,zero); } return p; } } @@ -225,7 +233,7 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block // Check for heap block overflow by setting up padding at the end of the block // --------------------------------------------------------------------------- -#if (MI_PADDING>0) && defined(MI_ENCODE_FREELIST) && !MI_TRACK_ENABLED +#if MI_PADDING // && !MI_TRACK_ENABLED static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) { *bsize = mi_page_usable_block_size(page); const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); @@ -249,6 +257,40 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl return (ok ? bsize - delta : 0); } +// When a non-thread-local block is freed, it becomes part of the thread delayed free +// list that is freed later by the owning heap. If the exact usable size is too small to +// contain the pointer for the delayed list, then shrink the padding (by decreasing delta) +// so it will later not trigger an overflow error in `mi_free_block`. +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { + size_t bsize; + size_t delta; + bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + mi_assert_internal(ok); + if (!ok || (bsize - delta) >= min_size) return; // usually already enough space + mi_assert_internal(bsize >= min_size); + if (bsize < min_size) return; // should never happen + size_t new_delta = (bsize - min_size); + mi_assert_internal(new_delta < bsize); + mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize); + mi_track_mem_defined(padding,sizeof(mi_padding_t)); + padding->delta = (uint32_t)new_delta; + mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); +} +#else +static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) { + MI_UNUSED(block); + return mi_page_usable_block_size(page); +} + +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { + MI_UNUSED(page); + MI_UNUSED(block); + MI_UNUSED(min_size); +} +#endif + +#if MI_PADDING && MI_PADDING_CHECK + static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) { size_t bsize; size_t delta; @@ -281,39 +323,13 @@ static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { } } -// When a non-thread-local block is freed, it becomes part of the thread delayed free -// list that is freed later by the owning heap. If the exact usable size is too small to -// contain the pointer for the delayed list, then shrink the padding (by decreasing delta) -// so it will later not trigger an overflow error in `mi_free_block`. -static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { - size_t bsize; - size_t delta; - bool ok = mi_page_decode_padding(page, block, &delta, &bsize); - mi_assert_internal(ok); - if (!ok || (bsize - delta) >= min_size) return; // usually already enough space - mi_assert_internal(bsize >= min_size); - if (bsize < min_size) return; // should never happen - size_t new_delta = (bsize - min_size); - mi_assert_internal(new_delta < bsize); - mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize); - padding->delta = (uint32_t)new_delta; -} #else + static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(page); MI_UNUSED(block); } -static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) { - MI_UNUSED(block); - return mi_page_usable_block_size(page); -} - -static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { - MI_UNUSED(page); - MI_UNUSED(block); - MI_UNUSED(min_size); -} #endif // only maintain stats for smaller objects if requested @@ -382,7 +398,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc // The padding check may access the non-thread-owned page for the key values. // that is safe as these are constant and the page won't be freed (as the block is not freed yet). mi_check_padding(page, block); - mi_padding_shrink(page, block, sizeof(mi_block_t)); // for small size, ensure we can fit the delayed thread pointers without triggering overflow detection + _mi_padding_shrink(page, block, sizeof(mi_block_t)); // for small size, ensure we can fit the delayed thread pointers without triggering overflow detection mi_segment_t* const segment = _mi_page_segment(page); if (segment->page_kind == MI_PAGE_HUGE) { @@ -677,9 +693,10 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.) const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL (with size 0) if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) - // todo: adjust potential padding to reflect the new size? - mi_track_free_size(p, size); - mi_track_malloc(p,newsize,true); + mi_assert_internal(p!=NULL); + // todo: do not track as the usable size is still the same in the free; adjust potential padding? + // mi_track_free(p); + // mi_track_malloc(p,newsize,true); return p; // reallocation still fits and not more than 50% waste } void* newp = mi_heap_malloc(heap,newsize); diff --git a/src/init.c b/src/init.c index 11c66a67..a2a6be75 100644 --- a/src/init.c +++ b/src/init.c @@ -22,7 +22,7 @@ const mi_page_t _mi_page_empty = { 0, // used 0, // xblock_size NULL, // local_free - #if MI_ENCODE_FREELIST + #if (MI_PADDING || MI_ENCODE_FREELIST) { 0, 0 }, #endif MI_ATOMIC_VAR_INIT(0), // xthread_free diff --git a/src/page.c b/src/page.c index 91dd0c06..70cb0d89 100644 --- a/src/page.c +++ b/src/page.c @@ -659,7 +659,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi mi_assert_internal(page_size / block_size < (1L<<16)); page->reserved = (uint16_t)(page_size / block_size); mi_assert_internal(page->reserved > 0); - #ifdef MI_ENCODE_FREELIST + #if (MI_PADDING || MI_ENCODE_FREELIST) page->keys[0] = _mi_heap_random_next(heap); page->keys[1] = _mi_heap_random_next(heap); #endif @@ -677,7 +677,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi mi_assert_internal(page->prev == NULL); mi_assert_internal(page->retire_expire == 0); mi_assert_internal(!mi_page_has_aligned(page)); - #if (MI_ENCODE_FREELIST) + #if (MI_PADDING || MI_ENCODE_FREELIST) mi_assert_internal(page->keys[0] != 0); mi_assert_internal(page->keys[1] != 0); #endif diff --git a/test/test-api.c b/test/test-api.c index e47bc1e4..20050ce8 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -51,7 +51,7 @@ bool test_stl_allocator2(void); // --------------------------------------------------------------------------- int main(void) { mi_option_disable(mi_option_verbose); - + // --------------------------------------------------- // Malloc // --------------------------------------------------- @@ -149,7 +149,8 @@ int main(void) { for (size_t align = 1; align <= MI_ALIGNMENT_MAX && ok; align *= 2) { void* ps[8]; for (int i = 0; i < 8 && ok; i++) { - ps[i] = mi_malloc_aligned(align*13 /*size*/, align); + ps[i] = mi_malloc_aligned(align*13 // size + , align); if (ps[i] == NULL || (uintptr_t)(ps[i]) % align != 0) { ok = false; } From 3c906bde8b0612bac5cec5207432f5dedc32da1b Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 4 Mar 2023 15:17:13 -0800 Subject: [PATCH 014/102] better track_free_size --- include/mimalloc-track.h | 13 +++++++++---- src/alloc-aligned.c | 4 +--- src/alloc.c | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/include/mimalloc-track.h b/include/mimalloc-track.h index 35fb786a..3a2c2741 100644 --- a/include/mimalloc-track.h +++ b/include/mimalloc-track.h @@ -8,11 +8,16 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_TRACK_H #define MIMALLOC_TRACK_H -// ------------------------------------------------------ -// Track memory ranges with macros for tools like Valgrind -// address sanitizer, or other memory checkers. -// ------------------------------------------------------ +/* ------------------------------------------------------------------------------------------------------ +Track memory ranges with macros for tools like Valgrind +address sanitizer, or other memory checkers. +The macros are set up such that the size passed to `mi_track_free_size` +matches the size of the allocation, or the newsize of a `mi_track_resize` (currently unused though). + +The `size` is either byte precise (and what the user requested) if `MI_PADDING` is enabled, +or otherwise it is the full block size which may be larger than the original request. +-------------------------------------------------------------------------------------------------------*/ #if MI_VALGRIND diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 8059f4b5..a75c9947 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -65,6 +65,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* mi_page_set_has_aligned(page, true); _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); } + // todo: expand padding if overallocated and p==aligned_p ? mi_assert_internal(mi_page_usable_block_size(_mi_ptr_page(p)) >= adjust + size); mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p), _mi_ptr_page(aligned_p), aligned_p)); @@ -85,9 +86,6 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* mi_track_free(p); mi_track_malloc(aligned_p, size, zero); } - else { - mi_track_resize(aligned_p, oversize, size); - } #endif return aligned_p; } diff --git a/src/alloc.c b/src/alloc.c index 9a91cc14..8ef78339 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -498,7 +498,7 @@ mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* p void mi_decl_noinline _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept { mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p); - mi_stat_free(page, block); // stat_free may access the padding + mi_stat_free(page, block); // stat_free may access the padding mi_track_free(p); _mi_free_block(page, is_local, block); } @@ -559,7 +559,7 @@ void mi_free(void* p) mi_attr_noexcept #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif - mi_track_free(p); + mi_track_free_size(p, mi_page_usable_size_of(page,block)); // faster then mi_usable_size as we already now the page and that p is unaligned mi_block_set_next(page, block, page->local_free); page->local_free = block; if mi_unlikely(--page->used == 0) { // using this expression generates better code than: page->used--; if (mi_page_all_free(page)) From 20ae35a1d4ef475631acc3674906f1051ced2c4f Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 4 Mar 2023 16:03:14 -0800 Subject: [PATCH 015/102] remove accidental -fsanitize --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 46bbc109..2e7b81b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -370,7 +370,7 @@ if(MI_BUILD_SHARED) set_target_properties(mimalloc PROPERTIES VERSION ${mi_version} SOVERSION ${mi_version_major} OUTPUT_NAME ${mi_basename} ) target_compile_definitions(mimalloc PRIVATE ${mi_defines} MI_SHARED_LIB MI_SHARED_LIB_EXPORT) target_compile_options(mimalloc PRIVATE ${mi_cflags}) - target_link_libraries(mimalloc PRIVATE ${mi_libraries} -fsanitize=address) + target_link_libraries(mimalloc PRIVATE ${mi_libraries}) target_include_directories(mimalloc PUBLIC $ $ From 056c2ce45b54ee96fd93c05b6085361c785fd850 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 5 Mar 2023 11:01:51 -0800 Subject: [PATCH 016/102] match track free size to tracked malloc size --- CMakeLists.txt | 14 +++++++--- include/mimalloc-track.h | 56 ++++++++++++++++++++++++++-------------- src/alloc-aligned.c | 9 ++++--- src/alloc.c | 23 +++++------------ 4 files changed, 57 insertions(+), 45 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e7b81b8..6b6ed554 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ set(CMAKE_CXX_STANDARD 17) option(MI_SECURE "Use full security mitigations (like guard pages, allocation randomization, double-free mitigation, and free-list corruption detection)" OFF) option(MI_DEBUG_FULL "Use full internal heap invariant checking in DEBUG mode (expensive)" OFF) -option(MI_PADDING "Enable padding to detect heap block overflow (always on in DEBUG mode or with Valgrind)" OFF) +option(MI_PADDING "Enable padding to detect heap block overflow (always on in DEBUG or SECURE mode, or with Valgrind/ASAN)" OFF) option(MI_OVERRIDE "Override the standard malloc interface (e.g. define entry points for malloc() etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) @@ -25,6 +25,7 @@ option(MI_BUILD_TESTS "Build test executables" ON) option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) option(MI_SKIP_COLLECT_ON_EXIT, "Skip collecting memory on program exit" OFF) +option(MI_NO_PADDING "Force no use of padding even in DEBUG mode ets." OFF) # deprecated options option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) @@ -186,9 +187,14 @@ if(MI_DEBUG_FULL) list(APPEND mi_defines MI_DEBUG=3) # full invariant checking endif() -if(MI_PADDING) - message(STATUS "Enable padding of heap blocks explicitly (MI_PADDING=ON)") - list(APPEND mi_defines MI_PADDING=1) +if(MI_NO_PADDING) + message(STATUS "Suppress any padding of heap blocks (MI_NO_PADDING=ON)") + list(APPEND mi_defines MI_PADDING=0) +else() + if(MI_PADDING) + message(STATUS "Enable explicit padding of heap blocks (MI_PADDING=ON)") + list(APPEND mi_defines MI_PADDING=1) + endif() endif() if(MI_XMALLOC) diff --git a/include/mimalloc-track.h b/include/mimalloc-track.h index 3a2c2741..8326c620 100644 --- a/include/mimalloc-track.h +++ b/include/mimalloc-track.h @@ -13,10 +13,13 @@ Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers. The macros are set up such that the size passed to `mi_track_free_size` -matches the size of the allocation, or the newsize of a `mi_track_resize` (currently unused though). +matches the size of the allocation, or the new size of a `mi_track_resize` (currently unused though). The `size` is either byte precise (and what the user requested) if `MI_PADDING` is enabled, or otherwise it is the full block size which may be larger than the original request. +Aligned pointers in a block are signaled right after a `mi_track_malloc` +with the `mi_track_align` macro. The corresponding `mi_track_free` still +uses the block start pointer and original size (corresponding to the `mi_track_malloc`). -------------------------------------------------------------------------------------------------------*/ #if MI_VALGRIND @@ -27,12 +30,12 @@ or otherwise it is the full block size which may be larger than the original req #include #include -#define mi_track_malloc(p,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero) -#define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/) -#define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/) -#define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size) -#define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size) -#define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) +#define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero) +#define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/) +#define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/) +#define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size) +#define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size) +#define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) #elif MI_ASAN @@ -41,34 +44,47 @@ or otherwise it is the full block size which may be larger than the original req #include -#define mi_track_malloc(p,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size) -#define mi_track_resize(p,oldsize,newsize) ASAN_POISON_MEMORY_REGION(p,oldsize); ASAN_UNPOISON_MEMORY_REGION(p,newsize) -#define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size) -#define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) -#define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) -#define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size) +#define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size) +#define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size) #else #define MI_TRACK_ENABLED 0 #define MI_TRACK_TOOL "none" -#define mi_track_malloc(p,size,zero) -#define mi_track_resize(p,oldsize,newsize) +#define mi_track_malloc_size(p,reqsize,size,zero) #define mi_track_free_size(p,_size) +#define mi_track_align(p,alignedp,offset,size) +#define mi_track_resize(p,oldsize,newsize) #define mi_track_mem_defined(p,size) #define mi_track_mem_undefined(p,size) #define mi_track_mem_noaccess(p,size) #endif -#ifndef mi_track_free -#define mi_track_free(p) mi_track_free_size(p,mi_usable_size(p)); -#endif - #ifndef mi_track_resize -#define mi_track_resize(p,oldsize,newsize) mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false) +#define mi_track_resize(p,oldsize,newsize) mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false) #endif +#ifndef mi_track_align +#define mi_track_align(p,alignedp,offset,size) mi_track_mem_noaccess(p,offset) +#endif + +#if MI_PADDING +#define mi_track_malloc(p,reqsize,zero) \ + if ((p)!=NULL) { \ + mi_assert_internal(mi_usable_size(p)==(reqsize)); \ + mi_track_malloc_size(p,reqsize,reqsize,zero); \ + } +#else +#define mi_track_malloc(p,reqsize,zero) \ + if ((p)!=NULL) { \ + mi_assert_internal(mi_usable_size(p)>=(reqsize)); \ + mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \ + } +#endif #endif diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index a75c9947..c24f5c9d 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -65,12 +65,14 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* mi_page_set_has_aligned(page, true); _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); } - // todo: expand padding if overallocated and p==aligned_p ? + // todo: expand padding if overallocated ? mi_assert_internal(mi_page_usable_block_size(_mi_ptr_page(p)) >= adjust + size); mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p), _mi_ptr_page(aligned_p), aligned_p)); mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); - + mi_assert_internal(mi_usable_size(aligned_p)>=size); + mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); + // now zero the block if needed if (zero && alignment > MI_ALIGNMENT_MAX) { const ptrdiff_t diff = (uint8_t*)aligned_p - (uint8_t*)p; @@ -83,8 +85,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* #if MI_TRACK_ENABLED if (p != aligned_p) { - mi_track_free(p); - mi_track_malloc(aligned_p, size, zero); + mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); } #endif return aligned_p; diff --git a/src/alloc.c b/src/alloc.c index 8ef78339..04c4c48c 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -102,19 +102,14 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, if (size == 0) { size = sizeof(void*); } #endif mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE); - void* p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE, zero); - #if MI_PADDING - mi_assert_internal(p == NULL || mi_usable_size(p) == size); - #else - mi_assert_internal(p == NULL || mi_usable_size(p) >= size); - #endif + void* const p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE, zero); + mi_track_malloc(p,size,zero); #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_get_default_heap(); } mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); } #endif - if (p!=NULL) { mi_track_malloc(p,size,zero); } return p; } @@ -137,18 +132,13 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic - #if MI_PADDING - mi_assert_internal(p == NULL || mi_usable_size(p) == size); - #else - mi_assert_internal(p == NULL || mi_usable_size(p) >= size); - #endif + mi_track_malloc(p,size,zero); #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_get_default_heap(); } mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); } #endif - if (p!=NULL) { mi_track_malloc(p,size,zero); } return p; } } @@ -499,7 +489,7 @@ mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* p void mi_decl_noinline _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept { mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p); mi_stat_free(page, block); // stat_free may access the padding - mi_track_free(p); + mi_track_free_size(block, mi_page_usable_size_of(page,block)); _mi_free_block(page, is_local, block); } @@ -559,7 +549,7 @@ void mi_free(void* p) mi_attr_noexcept #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif - mi_track_free_size(p, mi_page_usable_size_of(page,block)); // faster then mi_usable_size as we already now the page and that p is unaligned + mi_track_free_size(p, mi_page_usable_size_of(page,block)); // faster then mi_usable_size as we already know the page and that p is unaligned mi_block_set_next(page, block, page->local_free); page->local_free = block; if mi_unlikely(--page->used == 0) { // using this expression generates better code than: page->used--; if (mi_page_all_free(page)) @@ -695,8 +685,7 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) mi_assert_internal(p!=NULL); // todo: do not track as the usable size is still the same in the free; adjust potential padding? - // mi_track_free(p); - // mi_track_malloc(p,newsize,true); + // mi_track_resize(p,size,newsize) return p; // reallocation still fits and not more than 50% waste } void* newp = mi_heap_malloc(heap,newsize); From 82c85d1a130eb1e204177169476785728888c0ae Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 5 Mar 2023 18:03:04 -0800 Subject: [PATCH 017/102] fix valgrind mem for large alignment --- src/alloc-aligned.c | 17 +++++++---------- test/test-wrong.c | 24 +++++++++++++++++++++++- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index c24f5c9d..08ad9814 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -74,20 +74,17 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); // now zero the block if needed - if (zero && alignment > MI_ALIGNMENT_MAX) { - const ptrdiff_t diff = (uint8_t*)aligned_p - (uint8_t*)p; - ptrdiff_t zsize = mi_page_usable_block_size(_mi_ptr_page(p)) - diff - MI_PADDING_SIZE; - #if MI_PADDING - zsize -= MI_MAX_ALIGN_SIZE; - #endif - if (zsize > 0) { _mi_memzero(aligned_p, zsize); } + if (alignment > MI_ALIGNMENT_MAX) { + // for the tracker, on huge aligned allocations only from the start of the large block is defined + mi_track_mem_undefined(aligned_p, size); + if (zero) { + _mi_memzero(aligned_p, mi_usable_size(aligned_p)); + } } - #if MI_TRACK_ENABLED if (p != aligned_p) { mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); - } - #endif + } return aligned_p; } diff --git a/test/test-wrong.c b/test/test-wrong.c index 17d253b6..aaaf60b9 100644 --- a/test/test-wrong.c +++ b/test/test-wrong.c @@ -5,7 +5,10 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -/* test file for valgrind support. +/* test file for valgrind/asan support. + + VALGRIND: + ---------- Compile in an "out/debug" folder: > cd out/debug @@ -19,6 +22,25 @@ terms of the MIT license. A copy of the license can be found in the file and test as: > valgrind ./test-wrong + + + ASAN + ---------- + Compile in an "out/debug" folder: + + > cd out/debug + > cmake ../.. -DMI_ASAN=1 + > make -j8 + + and then compile this file as: + + > clang -g -o test-wrong -I../../include ../../test/test-wrong.c libmimalloc-asan-debug.a -lpthread -fsanitize=address -fsanitize-recover=address + + and test as: + + > ASAN_OPTIONS=verbosity=1:halt_on_error=0 ./test-wrong + + */ #include #include From 465eb81d30187ad9139141d830fb1753ebb3742d Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 5 Mar 2023 18:18:41 -0800 Subject: [PATCH 018/102] track free blocks in valgrind for heap_destroy as well --- include/mimalloc-track.h | 15 +++++++++------ src/heap.c | 11 +++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/include/mimalloc-track.h b/include/mimalloc-track.h index 8326c620..b2404f8d 100644 --- a/include/mimalloc-track.h +++ b/include/mimalloc-track.h @@ -24,8 +24,9 @@ uses the block start pointer and original size (corresponding to the `mi_track_m #if MI_VALGRIND -#define MI_TRACK_ENABLED 1 -#define MI_TRACK_TOOL "valgrind" +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on heap_destroy +#define MI_TRACK_TOOL "valgrind" #include #include @@ -39,8 +40,9 @@ uses the block start pointer and original size (corresponding to the `mi_track_m #elif MI_ASAN -#define MI_TRACK_ENABLED 1 -#define MI_TRACK_TOOL "asan" +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_TOOL "asan" #include @@ -52,8 +54,9 @@ uses the block start pointer and original size (corresponding to the `mi_track_m #else -#define MI_TRACK_ENABLED 0 -#define MI_TRACK_TOOL "none" +#define MI_TRACK_ENABLED 0 +#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_TOOL "none" #define mi_track_malloc_size(p,reqsize,size,zero) #define mi_track_free_size(p,_size) diff --git a/src/heap.c b/src/heap.c index 0ed0ab2c..6fe1fb17 100644 --- a/src/heap.c +++ b/src/heap.c @@ -8,6 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" +#include "mimalloc-track.h" #include // memset, memcpy @@ -310,6 +311,12 @@ void _mi_heap_destroy_pages(mi_heap_t* heap) { mi_heap_reset_pages(heap); } +static bool mi_cdecl mi_heap_track_block_free(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) { + MI_UNUSED(heap); MI_UNUSED(area); MI_UNUSED(arg); MI_UNUSED(block_size); + mi_track_free_size(block,mi_usable_size(block)); + return true; +} + void mi_heap_destroy(mi_heap_t* heap) { mi_assert(heap != NULL); mi_assert(mi_heap_is_initialized(heap)); @@ -321,6 +328,10 @@ void mi_heap_destroy(mi_heap_t* heap) { mi_heap_delete(heap); } else { + // track all blocks as freed + #if MI_TRACK_HEAP_DESTROY + mi_heap_visit_blocks(heap, true, mi_heap_track_block_free, NULL); + #endif // free all pages _mi_heap_destroy_pages(heap); mi_heap_free(heap); From 6f31115c7f5e62a33eb8fe99888591bc6c045173 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 5 Mar 2023 22:11:42 -0800 Subject: [PATCH 019/102] fix segment defined memory for valgrind --- src/segment.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/segment.c b/src/segment.c index 57ba7068..2698d578 100644 --- a/src/segment.c +++ b/src/segment.c @@ -874,11 +874,18 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi if (segment == NULL) return NULL; // zero the segment info? -- not always needed as it may be zero initialized from the OS + mi_track_mem_defined(segment, offsetof(mi_segment_t, next)); // needed for valgrind mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, NULL); // tsan - if (!is_zero) { + { ptrdiff_t ofs = offsetof(mi_segment_t, next); size_t prefix = offsetof(mi_segment_t, slices) - ofs; - memset((uint8_t*)segment+ofs, 0, prefix + sizeof(mi_slice_t)*(segment_slices+1)); // one more + size_t zsize = prefix + sizeof(mi_slice_t) * (segment_slices + 1); // one more + if (!is_zero) { + memset((uint8_t*)segment + ofs, 0, zsize); + } + else { + mi_track_mem_defined((uint8_t*)segment + ofs, zsize); // todo: somehow needed for valgrind? + } } segment->commit_mask = commit_mask; // on lazy commit, the initial part is always committed From b3f3a0de3b34ab0650228f4341c347f88c123420 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 5 Mar 2023 22:22:36 -0800 Subject: [PATCH 020/102] include psapi.h instead of defining PROCESS_MEMORY_COUNTERS on windows --- src/heap.c | 2 ++ src/stats.c | 14 +------------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/heap.c b/src/heap.c index 6fe1fb17..94fed2b5 100644 --- a/src/heap.c +++ b/src/heap.c @@ -311,11 +311,13 @@ void _mi_heap_destroy_pages(mi_heap_t* heap) { mi_heap_reset_pages(heap); } +#if MI_TRACK_HEAP_DESTROY static bool mi_cdecl mi_heap_track_block_free(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) { MI_UNUSED(heap); MI_UNUSED(area); MI_UNUSED(arg); MI_UNUSED(block_size); mi_track_free_size(block,mi_usable_size(block)); return true; } +#endif void mi_heap_destroy(mi_heap_t* heap) { mi_assert(heap != NULL); diff --git a/src/stats.c b/src/stats.c index 363c4400..84d677fa 100644 --- a/src/stats.c +++ b/src/stats.c @@ -465,6 +465,7 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) { #if defined(_WIN32) #include +#include static mi_msecs_t filetime_msecs(const FILETIME* ftime) { ULARGE_INTEGER i; @@ -474,19 +475,6 @@ static mi_msecs_t filetime_msecs(const FILETIME* ftime) { return msecs; } -typedef struct _PROCESS_MEMORY_COUNTERS { - DWORD cb; - DWORD PageFaultCount; - SIZE_T PeakWorkingSetSize; - SIZE_T WorkingSetSize; - SIZE_T QuotaPeakPagedPoolUsage; - SIZE_T QuotaPagedPoolUsage; - SIZE_T QuotaPeakNonPagedPoolUsage; - SIZE_T QuotaNonPagedPoolUsage; - SIZE_T PagefileUsage; - SIZE_T PeakPagefileUsage; -} PROCESS_MEMORY_COUNTERS; -typedef PROCESS_MEMORY_COUNTERS* PPROCESS_MEMORY_COUNTERS; typedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD); static PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL; From e912697d90fef95ff9004ee4a9543d333dbb128c Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 5 Mar 2023 22:26:05 -0800 Subject: [PATCH 021/102] fix warning with zero padding --- src/page.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page.c b/src/page.c index 70cb0d89..3205f975 100644 --- a/src/page.c +++ b/src/page.c @@ -856,7 +856,7 @@ static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size, size_t huge_alignme } else { // otherwise find a page with free blocks in our size segregated queues - mi_assert_internal(size >= MI_PADDING_SIZE); + mi_assert_internal((ptrdiff_t)size >= MI_PADDING_SIZE); // cast to signed to avoid error if there is no padding return mi_find_free_page(heap, size); } } From 64fb009695a7e659059d6ea8c36bb23cee141193 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 5 Mar 2023 22:27:45 -0800 Subject: [PATCH 022/102] fix warning with zero padding --- src/page.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/page.c b/src/page.c index 3205f975..46dfd26f 100644 --- a/src/page.c +++ b/src/page.c @@ -856,7 +856,9 @@ static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size, size_t huge_alignme } else { // otherwise find a page with free blocks in our size segregated queues - mi_assert_internal((ptrdiff_t)size >= MI_PADDING_SIZE); // cast to signed to avoid error if there is no padding + #if MI_PADDING + mi_assert_internal(size >= MI_PADDING_SIZE); + #endif return mi_find_free_page(heap, size); } } From 7ec798e19726d4314b90c61c68202457a380b1fa Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 5 Mar 2023 22:54:10 -0800 Subject: [PATCH 023/102] make test-stress match the one in dev --- test/test-stress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-stress.c b/test/test-stress.c index 8b96a5ae..69650556 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -91,7 +91,7 @@ static bool chance(size_t perc, random_t r) { static void* alloc_items(size_t items, random_t r) { if (chance(1, r)) { - if (chance(1, r) && allow_large_objects) items *= 50000; // 0.01% giant + if (chance(1, r) && allow_large_objects) items *= 10000; // 0.01% giant else if (chance(10, r) && allow_large_objects) items *= 1000; // 0.1% huge else items *= 100; // 1% large objects; } From 2e6ab0f23033bfca96b51e15b9567b07417c9268 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 6 Mar 2023 09:02:38 -0800 Subject: [PATCH 024/102] add documentation for tracking tools; rename with prefix MI_TRACK_tool --- CMakeLists.txt | 47 ++++++++++++++++++--------------------- include/mimalloc-track.h | 44 +++++++++++++++++++++++++++--------- include/mimalloc-types.h | 7 +++--- include/mimalloc.h | 2 +- readme.md | 48 +++++++++++++++++++++++++++++++++------- test/test-api-fill.c | 4 ++-- test/test-wrong.c | 4 ++-- 7 files changed, 104 insertions(+), 52 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b6ed554..26f5eed8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,8 +10,8 @@ option(MI_PADDING "Enable padding to detect heap block overflow (alway option(MI_OVERRIDE "Override the standard malloc interface (e.g. define entry points for malloc() etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) -option(MI_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) -option(MI_ASAN "Compile with address sanitizer support (adds a small overhead)" OFF) +option(MI_TRACK_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) +option(MI_TRACK_ASAN "Compile with address sanitizer support (adds a small overhead)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) @@ -25,7 +25,7 @@ option(MI_BUILD_TESTS "Build test executables" ON) option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) option(MI_SKIP_COLLECT_ON_EXIT, "Skip collecting memory on program exit" OFF) -option(MI_NO_PADDING "Force no use of padding even in DEBUG mode ets." OFF) +option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) # deprecated options option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) @@ -125,42 +125,39 @@ endif() if(MI_SECURE) message(STATUS "Set full secure build (MI_SECURE=ON)") - list(APPEND mi_defines MI_SECURE=4) - #if (MI_VALGRIND) - # message(WARNING "Secure mode is a bit weakened when compiling with Valgrind support as buffer overflow detection is no longer byte-precise (if running without valgrind)") - #endif() + list(APPEND mi_defines MI_SECURE=4) endif() -if(MI_VALGRIND) +if(MI_TRACK_VALGRIND) CHECK_INCLUDE_FILES("valgrind/valgrind.h;valgrind/memcheck.h" MI_HAS_VALGRINDH) if (NOT MI_HAS_VALGRINDH) - set(MI_VALGRIND OFF) + set(MI_TRACK_VALGRIND OFF) message(WARNING "Cannot find the 'valgrind/valgrind.h' and 'valgrind/memcheck.h' -- install valgrind first") - message(STATUS "Compile **without** Valgrind support (MI_VALGRIND=OFF)") + message(STATUS "Compile **without** Valgrind support (MI_TRACK_VALGRIND=OFF)") else() - message(STATUS "Compile with Valgrind support (MI_VALGRIND=ON)") - list(APPEND mi_defines MI_VALGRIND=1) + message(STATUS "Compile with Valgrind support (MI_TRACK_VALGRIND=ON)") + list(APPEND mi_defines MI_TRACK_VALGRIND=1) endif() endif() -if(MI_ASAN) +if(MI_TRACK_ASAN) if (APPLE AND MI_OVERRIDE) - set(MI_ASAN OFF) - message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_ASAN=OFF)") + set(MI_TRACK_ASAN OFF) + message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_TRACK_ASAN=OFF)") endif() - if (MI_VALGRIND) - set(MI_ASAN OFF) - message(WARNING "Cannot enable address sanitizer support with also Valgrind support enabled (MI_ASAN=OFF)") + if (MI_TRACK_VALGRIND) + set(MI_TRACK_ASAN OFF) + message(WARNING "Cannot enable address sanitizer support with also Valgrind support enabled (MI_TRACK_ASAN=OFF)") endif() - if(MI_ASAN) + if(MI_TRACK_ASAN) CHECK_INCLUDE_FILES("sanitizer/asan_interface.h" MI_HAS_ASANH) if (NOT MI_HAS_ASANH) - set(MI_ASAN OFF) + set(MI_TRACK_ASAN OFF) message(WARNING "Cannot find the 'sanitizer/asan_interface.h' -- install address sanitizer support first") - message(STATUS "Compile **without** address sanitizer support (MI_ASAN=OFF)") + message(STATUS "Compile **without** address sanitizer support (MI_TRACK_ASAN=OFF)") else() - message(STATUS "Compile with address sanitizer support (MI_ASAN=ON)") - list(APPEND mi_defines MI_ASAN=1) + message(STATUS "Compile with address sanitizer support (MI_TRACK_ASAN=ON)") + list(APPEND mi_defines MI_TRACK_ASAN=1) list(APPEND mi_cflags -fsanitize=address) list(APPEND mi_libraries -fsanitize=address) endif() @@ -327,10 +324,10 @@ set(mi_basename "mimalloc") if(MI_SECURE) set(mi_basename "${mi_basename}-secure") endif() -if(MI_VALGRIND) +if(MI_TRACK_VALGRIND) set(mi_basename "${mi_basename}-valgrind") endif() -if(MI_ASAN) +if(MI_TRACK_ASAN) set(mi_basename "${mi_basename}-asan") endif() string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) diff --git a/include/mimalloc-track.h b/include/mimalloc-track.h index b2404f8d..272ca1b8 100644 --- a/include/mimalloc-track.h +++ b/include/mimalloc-track.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -9,20 +9,39 @@ terms of the MIT license. A copy of the license can be found in the file #define MIMALLOC_TRACK_H /* ------------------------------------------------------------------------------------------------------ -Track memory ranges with macros for tools like Valgrind -address sanitizer, or other memory checkers. +Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers. +These can be defined for tracking allocation: + + #define mi_track_malloc_size(p,reqsize,size,zero) + #define mi_track_free_size(p,_size) The macros are set up such that the size passed to `mi_track_free_size` -matches the size of the allocation, or the new size of a `mi_track_resize` (currently unused though). +always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`). +The `reqsize` is what the user requested, and `size >= reqsize`. +The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled, +or otherwise it is the usable block size which may be larger than the original request. +Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc). +The `zero` parameter is `true` if the allocated block is zero initialized. + +Optional: + + #define mi_track_align(p,alignedp,offset,size) + #define mi_track_resize(p,oldsize,newsize) + +The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block. +The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`). +The `mi_track_resize` is currently unused but could be called on reallocations within a block. + +The following macros are for tools like asan and valgrind to track whether memory is +defined, undefined, or not accessible at all: + + #define mi_track_mem_defined(p,size) + #define mi_track_mem_undefined(p,size) + #define mi_track_mem_noaccess(p,size) -The `size` is either byte precise (and what the user requested) if `MI_PADDING` is enabled, -or otherwise it is the full block size which may be larger than the original request. -Aligned pointers in a block are signaled right after a `mi_track_malloc` -with the `mi_track_align` macro. The corresponding `mi_track_free` still -uses the block start pointer and original size (corresponding to the `mi_track_malloc`). -------------------------------------------------------------------------------------------------------*/ -#if MI_VALGRIND +#if MI_TRACK_VALGRIND #define MI_TRACK_ENABLED 1 #define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on heap_destroy @@ -38,7 +57,7 @@ uses the block start pointer and original size (corresponding to the `mi_track_m #define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size) #define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) -#elif MI_ASAN +#elif MI_TRACK_ASAN #define MI_TRACK_ENABLED 1 #define MI_TRACK_HEAP_DESTROY 0 @@ -68,6 +87,9 @@ uses the block start pointer and original size (corresponding to the `mi_track_m #endif +// ------------------- +// Utility definitions + #ifndef mi_track_resize #define mi_track_resize(p,oldsize,newsize) mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false) #endif diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index e365a8f5..9b3f5972 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -29,8 +29,9 @@ terms of the MIT license. A copy of the license can be found in the file // Define NDEBUG in the release version to disable assertions. // #define NDEBUG -// Define MI_VALGRIND to enable valgrind support -// #define MI_VALGRIND 1 +// Define MI_TRACK_ to enable tracking support +// #define MI_TRACK_VALGRIND 1 +// #define MI_TRACK_ASAN 1 // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). // #define MI_STAT 1 @@ -59,7 +60,7 @@ terms of the MIT license. A copy of the license can be found in the file // Reserve extra padding at the end of each block to be more resilient against heap block overflows. // The padding can detect buffer overflow on free. -#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || MI_VALGRIND || MI_ASAN) +#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || MI_TRACK_VALGRIND || MI_TRACK_ASAN) #define MI_PADDING 1 #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index c13bda23..27f6b331 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2022, Microsoft Research, Daan Leijen +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/readme.md b/readme.md index 364b974b..b102a50f 100644 --- a/readme.md +++ b/readme.md @@ -78,7 +78,7 @@ Note: the `v2.x` version has a new algorithm for managing internal mimalloc page and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance (see [below](#performance)); please report if you observe any significant performance regression. -* 2022-12-23, `v1.7.9`, `v2.0.9`: Supports building with asan and improved [Valgrind] support. Support abitrary large +* 2022-12-23, `v1.7.9`, `v2.0.9`: Supports building with [#asan] and improved [#Valgrind] support. Support abitrary large alignments (in particular for `std::pmr` pools). Added C++ STL allocators attached to a specific heap (thanks @vmarkovtsev). Heap walks now visit all object (including huge objects). Support Windows nano server containers (by Johannes Schindelin,@dscho). @@ -347,16 +347,19 @@ When _mimalloc_ is built using debug mode, various checks are done at runtime to - Double free's, and freeing invalid heap pointers are detected. - Corrupted free-lists and some forms of use-after-free are detected. -## Valgrind +## Tools -Generally, we recommend using the standard allocator with the amazing [Valgrind] tool (and -also for other address sanitizers). -However, it is possible to build mimalloc with Valgrind support. This has a small performance -overhead but does allow detecting memory leaks and byte-precise buffer overflows directly on final -executables. To build with valgrind support, use the `MI_VALGRIND=ON` cmake option: +Generally, we recommend using the standard allocator with memory tracking tools, but mimalloc +can also be build to support the [address sanitizer][asan] or the excellent [Valgrind] tool. +This has a small performance overhead but does allow detecting memory leaks and byte-precise +buffer overflows directly on final executables. See also the `test/test-wrong.c` file to test with various tools. + +### Valgrind + +To build with valgrind support, use the `MI_TRACK_VALGRIND=ON` cmake option: ``` -> cmake ../.. -DMI_VALGRIND=ON +> cmake ../.. -DMI_TRACK_VALGRIND=ON ``` This can also be combined with secure mode or debug mode. @@ -385,6 +388,35 @@ Valgrind support is in its initial development -- please report any issues. [Valgrind]: https://valgrind.org/ [valgrind-soname]: https://valgrind.org/docs/manual/manual-core.html#opt.soname-synonyms +### ASAN + +To build with the address sanitizer, use the `-DMI_TRACK_ASAN=ON` cmake option: + +``` +> cmake ../.. -DMI_TRACK_ASAN=ON +``` + +This can also be combined with secure mode or debug mode. +You can then run your programs as:' + +``` +> ASAN_OPTIONS=verbosity=1 +``` + +When you link a program with an address sanitizer build of mimalloc, you should +generally compile that program too with the address sanitizer enabled. +For example, assuming you build mimalloc in `out/debug`: + +``` +clang -g -o test-wrong -Iinclude test/test-wrong.c out/debug/libmimalloc-asan-debug.a -lpthread -fsanitize=address -fsanitize-recover=address +``` + +Since the address sanitizer redirects the standard allocation functions, on some platforms (macOSX for example) +it is required to compile mimalloc with `-DMI_OVERRIDE=OFF`. +Adress sanitizer support is in its initial development -- please report any issues. + +[asan]: https://github.com/google/sanitizers/wiki/AddressSanitizer + # Overriding Standard Malloc diff --git a/test/test-api-fill.c b/test/test-api-fill.c index 85d8524f..a32dfa27 100644 --- a/test/test-api-fill.c +++ b/test/test-api-fill.c @@ -309,7 +309,7 @@ bool check_zero_init(uint8_t* p, size_t size) { #if MI_DEBUG >= 2 bool check_debug_fill_uninit(uint8_t* p, size_t size) { -#if MI_VALGRIND +#if MI_TRACK_VALGRIND (void)p; (void)size; return true; // when compiled with valgrind we don't init on purpose #else @@ -325,7 +325,7 @@ bool check_debug_fill_uninit(uint8_t* p, size_t size) { } bool check_debug_fill_freed(uint8_t* p, size_t size) { -#if MI_VALGRIND +#if MI_TRACK_VALGRIND (void)p; (void)size; return true; // when compiled with valgrind we don't fill on purpose #else diff --git a/test/test-wrong.c b/test/test-wrong.c index aaaf60b9..56a2339a 100644 --- a/test/test-wrong.c +++ b/test/test-wrong.c @@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file Compile in an "out/debug" folder: > cd out/debug - > cmake ../.. -DMI_VALGRIND=1 + > cmake ../.. -DMI_TRACK_VALGRIND=1 > make -j8 and then compile this file as: @@ -29,7 +29,7 @@ terms of the MIT license. A copy of the license can be found in the file Compile in an "out/debug" folder: > cd out/debug - > cmake ../.. -DMI_ASAN=1 + > cmake ../.. -DMI_TRACK_ASAN=1 > make -j8 and then compile this file as: From a90737a7fa445b3a1afcb899c162cf670bc473fb Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 6 Mar 2023 10:44:43 -0800 Subject: [PATCH 025/102] fix valgrind tracking for zero initialized segments --- src/segment.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/segment.c b/src/segment.c index 2698d578..78171907 100644 --- a/src/segment.c +++ b/src/segment.c @@ -796,8 +796,6 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment const size_t extra = align_offset - info_size; // recalculate due to potential guard pages *psegment_slices = mi_segment_calculate_slices(required + extra, ppre_size, pinfo_slices); - //segment_size += _mi_align_up(align_offset - info_size, MI_SEGMENT_SLICE_SIZE); - //segment_slices = segment_size / MI_SEGMENT_SLICE_SIZE; } const size_t segment_size = (*psegment_slices) * MI_SEGMENT_SLICE_SIZE; mi_segment_t* segment = NULL; @@ -831,7 +829,10 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment if (!ok) return NULL; // failed to commit mi_commit_mask_set(pcommit_mask, &commit_needed_mask); } - mi_track_mem_undefined(segment,commit_needed*MI_COMMIT_SIZE); + else if (*is_zero) { + // track zero initialization for valgrind + mi_track_mem_defined(segment, commit_needed * MI_COMMIT_SIZE); + } segment->memid = memid; segment->mem_is_pinned = is_pinned; segment->mem_is_large = mem_large; @@ -874,18 +875,14 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi if (segment == NULL) return NULL; // zero the segment info? -- not always needed as it may be zero initialized from the OS - mi_track_mem_defined(segment, offsetof(mi_segment_t, next)); // needed for valgrind mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, NULL); // tsan { - ptrdiff_t ofs = offsetof(mi_segment_t, next); + ptrdiff_t ofs = offsetof(mi_segment_t, next); size_t prefix = offsetof(mi_segment_t, slices) - ofs; - size_t zsize = prefix + sizeof(mi_slice_t) * (segment_slices + 1); // one more + size_t zsize = prefix + (sizeof(mi_slice_t) * (segment_slices + 1)); // one more if (!is_zero) { memset((uint8_t*)segment + ofs, 0, zsize); - } - else { - mi_track_mem_defined((uint8_t*)segment + ofs, zsize); // todo: somehow needed for valgrind? - } + } } segment->commit_mask = commit_mask; // on lazy commit, the initial part is always committed From 08a01d26dc079756c8e94409fba051fd2eb5bd2c Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 14 Mar 2023 16:54:46 -0700 Subject: [PATCH 026/102] initial commit of new primitive layer --- CMakeLists.txt | 3 +- ide/vs2022/mimalloc-override.vcxproj | 1 + ide/vs2022/mimalloc.vcxproj | 1 + include/mimalloc-internal.h | 4 +- src/os.c | 1085 +++----------------------- src/prim/prim-unix.c | 483 ++++++++++++ src/prim/prim-wasi.c | 154 ++++ src/prim/prim-windows.c | 385 +++++++++ src/prim/prim.c | 18 + src/prim/prim.h | 64 ++ src/prim/readme.md | 6 + 11 files changed, 1221 insertions(+), 983 deletions(-) create mode 100644 src/prim/prim-unix.c create mode 100644 src/prim/prim-wasi.c create mode 100644 src/prim/prim-windows.c create mode 100644 src/prim/prim.c create mode 100644 src/prim/prim.h create mode 100644 src/prim/readme.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 26f5eed8..28dfe830 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,7 +50,8 @@ set(mi_sources src/alloc-posix.c src/heap.c src/options.c - src/init.c) + src/init.c + src/prim/prim.c) set(mi_cflags "") set(mi_libraries "") diff --git a/ide/vs2022/mimalloc-override.vcxproj b/ide/vs2022/mimalloc-override.vcxproj index e7133af4..a1d25d28 100644 --- a/ide/vs2022/mimalloc-override.vcxproj +++ b/ide/vs2022/mimalloc-override.vcxproj @@ -237,6 +237,7 @@ + diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index 9081881c..335125c1 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -225,6 +225,7 @@ + diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index b3cdf716..ee26bfb8 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -86,7 +86,9 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats); void* _mi_os_alloc_aligned_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool* large, mi_stats_t* tld_stats); void _mi_os_free_aligned(void* p, size_t size, size_t alignment, size_t align_offset, bool was_committed, mi_stats_t* tld_stats); - +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); +bool _mi_os_use_large_page(size_t size, size_t alignment); +size_t _mi_os_large_page_size(void); // memory.c void* _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t offset, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* id, mi_os_tld_t* tld); diff --git a/src/os.c b/src/os.c index 5277e5e4..8ef72e04 100644 --- a/src/os.c +++ b/src/os.c @@ -1,72 +1,69 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#ifndef _DEFAULT_SOURCE -#define _DEFAULT_SOURCE // ensure mmap flags are defined -#endif - -#if defined(__sun) -// illumos provides new mman.h api when any of these are defined -// otherwise the old api based on caddr_t which predates the void pointers one. -// stock solaris provides only the former, chose to atomically to discard those -// flags only here rather than project wide tough. -#undef _XOPEN_SOURCE -#undef _POSIX_C_SOURCE -#endif #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" +#include "prim/prim.h" -#include // strerror - -#ifdef _MSC_VER -#pragma warning(disable:4996) // strerror -#endif - -#if defined(__wasi__) -#define MI_USE_SBRK -#endif - -#if defined(_WIN32) -#include -#elif defined(__wasi__) -#include // sbrk -#else -#include // mmap -#include // sysconf -#if defined(__linux__) -#include -#include -#if defined(__GLIBC__) -#include // linux mmap flags -#else -#include -#endif -#endif -#if defined(__APPLE__) -#include -#if !TARGET_IOS_IPHONE && !TARGET_IOS_SIMULATOR -#include -#endif -#endif -#if defined(__FreeBSD__) || defined(__DragonFly__) -#include -#if __FreeBSD_version >= 1200000 -#include -#include -#endif -#include -#endif -#endif /* ----------------------------------------------------------- Initialization. On windows initializes support for aligned allocation and large OS pages (if MIMALLOC_LARGE_OS_PAGES is true). ----------------------------------------------------------- */ + +static mi_os_mem_config_t mi_os_mem_config = { + 4096, // page size + 0, // large page size (usually 2MiB) + 4096, // allocation granularity + true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems) + false // must free whole? +}; + +bool _mi_os_has_overcommit(void) { + return mi_os_mem_config.has_overcommit; +} + +// OS (small) page size +size_t _mi_os_page_size(void) { + return mi_os_mem_config.page_size; +} + +// if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB) +size_t _mi_os_large_page_size(void) { + return (mi_os_mem_config.large_page_size != 0 ? mi_os_mem_config.large_page_size : _mi_os_page_size()); +} + +bool _mi_os_use_large_page(size_t size, size_t alignment) { + // if we have access, check the size and alignment requirements + if (mi_os_mem_config.large_page_size == 0 || !mi_option_is_enabled(mi_option_large_os_pages)) return false; + return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0); +} + +// round to a good OS allocation size (bounded by max 12.5% waste) +size_t _mi_os_good_alloc_size(size_t size) { + size_t align_size; + if (size < 512*MI_KiB) align_size = _mi_os_page_size(); + else if (size < 2*MI_MiB) align_size = 64*MI_KiB; + else if (size < 8*MI_MiB) align_size = 256*MI_KiB; + else if (size < 32*MI_MiB) align_size = 1*MI_MiB; + else align_size = 4*MI_MiB; + if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow? + return _mi_align_up(size, align_size); +} + +void _mi_os_init(void) { + _mi_prim_mem_init(&mi_os_mem_config); +} + + +/* ----------------------------------------------------------- + Util +-------------------------------------------------------------- */ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats); @@ -90,226 +87,6 @@ static void* mi_align_down_ptr(void* p, size_t alignment) { } -// page size (initialized properly in `os_init`) -static size_t os_page_size = 4096; - -// minimal allocation granularity -static size_t os_alloc_granularity = 4096; - -// if non-zero, use large page allocation -static size_t large_os_page_size = 0; - -// is memory overcommit allowed? -// set dynamically in _mi_os_init (and if true we use MAP_NORESERVE) -static bool os_overcommit = true; - -bool _mi_os_has_overcommit(void) { - return os_overcommit; -} - -// OS (small) page size -size_t _mi_os_page_size(void) { - return os_page_size; -} - -// if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB) -size_t _mi_os_large_page_size(void) { - return (large_os_page_size != 0 ? large_os_page_size : _mi_os_page_size()); -} - -#if !defined(MI_USE_SBRK) && !defined(__wasi__) -static bool use_large_os_page(size_t size, size_t alignment) { - // if we have access, check the size and alignment requirements - if (large_os_page_size == 0 || !mi_option_is_enabled(mi_option_large_os_pages)) return false; - return ((size % large_os_page_size) == 0 && (alignment % large_os_page_size) == 0); -} -#endif - -// round to a good OS allocation size (bounded by max 12.5% waste) -size_t _mi_os_good_alloc_size(size_t size) { - size_t align_size; - if (size < 512*MI_KiB) align_size = _mi_os_page_size(); - else if (size < 2*MI_MiB) align_size = 64*MI_KiB; - else if (size < 8*MI_MiB) align_size = 256*MI_KiB; - else if (size < 32*MI_MiB) align_size = 1*MI_MiB; - else align_size = 4*MI_MiB; - if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow? - return _mi_align_up(size, align_size); -} - -#if defined(_WIN32) -// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016. -// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility) -// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB) -// We define a minimal MEM_EXTENDED_PARAMETER ourselves in order to be able to compile with older SDK's. -typedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E { - MiMemExtendedParameterInvalidType = 0, - MiMemExtendedParameterAddressRequirements, - MiMemExtendedParameterNumaNode, - MiMemExtendedParameterPartitionHandle, - MiMemExtendedParameterUserPhysicalHandle, - MiMemExtendedParameterAttributeFlags, - MiMemExtendedParameterMax -} MI_MEM_EXTENDED_PARAMETER_TYPE; - -typedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S { - struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type; - union { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg; -} MI_MEM_EXTENDED_PARAMETER; - -typedef struct MI_MEM_ADDRESS_REQUIREMENTS_S { - PVOID LowestStartingAddress; - PVOID HighestEndingAddress; - SIZE_T Alignment; -} MI_MEM_ADDRESS_REQUIREMENTS; - -#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE 0x00000010 - -#include -typedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); -typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); -static PVirtualAlloc2 pVirtualAlloc2 = NULL; -static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL; - -// Similarly, GetNumaProcesorNodeEx is only supported since Windows 7 -typedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER; - -typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber); -typedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber); -typedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask); -typedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber); -static PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL; -static PGetNumaProcessorNodeEx pGetNumaProcessorNodeEx = NULL; -static PGetNumaNodeProcessorMaskEx pGetNumaNodeProcessorMaskEx = NULL; -static PGetNumaProcessorNode pGetNumaProcessorNode = NULL; - -static bool mi_win_enable_large_os_pages(void) -{ - if (large_os_page_size > 0) return true; - - // Try to see if large OS pages are supported - // To use large pages on Windows, we first need access permission - // Set "Lock pages in memory" permission in the group policy editor - // - unsigned long err = 0; - HANDLE token = NULL; - BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token); - if (ok) { - TOKEN_PRIVILEGES tp; - ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid); - if (ok) { - tp.PrivilegeCount = 1; - tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; - ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0); - if (ok) { - err = GetLastError(); - ok = (err == ERROR_SUCCESS); - if (ok) { - large_os_page_size = GetLargePageMinimum(); - } - } - } - CloseHandle(token); - } - if (!ok) { - if (err == 0) err = GetLastError(); - _mi_warning_message("cannot enable large OS page support, error %lu\n", err); - } - return (ok!=0); -} - -void _mi_os_init(void) -{ - os_overcommit = false; - // get the page size - SYSTEM_INFO si; - GetSystemInfo(&si); - if (si.dwPageSize > 0) os_page_size = si.dwPageSize; - if (si.dwAllocationGranularity > 0) os_alloc_granularity = si.dwAllocationGranularity; - // get the VirtualAlloc2 function - HINSTANCE hDll; - hDll = LoadLibrary(TEXT("kernelbase.dll")); - if (hDll != NULL) { - // use VirtualAlloc2FromApp if possible as it is available to Windows store apps - pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp"); - if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2"); - FreeLibrary(hDll); - } - // NtAllocateVirtualMemoryEx is used for huge page allocation - hDll = LoadLibrary(TEXT("ntdll.dll")); - if (hDll != NULL) { - pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx"); - FreeLibrary(hDll); - } - // Try to use Win7+ numa API - hDll = LoadLibrary(TEXT("kernel32.dll")); - if (hDll != NULL) { - pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx"); - pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx"); - pGetNumaNodeProcessorMaskEx = (PGetNumaNodeProcessorMaskEx)(void (*)(void))GetProcAddress(hDll, "GetNumaNodeProcessorMaskEx"); - pGetNumaProcessorNode = (PGetNumaProcessorNode)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNode"); - FreeLibrary(hDll); - } - if (mi_option_is_enabled(mi_option_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { - mi_win_enable_large_os_pages(); - } -} -#elif defined(__wasi__) -void _mi_os_init(void) { - os_overcommit = false; - os_page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB - os_alloc_granularity = 16; -} - -#else // generic unix - -static void os_detect_overcommit(void) { -#if defined(__linux__) - int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); - if (fd < 0) return; - char buf[32]; - ssize_t nread = read(fd, &buf, sizeof(buf)); - close(fd); - // - // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) - if (nread >= 1) { - os_overcommit = (buf[0] == '0' || buf[0] == '1'); - } -#elif defined(__FreeBSD__) - int val = 0; - size_t olen = sizeof(val); - if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { - os_overcommit = (val != 0); - } -#else - // default: overcommit is true -#endif -} - -void _mi_os_init(void) { - // get the page size - long result = sysconf(_SC_PAGESIZE); - if (result > 0) { - os_page_size = (size_t)result; - os_alloc_granularity = os_page_size; - } - large_os_page_size = 2*MI_MiB; // TODO: can we query the OS for this? - os_detect_overcommit(); -} -#endif - - -#if defined(MADV_NORMAL) -static int mi_madvise(void* addr, size_t length, int advice) { - #if defined(__sun) - return madvise((caddr_t)addr, length, advice); // Solaris needs cast (issue #520) - #else - return madvise(addr, length, advice); - #endif -} -#endif - - /* ----------------------------------------------------------- aligned hinting -------------------------------------------------------------- */ @@ -330,7 +107,7 @@ static mi_decl_cache_align _Atomic(uintptr_t)aligned_base; #define MI_HINT_AREA ((uintptr_t)4 << 40) // upto 6TiB (since before win8 there is "only" 8TiB available to processes) #define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) -static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL; size = _mi_align_up(size, MI_SEGMENT_SIZE); @@ -354,354 +131,32 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) return (void*)hint; } #else -static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { MI_UNUSED(try_alignment); MI_UNUSED(size); return NULL; } #endif + /* ----------------------------------------------------------- Free memory -------------------------------------------------------------- */ -static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats_t* stats) +void _mi_os_free_ex(void* addr, size_t size, bool was_committed, mi_stats_t* tld_stats) { - if (addr == NULL || size == 0) return true; // || _mi_os_is_huge_reserved(addr) - bool err = false; -#if defined(_WIN32) - DWORD errcode = 0; - err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); - if (err) { errcode = GetLastError(); } - if (errcode == ERROR_INVALID_ADDRESS) { - // In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside - // the memory region returned by VirtualAlloc; in that case we need to free using - // the start of the region. - MEMORY_BASIC_INFORMATION info = { 0 }; - VirtualQuery(addr, &info, sizeof(info)); - if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)MI_SEGMENT_SIZE) { - errcode = 0; - err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0); - if (err) { errcode = GetLastError(); } - } - } - if (errcode != 0) { - _mi_warning_message("unable to release OS memory: error code 0x%x, addr: %p, size: %zu\n", errcode, addr, size); - } -#elif defined(MI_USE_SBRK) || defined(__wasi__) - err = false; // sbrk heap cannot be shrunk -#else - err = (munmap(addr, size) == -1); - if (err) { - _mi_warning_message("unable to release OS memory: %s, addr: %p, size: %zu\n", strerror(errno), addr, size); - } -#endif + MI_UNUSED(tld_stats); + mi_stats_t* stats = &_mi_stats_main; + if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) + const size_t csize = _mi_os_good_alloc_size(size); + _mi_prim_free(addr, csize); if (was_committed) { _mi_stat_decrease(&stats->committed, size); } _mi_stat_decrease(&stats->reserved, size); - return !err; } - -/* ----------------------------------------------------------- - Raw allocation on Windows (VirtualAlloc) --------------------------------------------------------------- */ - -#ifdef _WIN32 -static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { -#if (MI_INTPTR_SIZE >= 8) - // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations - if (addr == NULL) { - void* hint = mi_os_get_aligned_hint(try_alignment,size); - if (hint != NULL) { - void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); - if (p != NULL) return p; - _mi_verbose_message("warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), hint, try_alignment, flags); - // fall through on error - } - } -#endif - // on modern Windows try use VirtualAlloc2 for aligned allocation - if (try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { - MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 }; - reqs.Alignment = try_alignment; - MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} }; - param.Type.Type = MiMemExtendedParameterAddressRequirements; - param.Arg.Pointer = &reqs; - void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); - if (p != NULL) return p; - _mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags); - // fall through on error - } - // last resort - return VirtualAlloc(addr, size, flags, PAGE_READWRITE); +void _mi_os_free(void* p, size_t size, mi_stats_t* tld_stats) { + _mi_os_free_ex(p, size, true, tld_stats); } -static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) { - mi_assert_internal(!(large_only && !allow_large)); - static _Atomic(size_t) large_page_try_ok; // = 0; - void* p = NULL; - // Try to allocate large OS pages (2MiB) if allowed or required. - if ((large_only || use_large_os_page(size, try_alignment)) - && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) { - size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); - if (!large_only && try_ok > 0) { - // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. - // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times. - mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1); - } - else { - // large OS pages must always reserve and commit. - *is_large = true; - p = mi_win_virtual_allocx(addr, size, try_alignment, flags | MEM_LARGE_PAGES); - if (large_only) return p; - // fall back to non-large page allocation on error (`p == NULL`). - if (p == NULL) { - mi_atomic_store_release(&large_page_try_ok,10UL); // on error, don't try again for the next N allocations - } - } - } - // Fall back to regular page allocation - if (p == NULL) { - *is_large = ((flags&MEM_LARGE_PAGES) != 0); - p = mi_win_virtual_allocx(addr, size, try_alignment, flags); - } - if (p == NULL) { - _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); - } - return p; -} - -/* ----------------------------------------------------------- - Raw allocation using `sbrk` or `wasm_memory_grow` --------------------------------------------------------------- */ - -#elif defined(MI_USE_SBRK) || defined(__wasi__) -#if defined(MI_USE_SBRK) - static void* mi_memory_grow( size_t size ) { - void* p = sbrk(size); - if (p == (void*)(-1)) return NULL; - #if !defined(__wasi__) // on wasi this is always zero initialized already (?) - memset(p,0,size); - #endif - return p; - } -#elif defined(__wasi__) - static void* mi_memory_grow( size_t size ) { - size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size())) - : __builtin_wasm_memory_size(0)); - if (base == SIZE_MAX) return NULL; - return (void*)(base * _mi_os_page_size()); - } -#endif - -#if defined(MI_USE_PTHREADS) -static pthread_mutex_t mi_heap_grow_mutex = PTHREAD_MUTEX_INITIALIZER; -#endif - -static void* mi_heap_grow(size_t size, size_t try_alignment) { - void* p = NULL; - if (try_alignment <= 1) { - // `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now) - #if defined(MI_USE_PTHREADS) - pthread_mutex_lock(&mi_heap_grow_mutex); - #endif - p = mi_memory_grow(size); - #if defined(MI_USE_PTHREADS) - pthread_mutex_unlock(&mi_heap_grow_mutex); - #endif - } - else { - void* base = NULL; - size_t alloc_size = 0; - // to allocate aligned use a lock to try to avoid thread interaction - // between getting the current size and actual allocation - // (also, `sbrk` is not thread safe in general) - #if defined(MI_USE_PTHREADS) - pthread_mutex_lock(&mi_heap_grow_mutex); - #endif - { - void* current = mi_memory_grow(0); // get current size - if (current != NULL) { - void* aligned_current = mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space - alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size()); - base = mi_memory_grow(alloc_size); - } - } - #if defined(MI_USE_PTHREADS) - pthread_mutex_unlock(&mi_heap_grow_mutex); - #endif - if (base != NULL) { - p = mi_align_up_ptr(base, try_alignment); - if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) { - // another thread used wasm_memory_grow/sbrk in-between and we do not have enough - // space after alignment. Give up (and waste the space as we cannot shrink :-( ) - // (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align) - p = NULL; - } - } - } - if (p == NULL) { - _mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment); - errno = ENOMEM; - return NULL; - } - mi_assert_internal( try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); - return p; -} - -/* ----------------------------------------------------------- - Raw allocation on Unix's (mmap) --------------------------------------------------------------- */ -#else -#define MI_OS_USE_MMAP -static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { - MI_UNUSED(try_alignment); - #if defined(MAP_ALIGNED) // BSD - if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { - size_t n = mi_bsr(try_alignment); - if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB - flags |= MAP_ALIGNED(n); - void* p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); - if (p!=MAP_FAILED) return p; - // fall back to regular mmap - } - } - #elif defined(MAP_ALIGN) // Solaris - if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { - void* p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); // addr parameter is the required alignment - if (p!=MAP_FAILED) return p; - // fall back to regular mmap - } - #endif - #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) - // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations - if (addr == NULL) { - void* hint = mi_os_get_aligned_hint(try_alignment, size); - if (hint != NULL) { - void* p = mmap(hint, size, protect_flags, flags, fd, 0); - if (p!=MAP_FAILED) return p; - // fall back to regular mmap - } - } - #endif - // regular mmap - void* p = mmap(addr, size, protect_flags, flags, fd, 0); - if (p!=MAP_FAILED) return p; - // failed to allocate - return NULL; -} - -static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { - void* p = NULL; - #if !defined(MAP_ANONYMOUS) - #define MAP_ANONYMOUS MAP_ANON - #endif - #if !defined(MAP_NORESERVE) - #define MAP_NORESERVE 0 - #endif - int flags = MAP_PRIVATE | MAP_ANONYMOUS; - int fd = -1; - if (_mi_os_has_overcommit()) { - flags |= MAP_NORESERVE; - } - #if defined(PROT_MAX) - protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD - #endif - #if defined(VM_MAKE_TAG) - // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) - int os_tag = (int)mi_option_get(mi_option_os_tag); - if (os_tag < 100 || os_tag > 255) { os_tag = 100; } - fd = VM_MAKE_TAG(os_tag); - #endif - // huge page allocation - if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) { - static _Atomic(size_t) large_page_try_ok; // = 0; - size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); - if (!large_only && try_ok > 0) { - // If the OS is not configured for large OS pages, or the user does not have - // enough permission, the `mmap` will always fail (but it might also fail for other reasons). - // Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times - // to avoid too many failing calls to mmap. - mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1); - } - else { - int lflags = flags & ~MAP_NORESERVE; // using NORESERVE on huge pages seems to fail on Linux - int lfd = fd; - #ifdef MAP_ALIGNED_SUPER - lflags |= MAP_ALIGNED_SUPER; - #endif - #ifdef MAP_HUGETLB - lflags |= MAP_HUGETLB; - #endif - #ifdef MAP_HUGE_1GB - static bool mi_huge_pages_available = true; - if ((size % MI_GiB) == 0 && mi_huge_pages_available) { - lflags |= MAP_HUGE_1GB; - } - else - #endif - { - #ifdef MAP_HUGE_2MB - lflags |= MAP_HUGE_2MB; - #endif - } - #ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB - lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB; - #endif - if (large_only || lflags != flags) { - // try large OS page allocation - *is_large = true; - p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd); - #ifdef MAP_HUGE_1GB - if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) { - mi_huge_pages_available = false; // don't try huge 1GiB pages again - _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (error %i)\n", errno); - lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB); - p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd); - } - #endif - if (large_only) return p; - if (p == NULL) { - mi_atomic_store_release(&large_page_try_ok, (size_t)8); // on error, don't try again for the next N allocations - } - } - } - } - // regular allocation - if (p == NULL) { - *is_large = false; - p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); - if (p != NULL) { - #if defined(MADV_HUGEPAGE) - // Many Linux systems don't allow MAP_HUGETLB but they support instead - // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE - // though since properly aligned allocations will already use large pages if available - // in that case -- in particular for our large regions (in `memory.c`). - // However, some systems only allow THP if called with explicit `madvise`, so - // when large OS pages are enabled for mimalloc, we call `madvise` anyways. - if (allow_large && use_large_os_page(size, try_alignment)) { - if (mi_madvise(p, size, MADV_HUGEPAGE) == 0) { - *is_large = true; // possibly - }; - } - #elif defined(__sun) - if (allow_large && use_large_os_page(size, try_alignment)) { - struct memcntl_mha cmd = {0}; - cmd.mha_pagesize = large_os_page_size; - cmd.mha_cmd = MHA_MAPSIZE_VA; - if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { - *is_large = true; - } - } - #endif - } - } - if (p == NULL) { - _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, errno, addr, large_only, allow_large); - } - return p; -} -#endif - /* ----------------------------------------------------------- Primitive allocation from the OS. @@ -714,7 +169,7 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo if (!commit) allow_large = false; if (try_alignment == 0) try_alignment = 1; // avoid 0 to ensure there will be no divide by zero when aligning - void* p = NULL; + void* p = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large); /* if (commit && allow_large) { p = _mi_os_try_alloc_from_huge_reserved(size, try_alignment); @@ -725,18 +180,6 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo } */ - #if defined(_WIN32) - int flags = MEM_RESERVE; - if (commit) { flags |= MEM_COMMIT; } - p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); - #elif defined(MI_USE_SBRK) || defined(__wasi__) - MI_UNUSED(allow_large); - *is_large = false; - p = mi_heap_grow(size, try_alignment); - #else - int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); - p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); - #endif mi_stat_counter_increase(stats->mmap_calls, 1); if (p != NULL) { _mi_stat_increase(&stats->reserved, size); @@ -762,40 +205,41 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, // if not aligned, free it, overallocate, and unmap around it if (((uintptr_t)p % alignment != 0)) { - mi_os_mem_free(p, size, commit, stats); + _mi_os_free_ex(p, size, commit, stats); _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (%zu bytes, address: %p, alignment: %zu, commit: %d)\n", size, p, alignment, commit); if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; -#if _WIN32 - // over-allocate uncommitted (virtual) memory - p = mi_os_mem_alloc(over_size, 0 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, stats); - if (p == NULL) return NULL; + if (mi_os_mem_config.must_free_whole) { // win32 virtualAlloc cannot free parts of an allocate block + // over-allocate uncommitted (virtual) memory + p = mi_os_mem_alloc(over_size, 0 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, stats); + if (p == NULL) return NULL; - // set p to the aligned part in the full region - // note: this is dangerous on Windows as VirtualFree needs the actual region pointer - // but in mi_os_mem_free we handle this (hopefully exceptional) situation. - p = mi_align_up_ptr(p, alignment); + // set p to the aligned part in the full region + // note: this is dangerous on Windows as VirtualFree needs the actual region pointer + // but in mi_os_mem_free we handle this (hopefully exceptional) situation. + p = mi_align_up_ptr(p, alignment); - // explicitly commit only the aligned part - if (commit) { - _mi_os_commit(p, size, NULL, stats); + // explicitly commit only the aligned part + if (commit) { + _mi_os_commit(p, size, NULL, stats); + } + } + else { // mmap can free inside an allocation + // overallocate... + p = mi_os_mem_alloc(over_size, 1, commit, false, is_large, stats); + if (p == NULL) return NULL; + // and selectively unmap parts around the over-allocated area. (noop on sbrk) + void* aligned_p = mi_align_up_ptr(p, alignment); + size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; + size_t mid_size = _mi_align_up(size, _mi_os_page_size()); + size_t post_size = over_size - pre_size - mid_size; + mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); + if (pre_size > 0) _mi_os_free_ex(p, pre_size, commit, stats); + if (post_size > 0) _mi_os_free_ex((uint8_t*)aligned_p + mid_size, post_size, commit, stats); + // we can return the aligned pointer on `mmap` (and sbrk) systems + p = aligned_p; } -#else - // overallocate... - p = mi_os_mem_alloc(over_size, 1, commit, false, is_large, stats); - if (p == NULL) return NULL; - // and selectively unmap parts around the over-allocated area. (noop on sbrk) - void* aligned_p = mi_align_up_ptr(p, alignment); - size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; - size_t mid_size = _mi_align_up(size, _mi_os_page_size()); - size_t post_size = over_size - pre_size - mid_size; - mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size); - if (pre_size > 0) mi_os_mem_free(p, pre_size, commit, stats); - if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); - // we can return the aligned pointer on `mmap` (and sbrk) systems - p = aligned_p; -#endif } mi_assert_internal(p == NULL || (p != NULL && ((uintptr_t)p % alignment) == 0)); @@ -804,7 +248,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, /* ----------------------------------------------------------- - OS API: alloc, free, alloc_aligned + OS API: alloc and alloc_aligned ----------------------------------------------------------- */ void* _mi_os_alloc(size_t size, mi_stats_t* tld_stats) { @@ -816,21 +260,9 @@ void* _mi_os_alloc(size_t size, mi_stats_t* tld_stats) { return mi_os_mem_alloc(size, 0, true, false, &is_large, stats); } -void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* tld_stats) { - MI_UNUSED(tld_stats); - mi_stats_t* stats = &_mi_stats_main; - if (size == 0 || p == NULL) return; - size = _mi_os_good_alloc_size(size); - mi_os_mem_free(p, size, was_committed, stats); -} - -void _mi_os_free(void* p, size_t size, mi_stats_t* stats) { - _mi_os_free_ex(p, size, true, stats); -} - void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_stats_t* tld_stats) { - MI_UNUSED(&mi_os_get_aligned_hint); // suppress unused warnings + MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings MI_UNUSED(tld_stats); if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); @@ -883,11 +315,11 @@ void _mi_os_free_aligned(void* p, size_t size, size_t alignment, size_t align_of _mi_os_free_ex(start, size + extra, was_committed, tld_stats); } + /* ----------------------------------------------------------- OS memory API: reset, commit, decommit, protect, unprotect. ----------------------------------------------------------- */ - // OS page align within a given area, either conservative (pages inside the area only), // or not (straddling pages outside the area is possible) static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) { @@ -912,18 +344,6 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* return mi_os_page_align_areax(true, addr, size, newsize); } -static void mi_mprotect_hint(int err) { -#if defined(MI_OS_USE_MMAP) && (MI_SECURE>=2) // guard page around every mimalloc page - if (err == ENOMEM) { - _mi_warning_message("the previous warning may have been caused by a low memory map limit.\n" - " On Linux this is controlled by the vm.max_map_count. For example:\n" - " > sudo sysctl -w vm.max_map_count=262144\n"); - } -#else - MI_UNUSED(err); -#endif -} - // Commit/Decommit memory. // Usually commit is aligned liberal, while decommit is aligned conservative. // (but not for the reset version where we want commit to be conservative as well) @@ -933,7 +353,6 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ size_t csize; void* start = mi_os_page_align_areax(conservative, addr, size, &csize); if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)) - int err = 0; if (commit) { _mi_stat_increase(&stats->committed, size); // use size for precise commit vs. decommit _mi_stat_counter_increase(&stats->commit_calls, 1); @@ -942,56 +361,9 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ _mi_stat_decrease(&stats->committed, size); } - #if defined(_WIN32) - if (commit) { - // *is_zero = true; // note: if the memory was already committed, the call succeeds but the memory is not zero'd - void* p = VirtualAlloc(start, csize, MEM_COMMIT, PAGE_READWRITE); - err = (p == start ? 0 : GetLastError()); - } - else { - BOOL ok = VirtualFree(start, csize, MEM_DECOMMIT); - err = (ok ? 0 : GetLastError()); - } - #elif defined(__wasi__) - // WebAssembly guests can't control memory protection - #elif 0 && defined(MAP_FIXED) && !defined(__APPLE__) - // Linux: disabled for now as mmap fixed seems much more expensive than MADV_DONTNEED (and splits VMA's?) - if (commit) { - // commit: just change the protection - err = mprotect(start, csize, (PROT_READ | PROT_WRITE)); - if (err != 0) { err = errno; } - } - else { - // decommit: use mmap with MAP_FIXED to discard the existing memory (and reduce rss) - const int fd = mi_unix_mmap_fd(); - void* p = mmap(start, csize, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0); - if (p != start) { err = errno; } - } - #else - // Linux, macOSX and others. - if (commit) { - // commit: ensure we can access the area - err = mprotect(start, csize, (PROT_READ | PROT_WRITE)); - if (err != 0) { err = errno; } - } - else { - #if defined(MADV_DONTNEED) && MI_DEBUG == 0 && MI_SECURE == 0 - // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) - // (on the other hand, MADV_FREE would be good enough.. it is just not reflected in the stats :-( ) - err = madvise(start, csize, MADV_DONTNEED); - #else - // decommit: just disable access (also used in debug and secure mode to trap on illegal access) - err = mprotect(start, csize, PROT_NONE); - if (err != 0) { err = errno; } - #endif - //#if defined(MADV_FREE_REUSE) - // while ((err = mi_madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } - //#endif - } - #endif + int err = _mi_prim_commit(start, csize, commit); if (err != 0) { _mi_warning_message("%s error: start: %p, csize: 0x%zx, err: %i\n", commit ? "commit" : "decommit", start, csize, err); - mi_mprotect_hint(err); } mi_assert_internal(err == 0); return (err == 0); @@ -1033,39 +405,11 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) } #endif -#if defined(_WIN32) - // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory - void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); - mi_assert_internal(p == start); - #if 1 - if (p == start && start != NULL) { - VirtualUnlock(start,csize); // VirtualUnlock after MEM_RESET removes the memory from the working set - } - #endif - if (p != start) return false; -#else -#if defined(MADV_FREE) - static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE); - int oadvice = (int)mi_atomic_load_relaxed(&advice); - int err; - while ((err = mi_madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; - if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { - // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on - mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED); - err = mi_madvise(start, csize, MADV_DONTNEED); - } -#elif defined(__wasi__) - int err = 0; -#else - int err = mi_madvise(start, csize, MADV_DONTNEED); -#endif + int err = _mi_prim_reset(start, csize); if (err != 0) { - _mi_warning_message("madvise reset error: start: %p, csize: 0x%zx, errno: %i\n", start, csize, errno); + _mi_warning_message("madvise reset error: start: %p, csize: 0x%zx, errno: %i\n", start, csize, err); } - //mi_assert(err == 0); - if (err != 0) return false; -#endif - return true; + return (err == 0); } // Signal to the OS that the address range is no longer in use @@ -1097,20 +441,9 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) { _mi_warning_message("cannot mprotect memory allocated in huge OS pages\n"); } */ - int err = 0; -#ifdef _WIN32 - DWORD oldprotect = 0; - BOOL ok = VirtualProtect(start, csize, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect); - err = (ok ? 0 : GetLastError()); -#elif defined(__wasi__) - err = 0; -#else - err = mprotect(start, csize, protect ? PROT_NONE : (PROT_READ | PROT_WRITE)); - if (err != 0) { err = errno; } -#endif + int err = _mi_prim_protect(start,csize,protect); if (err != 0) { _mi_warning_message("mprotect error: start: %p, csize: 0x%zx, err: %i\n", start, csize, err); - mi_mprotect_hint(err); } return (err == 0); } @@ -1125,115 +458,12 @@ bool _mi_os_unprotect(void* addr, size_t size) { -bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) { - // page align conservatively within the range - mi_assert_internal(oldsize > newsize && p != NULL); - if (oldsize < newsize || p == NULL) return false; - if (oldsize == newsize) return true; - - // oldsize and newsize should be page aligned or we cannot shrink precisely - void* addr = (uint8_t*)p + newsize; - size_t size = 0; - void* start = mi_os_page_align_area_conservative(addr, oldsize - newsize, &size); - if (size == 0 || start != addr) return false; - -#ifdef _WIN32 - // we cannot shrink on windows, but we can decommit - return _mi_os_decommit(start, size, stats); -#else - return mi_os_mem_free(start, size, true, stats); -#endif -} - - /* ---------------------------------------------------------------------------- Support for allocating huge OS pages (1Gib) that are reserved up-front and possibly associated with a specific NUMA node. (use `numa_node>=0`) -----------------------------------------------------------------------------*/ #define MI_HUGE_OS_PAGE_SIZE (MI_GiB) -#if defined(_WIN32) && (MI_INTPTR_SIZE >= 8) -static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) -{ - mi_assert_internal(size%MI_GiB == 0); - mi_assert_internal(addr != NULL); - const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE; - - mi_win_enable_large_os_pages(); - - MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} }; - // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages - static bool mi_huge_pages_available = true; - if (pNtAllocateVirtualMemoryEx != NULL && mi_huge_pages_available) { - params[0].Type.Type = MiMemExtendedParameterAttributeFlags; - params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE; - ULONG param_count = 1; - if (numa_node >= 0) { - param_count++; - params[1].Type.Type = MiMemExtendedParameterNumaNode; - params[1].Arg.ULong = (unsigned)numa_node; - } - SIZE_T psize = size; - void* base = addr; - NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count); - if (err == 0 && base != NULL) { - return base; - } - else { - // fall back to regular large pages - mi_huge_pages_available = false; // don't try further huge pages - _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err); - } - } - // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation - if (pVirtualAlloc2 != NULL && numa_node >= 0) { - params[0].Type.Type = MiMemExtendedParameterNumaNode; - params[0].Arg.ULong = (unsigned)numa_node; - return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1); - } - - // otherwise use regular virtual alloc on older windows - return VirtualAlloc(addr, size, flags, PAGE_READWRITE); -} - -#elif defined(MI_OS_USE_MMAP) && (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) -#include -#ifndef MPOL_PREFERRED -#define MPOL_PREFERRED 1 -#endif -#if defined(SYS_mbind) -static long mi_os_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { - return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags); -} -#else -static long mi_os_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { - MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags); - return 0; -} -#endif -static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { - mi_assert_internal(size%MI_GiB == 0); - bool is_large = true; - void* p = mi_unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); - if (p == NULL) return NULL; - if (numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes - unsigned long numa_mask = (1UL << numa_node); - // TODO: does `mbind` work correctly for huge OS pages? should we - // use `set_mempolicy` before calling mmap instead? - // see: - long err = mi_os_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); - if (err != 0) { - _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d: %s\n", numa_node, strerror(errno)); - } - } - return p; -} -#else -static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { - MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node); - return NULL; -} -#endif #if (MI_INTPTR_SIZE >= 8) // To ensure proper alignment, use our own area for huge OS pages @@ -1252,10 +482,10 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { if (start == 0) { // Initialize the start address after the 32TiB area start = ((uintptr_t)32 << 40); // 32TiB virtual start address -#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode + #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB -#endif + #endif } end = start + size; mi_assert_internal(end % MI_SEGMENT_SIZE == 0); @@ -1288,7 +518,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse for (page = 0; page < pages; page++) { // allocate a page void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE); - void* p = mi_os_alloc_huge_os_pagesx(addr, MI_HUGE_OS_PAGE_SIZE, numa_node); + void* p = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node); // Did we succeed at a contiguous address? if (p != addr) { @@ -1340,113 +570,6 @@ void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) { /* ---------------------------------------------------------------------------- Support NUMA aware allocation -----------------------------------------------------------------------------*/ -#ifdef _WIN32 -static size_t mi_os_numa_nodex(void) { - USHORT numa_node = 0; - if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) { - // Extended API is supported - MI_PROCESSOR_NUMBER pnum; - (*pGetCurrentProcessorNumberEx)(&pnum); - USHORT nnode = 0; - BOOL ok = (*pGetNumaProcessorNodeEx)(&pnum, &nnode); - if (ok) { numa_node = nnode; } - } - else if (pGetNumaProcessorNode != NULL) { - // Vista or earlier, use older API that is limited to 64 processors. Issue #277 - DWORD pnum = GetCurrentProcessorNumber(); - UCHAR nnode = 0; - BOOL ok = pGetNumaProcessorNode((UCHAR)pnum, &nnode); - if (ok) { numa_node = nnode; } - } - return numa_node; -} - -static size_t mi_os_numa_node_countx(void) { - ULONG numa_max = 0; - GetNumaHighestNodeNumber(&numa_max); - // find the highest node number that has actual processors assigned to it. Issue #282 - while(numa_max > 0) { - if (pGetNumaNodeProcessorMaskEx != NULL) { - // Extended API is supported - GROUP_AFFINITY affinity; - if ((*pGetNumaNodeProcessorMaskEx)((USHORT)numa_max, &affinity)) { - if (affinity.Mask != 0) break; // found the maximum non-empty node - } - } - else { - // Vista or earlier, use older API that is limited to 64 processors. - ULONGLONG mask; - if (GetNumaNodeProcessorMask((UCHAR)numa_max, &mask)) { - if (mask != 0) break; // found the maximum non-empty node - }; - } - // max node was invalid or had no processor assigned, try again - numa_max--; - } - return ((size_t)numa_max + 1); -} -#elif defined(__linux__) -#include // getcpu -#include // access - -static size_t mi_os_numa_nodex(void) { -#ifdef SYS_getcpu - unsigned long node = 0; - unsigned long ncpu = 0; - long err = syscall(SYS_getcpu, &ncpu, &node, NULL); - if (err != 0) return 0; - return node; -#else - return 0; -#endif -} -static size_t mi_os_numa_node_countx(void) { - char buf[128]; - unsigned node = 0; - for(node = 0; node < 256; node++) { - // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation) - snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1); - if (access(buf,R_OK) != 0) break; - } - return (node+1); -} -#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000 -static size_t mi_os_numa_nodex(void) { - domainset_t dom; - size_t node; - int policy; - if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul; - for (node = 0; node < MAXMEMDOM; node++) { - if (DOMAINSET_ISSET(node, &dom)) return node; - } - return 0ul; -} -static size_t mi_os_numa_node_countx(void) { - size_t ndomains = 0; - size_t len = sizeof(ndomains); - if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul; - return ndomains; -} -#elif defined(__DragonFly__) -static size_t mi_os_numa_nodex(void) { - // TODO: DragonFly does not seem to provide any userland means to get this information. - return 0ul; -} -static size_t mi_os_numa_node_countx(void) { - size_t ncpus = 0, nvirtcoresperphys = 0; - size_t len = sizeof(size_t); - if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul; - if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul; - return nvirtcoresperphys * ncpus; -} -#else -static size_t mi_os_numa_nodex(void) { - return 0; -} -static size_t mi_os_numa_node_countx(void) { - return 1; -} -#endif _Atomic(size_t) _mi_numa_node_count; // = 0 // cache the node count @@ -1458,7 +581,7 @@ size_t _mi_os_numa_node_count_get(void) { count = (size_t)ncount; } else { - count = mi_os_numa_node_countx(); // or detect dynamically + count = _mi_prim_numa_node_count(); // or detect dynamically if (count == 0) count = 1; } mi_atomic_store_release(&_mi_numa_node_count, count); // save it @@ -1472,7 +595,7 @@ int _mi_os_numa_node_get(mi_os_tld_t* tld) { size_t numa_count = _mi_os_numa_node_count(); if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0 // never more than the node count and >= 0 - size_t numa_node = mi_os_numa_nodex(); + size_t numa_node = _mi_prim_numa_node(); if (numa_node >= numa_count) { numa_node = numa_node % numa_count; } return (int)numa_node; } diff --git a/src/prim/prim-unix.c b/src/prim/prim-unix.c new file mode 100644 index 00000000..fdbf8e9e --- /dev/null +++ b/src/prim/prim-unix.c @@ -0,0 +1,483 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // ensure mmap flags are defined +#endif + +#if defined(__sun) +// illumos provides new mman.h api when any of these are defined +// otherwise the old api based on caddr_t which predates the void pointers one. +// stock solaris provides only the former, chose to atomically to discard those +// flags only here rather than project wide tough. +#undef _XOPEN_SOURCE +#undef _POSIX_C_SOURCE +#endif + +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" +#include "prim.h" + +#include // mmap +#include // sysconf + +#if defined(__linux__) + #include + #include + #if defined(__GLIBC__) + #include // linux mmap flags + #else + #include + #endif +#elif defined(__APPLE__) + #include + #if !TARGET_IOS_IPHONE && !TARGET_IOS_SIMULATOR + #include + #endif +#elif defined(__FreeBSD__) || defined(__DragonFly__) + #include + #if __FreeBSD_version >= 1200000 + #include + #include + #endif + #include +#endif + +//--------------------------------------------- +// init +//--------------------------------------------- + +static bool unix_detect_overcommit(void) { + bool os_overcommit = true; +#if defined(__linux__) + int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd >= 0) { + char buf[32]; + ssize_t nread = read(fd, &buf, sizeof(buf)); + close(fd); + // + // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) + if (nread >= 1) { + os_overcommit = (buf[0] == '0' || buf[0] == '1'); + } + } +#elif defined(__FreeBSD__) + int val = 0; + size_t olen = sizeof(val); + if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { + os_overcommit = (val != 0); + } +#else + // default: overcommit is true +#endif + return os_overcommit; +} + +void _mi_prim_mem_init( mi_os_mem_config_t* config ) { + long psize = sysconf(_SC_PAGESIZE); + if (psize > 0) { + config->page_size = (size_t)psize; + config->alloc_granularity = (size_t)psize; + } + config->large_page_size = 2*MI_MiB; // TODO: can we query the OS for this? + config->has_overcommit = unix_detect_overcommit(); + config->must_free_whole = false; // mmap can free in parts +} + + +//--------------------------------------------- +// free +//--------------------------------------------- + +void _mi_prim_free(void* addr, size_t size ) { + bool err = (munmap(addr, size) == -1); + if (err) { + _mi_warning_message("unable to release OS memory: %s, addr: %p, size: %zu\n", strerror(errno), addr, size); + } +} + + +//--------------------------------------------- +// mmap +//--------------------------------------------- + +static int unix_madvise(void* addr, size_t size, int advice) { + #if defined(__sun) + return madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520) + #else + return madvise(addr, size, advice); + #endif +} + +static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { + MI_UNUSED(try_alignment); + #if defined(MAP_ALIGNED) // BSD + if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { + size_t n = mi_bsr(try_alignment); + if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB + flags |= MAP_ALIGNED(n); + void* p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + } + #elif defined(MAP_ALIGN) // Solaris + if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { + void* p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); // addr parameter is the required alignment + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + #endif + #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) + // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations + if (addr == NULL) { + void* hint = _mi_os_get_aligned_hint(try_alignment, size); + if (hint != NULL) { + void* p = mmap(hint, size, protect_flags, flags, fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + } + #endif + // regular mmap + void* p = mmap(addr, size, protect_flags, flags, fd, 0); + if (p!=MAP_FAILED) return p; + // failed to allocate + return NULL; +} + +static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { + void* p = NULL; + #if !defined(MAP_ANONYMOUS) + #define MAP_ANONYMOUS MAP_ANON + #endif + #if !defined(MAP_NORESERVE) + #define MAP_NORESERVE 0 + #endif + int flags = MAP_PRIVATE | MAP_ANONYMOUS; + int fd = -1; + if (_mi_os_has_overcommit()) { + flags |= MAP_NORESERVE; + } + #if defined(PROT_MAX) + protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD + #endif + #if defined(VM_MAKE_TAG) + // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) + int os_tag = (int)mi_option_get(mi_option_os_tag); + if (os_tag < 100 || os_tag > 255) { os_tag = 100; } + fd = VM_MAKE_TAG(os_tag); + #endif + // huge page allocation + if ((large_only || _mi_os_use_large_page(size, try_alignment)) && allow_large) { + static _Atomic(size_t) large_page_try_ok; // = 0; + size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); + if (!large_only && try_ok > 0) { + // If the OS is not configured for large OS pages, or the user does not have + // enough permission, the `mmap` will always fail (but it might also fail for other reasons). + // Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times + // to avoid too many failing calls to mmap. + mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1); + } + else { + int lflags = flags & ~MAP_NORESERVE; // using NORESERVE on huge pages seems to fail on Linux + int lfd = fd; + #ifdef MAP_ALIGNED_SUPER + lflags |= MAP_ALIGNED_SUPER; + #endif + #ifdef MAP_HUGETLB + lflags |= MAP_HUGETLB; + #endif + #ifdef MAP_HUGE_1GB + static bool mi_huge_pages_available = true; + if ((size % MI_GiB) == 0 && mi_huge_pages_available) { + lflags |= MAP_HUGE_1GB; + } + else + #endif + { + #ifdef MAP_HUGE_2MB + lflags |= MAP_HUGE_2MB; + #endif + } + #ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB + lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB; + #endif + if (large_only || lflags != flags) { + // try large OS page allocation + *is_large = true; + p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd); + #ifdef MAP_HUGE_1GB + if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) { + mi_huge_pages_available = false; // don't try huge 1GiB pages again + _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (error %i)\n", errno); + lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB); + p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd); + } + #endif + if (large_only) return p; + if (p == NULL) { + mi_atomic_store_release(&large_page_try_ok, (size_t)8); // on error, don't try again for the next N allocations + } + } + } + } + // regular allocation + if (p == NULL) { + *is_large = false; + p = unix_mmap_prim(addr, size, try_alignment, protect_flags, flags, fd); + if (p != NULL) { + #if defined(MADV_HUGEPAGE) + // Many Linux systems don't allow MAP_HUGETLB but they support instead + // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE + // though since properly aligned allocations will already use large pages if available + // in that case -- in particular for our large regions (in `memory.c`). + // However, some systems only allow THP if called with explicit `madvise`, so + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. + if (allow_large && _mi_os_use_large_page(size, try_alignment)) { + if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { + *is_large = true; // possibly + }; + } + #elif defined(__sun) + if (allow_large && _mi_os_use_large_page(size, try_alignment)) { + struct memcntl_mha cmd = {0}; + cmd.mha_pagesize = large_os_page_size; + cmd.mha_cmd = MHA_MAPSIZE_VA; + if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { + *is_large = true; + } + } + #endif + } + } + if (p == NULL) { + _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, errno, addr, large_only, allow_large); + } + return p; +} + +// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. +void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) { + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(commit || !allow_large); + mi_assert_internal(try_alignment > 0); + + int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); + return unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); +} + + +//--------------------------------------------- +// Commit/Reset +//--------------------------------------------- + +static void unix_mprotect_hint(int err) { + #if defined(__linux__) && (MI_SECURE>=2) // guard page around every mimalloc page + if (err == ENOMEM) { + _mi_warning_message("The next warning may be caused by a low memory map limit.\n" + " On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n" + " For example: sudo sysctl -w vm.max_map_count=262144\n"); + } + #else + MI_UNUSED(err); + #endif +} + + +int _mi_prim_commit(void* start, size_t size, bool commit) { + /* + #if 0 && defined(MAP_FIXED) && !defined(__APPLE__) + // Linux: disabled for now as mmap fixed seems much more expensive than MADV_DONTNEED (and splits VMA's?) + if (commit) { + // commit: just change the protection + err = mprotect(start, csize, (PROT_READ | PROT_WRITE)); + if (err != 0) { err = errno; } + } + else { + // decommit: use mmap with MAP_FIXED to discard the existing memory (and reduce rss) + const int fd = mi_unix_mmap_fd(); + void* p = mmap(start, csize, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0); + if (p != start) { err = errno; } + } + #else + */ + int err = 0; + if (commit) { + // commit: ensure we can access the area + err = mprotect(start, size, (PROT_READ | PROT_WRITE)); + if (err != 0) { err = errno; } + } + else { + #if defined(MADV_DONTNEED) && MI_DEBUG == 0 && MI_SECURE == 0 + // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) + // (on the other hand, MADV_FREE would be good enough.. it is just not reflected in the stats :-( ) + err = unix_madvise(start, size, MADV_DONTNEED); + #else + // decommit: just disable access (also used in debug and secure mode to trap on illegal access) + err = mprotect(start, size, PROT_NONE); + if (err != 0) { err = errno; } + #endif + } + unix_mprotect_hint(err); + return err; +} + +int _mi_prim_reset(void* start, size_t size) { + #if defined(MADV_FREE) + static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE); + int oadvice = (int)mi_atomic_load_relaxed(&advice); + int err; + while ((err = unix_madvise(start, size, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; + if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { + // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on + mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED); + err = unix_madvise(start, size, MADV_DONTNEED); + } + #else + int err = unix_madvise(start, csize, MADV_DONTNEED); + #endif + return err; +} + +int _mi_prim_protect(void* start, size_t size, bool protect) { + int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE)); + if (err != 0) { err = errno; } + unix_mprotect_hint(err); + return err; +} + + + +//--------------------------------------------- +// Huge page allocation +//--------------------------------------------- + +#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) + +#include + +#ifndef MPOL_PREFERRED +#define MPOL_PREFERRED 1 +#endif + +#if defined(SYS_mbind) +static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { + return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags); +} +#else +static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { + MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags); + return 0; +} +#endif + +void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) { + bool is_large = true; + void* p = unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); + if (p == NULL) return NULL; + if (numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes + unsigned long numa_mask = (1UL << numa_node); + // TODO: does `mbind` work correctly for huge OS pages? should we + // use `set_mempolicy` before calling mmap instead? + // see: + long err = mi_prim_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); + if (err != 0) { + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d: %s\n", numa_node, strerror(errno)); + } + } + return p; +} + +#else + +void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node); + return NULL; +} + +#endif + +//--------------------------------------------- +// NUMA nodes +//--------------------------------------------- + +#if defined(__linux__) + +#include // getcpu +#include // access + +size_t _mi_prim_numa_node(void) { + #ifdef SYS_getcpu + unsigned long node = 0; + unsigned long ncpu = 0; + long err = syscall(SYS_getcpu, &ncpu, &node, NULL); + if (err != 0) return 0; + return node; + #else + return 0; + #endif +} + +size_t _mi_prim_numa_node_count(void) { + char buf[128]; + unsigned node = 0; + for(node = 0; node < 256; node++) { + // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation) + snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1); + if (access(buf,R_OK) != 0) break; + } + return (node+1); +} + +#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000 + +size_t mi_prim_numa_node(void) { + domainset_t dom; + size_t node; + int policy; + if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul; + for (node = 0; node < MAXMEMDOM; node++) { + if (DOMAINSET_ISSET(node, &dom)) return node; + } + return 0ul; +} + +size_t _mi_prim_numa_node_count(void) { + size_t ndomains = 0; + size_t len = sizeof(ndomains); + if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul; + return ndomains; +} + +#elif defined(__DragonFly__) + +size_t _mi_prim_numa_node(void) { + // TODO: DragonFly does not seem to provide any userland means to get this information. + return 0ul; +} + +size_t _mi_prim_numa_node_count(void) { + size_t ncpus = 0, nvirtcoresperphys = 0; + size_t len = sizeof(size_t); + if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul; + if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul; + return nvirtcoresperphys * ncpus; +} + +#else + +size_t _mi_prim_numa_node(void) { + return 0; +} + +size_t _mi_prim_numa_node_count(void) { + return 1; +} + +#endif diff --git a/src/prim/prim-wasi.c b/src/prim/prim-wasi.c new file mode 100644 index 00000000..431113e3 --- /dev/null +++ b/src/prim/prim-wasi.c @@ -0,0 +1,154 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" +#include "prim.h" + +//--------------------------------------------- +// Initialize +//--------------------------------------------- + +void _mi_prim_mem_init( mi_os_mem_config_t* config ) { + config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB + config->alloc_granularity = 16; + config->has_overcommit = false; + config->must_free_whole = true; +} + +//--------------------------------------------- +// Free +//--------------------------------------------- + +void _mi_prim_free(void* addr, size_t size ) { + MI_UNUSED(addr); MI_UNUSED(size); + // wasi heap cannot be shrunk +} + + +//--------------------------------------------- +// Allocation: sbrk or memory_grow +//--------------------------------------------- + +#if defined(MI_USE_SBRK) + static void* mi_memory_grow( size_t size ) { + void* p = sbrk(size); + if (p == (void*)(-1)) return NULL; + #if !defined(__wasi__) // on wasi this is always zero initialized already (?) + memset(p,0,size); + #endif + return p; + } +#elif defined(__wasi__) + static void* mi_memory_grow( size_t size ) { + size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size())) + : __builtin_wasm_memory_size(0)); + if (base == SIZE_MAX) return NULL; + return (void*)(base * _mi_os_page_size()); + } +#endif + +#if defined(MI_USE_PTHREADS) +static pthread_mutex_t mi_heap_grow_mutex = PTHREAD_MUTEX_INITIALIZER; +#endif + +static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { + void* p = NULL; + if (try_alignment <= 1) { + // `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now) + #if defined(MI_USE_PTHREADS) + pthread_mutex_lock(&mi_heap_grow_mutex); + #endif + p = mi_memory_grow(size); + #if defined(MI_USE_PTHREADS) + pthread_mutex_unlock(&mi_heap_grow_mutex); + #endif + } + else { + void* base = NULL; + size_t alloc_size = 0; + // to allocate aligned use a lock to try to avoid thread interaction + // between getting the current size and actual allocation + // (also, `sbrk` is not thread safe in general) + #if defined(MI_USE_PTHREADS) + pthread_mutex_lock(&mi_heap_grow_mutex); + #endif + { + void* current = mi_memory_grow(0); // get current size + if (current != NULL) { + void* aligned_current = mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space + alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size()); + base = mi_memory_grow(alloc_size); + } + } + #if defined(MI_USE_PTHREADS) + pthread_mutex_unlock(&mi_heap_grow_mutex); + #endif + if (base != NULL) { + p = mi_align_up_ptr(base, try_alignment); + if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) { + // another thread used wasm_memory_grow/sbrk in-between and we do not have enough + // space after alignment. Give up (and waste the space as we cannot shrink :-( ) + // (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align) + p = NULL; + } + } + } + if (p == NULL) { + _mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment); + errno = ENOMEM; + return NULL; + } + mi_assert_internal( try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); + return p; +} + +// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. +void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) { + MI_UNUSED(allow_large); + *is_large = false; + return mi_prim_mem_grow(size, try_alignment); +} + + +//--------------------------------------------- +// Commit/Reset/Protect +//--------------------------------------------- + +int _mi_prim_commit(void* addr, size_t size, bool commit) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(commit); + return 0; +} + +int _mi_prim_reset(void* addr, size_t size) { + MI_UNUSED(addr); MI_UNUSED(size); + return 0; +} + +int _mi_prim_protect(void* addr, size_t size, bool protect) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); + return 0; +} + + +//--------------------------------------------- +// Huge pages and NUMA nodes +//--------------------------------------------- + +void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node); + return NULL; +} + +size_t _mi_prim_numa_node(void) { + return 0; +} + +size_t _mi_prim_numa_node_count(void) { + return 1; +} diff --git a/src/prim/prim-windows.c b/src/prim/prim-windows.c new file mode 100644 index 00000000..44ce9e8e --- /dev/null +++ b/src/prim/prim-windows.c @@ -0,0 +1,385 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" +#include "prim.h" +#include // strerror + +#ifdef _MSC_VER +#pragma warning(disable:4996) // strerror +#endif + +//--------------------------------------------- +// Dynamically bind Windows API points for portability +//--------------------------------------------- + +// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016. +// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility) +// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB) +// We define a minimal MEM_EXTENDED_PARAMETER ourselves in order to be able to compile with older SDK's. +typedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E { + MiMemExtendedParameterInvalidType = 0, + MiMemExtendedParameterAddressRequirements, + MiMemExtendedParameterNumaNode, + MiMemExtendedParameterPartitionHandle, + MiMemExtendedParameterUserPhysicalHandle, + MiMemExtendedParameterAttributeFlags, + MiMemExtendedParameterMax +} MI_MEM_EXTENDED_PARAMETER_TYPE; + +typedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S { + struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type; + union { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg; +} MI_MEM_EXTENDED_PARAMETER; + +typedef struct MI_MEM_ADDRESS_REQUIREMENTS_S { + PVOID LowestStartingAddress; + PVOID HighestEndingAddress; + SIZE_T Alignment; +} MI_MEM_ADDRESS_REQUIREMENTS; + +#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE 0x00000010 + +#include +typedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); +typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); +static PVirtualAlloc2 pVirtualAlloc2 = NULL; +static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL; + +// Similarly, GetNumaProcesorNodeEx is only supported since Windows 7 +typedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER; + +typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber); +typedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber); +typedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask); +typedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber); +static PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL; +static PGetNumaProcessorNodeEx pGetNumaProcessorNodeEx = NULL; +static PGetNumaNodeProcessorMaskEx pGetNumaNodeProcessorMaskEx = NULL; +static PGetNumaProcessorNode pGetNumaProcessorNode = NULL; + +//--------------------------------------------- +// Enable large page support dynamically (if possible) +//--------------------------------------------- + +static bool win_enable_large_os_pages(size_t* large_page_size) +{ + static bool large_initialized = false; + if (large_initialized) return (_mi_os_large_page_size() > 0); + large_initialized = true; + + // Try to see if large OS pages are supported + // To use large pages on Windows, we first need access permission + // Set "Lock pages in memory" permission in the group policy editor + // + unsigned long err = 0; + HANDLE token = NULL; + BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token); + if (ok) { + TOKEN_PRIVILEGES tp; + ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid); + if (ok) { + tp.PrivilegeCount = 1; + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0); + if (ok) { + err = GetLastError(); + ok = (err == ERROR_SUCCESS); + if (ok && large_page_size != NULL) { + *large_page_size = GetLargePageMinimum(); + } + } + } + CloseHandle(token); + } + if (!ok) { + if (err == 0) err = GetLastError(); + _mi_warning_message("cannot enable large OS page support, error %lu\n", err); + } + return (ok!=0); +} + + +//--------------------------------------------- +// Initialize +//--------------------------------------------- + +void _mi_prim_mem_init( mi_os_mem_config_t* config ) +{ + config->has_overcommit = false; + config->must_free_whole = true; + // get the page size + SYSTEM_INFO si; + GetSystemInfo(&si); + if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; } + if (si.dwAllocationGranularity > 0) { config->alloc_granularity = si.dwAllocationGranularity; } + // get the VirtualAlloc2 function + HINSTANCE hDll; + hDll = LoadLibrary(TEXT("kernelbase.dll")); + if (hDll != NULL) { + // use VirtualAlloc2FromApp if possible as it is available to Windows store apps + pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp"); + if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2"); + FreeLibrary(hDll); + } + // NtAllocateVirtualMemoryEx is used for huge page allocation + hDll = LoadLibrary(TEXT("ntdll.dll")); + if (hDll != NULL) { + pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx"); + FreeLibrary(hDll); + } + // Try to use Win7+ numa API + hDll = LoadLibrary(TEXT("kernel32.dll")); + if (hDll != NULL) { + pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx"); + pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx"); + pGetNumaNodeProcessorMaskEx = (PGetNumaNodeProcessorMaskEx)(void (*)(void))GetProcAddress(hDll, "GetNumaNodeProcessorMaskEx"); + pGetNumaProcessorNode = (PGetNumaProcessorNode)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNode"); + FreeLibrary(hDll); + } + if (mi_option_is_enabled(mi_option_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { + win_enable_large_os_pages(&config->large_page_size); + } +} + + +//--------------------------------------------- +// Free +//--------------------------------------------- + +void _mi_prim_free(void* addr, size_t size ) { + DWORD errcode = 0; + bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); + if (err) { errcode = GetLastError(); } + if (errcode == ERROR_INVALID_ADDRESS) { + // In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside + // the memory region returned by VirtualAlloc; in that case we need to free using + // the start of the region. + MEMORY_BASIC_INFORMATION info = { 0 }; + VirtualQuery(addr, &info, sizeof(info)); + if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)MI_SEGMENT_SIZE) { + errcode = 0; + err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0); + if (err) { errcode = GetLastError(); } + } + } + if (errcode != 0) { + _mi_warning_message("unable to release OS memory: error code 0x%x, addr: %p, size: %zu\n", errcode, addr, size); + } +} + + +//--------------------------------------------- +// VirtualAlloc +//--------------------------------------------- + +static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) { + #if (MI_INTPTR_SIZE >= 8) + // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations + if (addr == NULL) { + void* hint = _mi_os_get_aligned_hint(try_alignment,size); + if (hint != NULL) { + void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); + if (p != NULL) return p; + _mi_verbose_message("warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), hint, try_alignment, flags); + // fall through on error + } + } + #endif + // on modern Windows try use VirtualAlloc2 for aligned allocation + if (try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { + MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 }; + reqs.Alignment = try_alignment; + MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} }; + param.Type.Type = MiMemExtendedParameterAddressRequirements; + param.Arg.Pointer = &reqs; + void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); + if (p != NULL) return p; + _mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags); + // fall through on error + } + // last resort + return VirtualAlloc(addr, size, flags, PAGE_READWRITE); +} + +static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) { + mi_assert_internal(!(large_only && !allow_large)); + static _Atomic(size_t) large_page_try_ok; // = 0; + void* p = NULL; + // Try to allocate large OS pages (2MiB) if allowed or required. + if ((large_only || _mi_os_use_large_page(size, try_alignment)) + && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) { + size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); + if (!large_only && try_ok > 0) { + // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. + // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times. + mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1); + } + else { + // large OS pages must always reserve and commit. + *is_large = true; + p = win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES); + if (large_only) return p; + // fall back to non-large page allocation on error (`p == NULL`). + if (p == NULL) { + mi_atomic_store_release(&large_page_try_ok,10UL); // on error, don't try again for the next N allocations + } + } + } + // Fall back to regular page allocation + if (p == NULL) { + *is_large = ((flags&MEM_LARGE_PAGES) != 0); + p = win_virtual_alloc_prim(addr, size, try_alignment, flags); + } + if (p == NULL) { + _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); + } + return p; +} + +void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) { + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(commit || !allow_large); + mi_assert_internal(try_alignment > 0); + int flags = MEM_RESERVE; + if (commit) { flags |= MEM_COMMIT; } + return win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); +} + + +//--------------------------------------------- +// Commit/Reset/Protect +//--------------------------------------------- +#ifdef _MSC_VER +#pragma warning(disable:6250) // suppress warning calling VirtualFree without MEM_RELEASE (for decommit) +#endif + +int _mi_prim_commit(void* addr, size_t size, bool commit) { + if (commit) { + void* p = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE); + return (p == addr ? 0 : (int)GetLastError()); + } + else { + BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT); + return (ok ? 0 : (int)GetLastError()); + } +} + +int _mi_prim_reset(void* addr, size_t size) { + void* p = VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); + mi_assert_internal(p == addr); + #if 1 + if (p == addr && addr != NULL) { + VirtualUnlock(addr,size); // VirtualUnlock after MEM_RESET removes the memory from the working set + } + #endif + return (p == addr ? 0 : (int)GetLastError()); +} + +int _mi_prim_protect(void* addr, size_t size, bool protect) { + DWORD oldprotect = 0; + BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect); + return (ok ? 0 : (int)GetLastError()); +} + + +//--------------------------------------------- +// Huge page allocation +//--------------------------------------------- + +void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) +{ + const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE; + + win_enable_large_os_pages(NULL); + + MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} }; + // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages + static bool mi_huge_pages_available = true; + if (pNtAllocateVirtualMemoryEx != NULL && mi_huge_pages_available) { + params[0].Type.Type = MiMemExtendedParameterAttributeFlags; + params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE; + ULONG param_count = 1; + if (numa_node >= 0) { + param_count++; + params[1].Type.Type = MiMemExtendedParameterNumaNode; + params[1].Arg.ULong = (unsigned)numa_node; + } + SIZE_T psize = size; + void* base = addr; + NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count); + if (err == 0 && base != NULL) { + return base; + } + else { + // fall back to regular large pages + mi_huge_pages_available = false; // don't try further huge pages + _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err); + } + } + // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation + if (pVirtualAlloc2 != NULL && numa_node >= 0) { + params[0].Type.Type = MiMemExtendedParameterNumaNode; + params[0].Arg.ULong = (unsigned)numa_node; + return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1); + } + + // otherwise use regular virtual alloc on older windows + return VirtualAlloc(addr, size, flags, PAGE_READWRITE); +} + + +//--------------------------------------------- +// Numa nodes +//--------------------------------------------- + +size_t _mi_prim_numa_node(void) { + USHORT numa_node = 0; + if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) { + // Extended API is supported + MI_PROCESSOR_NUMBER pnum; + (*pGetCurrentProcessorNumberEx)(&pnum); + USHORT nnode = 0; + BOOL ok = (*pGetNumaProcessorNodeEx)(&pnum, &nnode); + if (ok) { numa_node = nnode; } + } + else if (pGetNumaProcessorNode != NULL) { + // Vista or earlier, use older API that is limited to 64 processors. Issue #277 + DWORD pnum = GetCurrentProcessorNumber(); + UCHAR nnode = 0; + BOOL ok = pGetNumaProcessorNode((UCHAR)pnum, &nnode); + if (ok) { numa_node = nnode; } + } + return numa_node; +} + +size_t _mi_prim_numa_node_count(void) { + ULONG numa_max = 0; + GetNumaHighestNodeNumber(&numa_max); + // find the highest node number that has actual processors assigned to it. Issue #282 + while(numa_max > 0) { + if (pGetNumaNodeProcessorMaskEx != NULL) { + // Extended API is supported + GROUP_AFFINITY affinity; + if ((*pGetNumaNodeProcessorMaskEx)((USHORT)numa_max, &affinity)) { + if (affinity.Mask != 0) break; // found the maximum non-empty node + } + } + else { + // Vista or earlier, use older API that is limited to 64 processors. + ULONGLONG mask; + if (GetNumaNodeProcessorMask((UCHAR)numa_max, &mask)) { + if (mask != 0) break; // found the maximum non-empty node + }; + } + // max node was invalid or had no processor assigned, try again + numa_max--; + } + return ((size_t)numa_max + 1); +} diff --git a/src/prim/prim.c b/src/prim/prim.c new file mode 100644 index 00000000..83b7abc1 --- /dev/null +++ b/src/prim/prim.c @@ -0,0 +1,18 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// Select the implementation of the primitives +// depending on the OS. + +#if defined(_WIN32) +#include "prim-windows.c" // VirtualAlloc (Windows) +#elif defined(__wasi__) +#define MI_USE_SBRK +#include "prim-wasi.h" // memory-grow or sbrk (Wasm) +#else +#include "prim-unix.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) +#endif diff --git a/src/prim/prim.h b/src/prim/prim.h new file mode 100644 index 00000000..ec001a9e --- /dev/null +++ b/src/prim/prim.h @@ -0,0 +1,64 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_PRIM_H +#define MIMALLOC_PRIM_H + +// note: on all primitive functions, we always get: +// addr != NULL and page aligned +// size > 0 and page aligned +// + +// OS memory configuration +typedef struct mi_os_mem_config_s { + size_t page_size; // 4KiB + size_t large_page_size; // 2MiB + size_t alloc_granularity; // smallest allocation size (on Windows 64KiB) + bool has_overcommit; // can we reserve more memory than can be actually committed? + bool must_free_whole; // must allocated blocks free as a whole (false for mmap, true for VirtualAlloc) +} mi_os_mem_config_t; + +// Initialize +void _mi_prim_mem_init( mi_os_mem_config_t* config ); + +// Free OS memory +// pre: addr != NULL, size > 0 +void _mi_prim_free(void* addr, size_t size ); + +// Allocate OS memory. +// The `try_alignment` is just a hint and the returned pointer does not have to be aligned. +// return NULL on error. +// pre: !commit => !allow_large +// try_alignment >= _mi_os_page_size() and a power of 2 +void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large); + +// Commit memory. Returns error code or 0 on success. +int _mi_prim_commit(void* addr, size_t size, bool commit); + +// Reset memory. The range keeps being accessible but the content might be reset. +// Returns error code or 0 on success. +int _mi_prim_reset(void* addr, size_t size); + +// Protect memory. Returns error code or 0 on success. +int _mi_prim_protect(void* addr, size_t size, bool protect); + +// Allocate huge (1GiB) pages possibly associated with a NUMA node. +// pre: size > 0 and a multiple of 1GiB. +// addr is either NULL or an address hint. +// numa_node is either negative (don't care), or a numa node number. +void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node); + +// Return the current NUMA node +size_t _mi_prim_numa_node(void); + +// Return the number of logical NUMA nodes +size_t _mi_prim_numa_node_count(void); + + +#endif // MIMALLOC_PRIM_H + + diff --git a/src/prim/readme.md b/src/prim/readme.md new file mode 100644 index 00000000..14248496 --- /dev/null +++ b/src/prim/readme.md @@ -0,0 +1,6 @@ +This is the portability layer where all primitives needed from the OS are defined. + +- `prim.h`: API definition +- `prim.c`: Selects one of `prim-unix.c`, `prim-wasi.c`, or `prim-windows.c` depending on the host platform. + +Note: still work in progress, there may be other places in the sources that still depend on OS ifdef's. \ No newline at end of file From 69cb30a874858ab1d161731732453202b62e1e2b Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 14 Mar 2023 17:15:52 -0700 Subject: [PATCH 027/102] move process info into primitives --- src/prim/prim-unix.c | 108 ++++++++++++++++++++++++ src/prim/prim-wasi.c | 46 ++++++++++ src/prim/prim-windows.c | 72 ++++++++++++++++ src/prim/prim.h | 9 ++ src/stats.c | 182 +++------------------------------------- 5 files changed, 248 insertions(+), 169 deletions(-) diff --git a/src/prim/prim-unix.c b/src/prim/prim-unix.c index fdbf8e9e..297dc2a9 100644 --- a/src/prim/prim-unix.c +++ b/src/prim/prim-unix.c @@ -481,3 +481,111 @@ size_t _mi_prim_numa_node_count(void) { } #endif + +// ---------------------------------------------------------------- +// Clock +// ---------------------------------------------------------------- + +#include + +#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) + +mi_msecs_t _mi_prim_clock_now(void) { + struct timespec t; + #ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &t); + #else + clock_gettime(CLOCK_REALTIME, &t); + #endif + return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000); +} + +#else + +// low resolution timer +mi_msecs_t _mi_prim_clock_now(void) { + return ((mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000)); +} + +#endif + + + + +//---------------------------------------------------------------- +// Process info +//---------------------------------------------------------------- + +#if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__) +#include +#include +#include + +#if defined(__APPLE__) +#include +#endif + +#if defined(__HAIKU__) +#include +#endif + +static mi_msecs_t timeval_secs(const struct timeval* tv) { + return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L); +} + +void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +{ + struct rusage rusage; + getrusage(RUSAGE_SELF, &rusage); + *utime = timeval_secs(&rusage.ru_utime); + *stime = timeval_secs(&rusage.ru_stime); +#if !defined(__HAIKU__) + *page_faults = rusage.ru_majflt; +#endif + // estimate commit using our stats + *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); + *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); + *current_rss = *current_commit; // estimate +#if defined(__HAIKU__) + // Haiku does not have (yet?) a way to + // get these stats per process + thread_info tid; + area_info mem; + ssize_t c; + get_thread_info(find_thread(0), &tid); + while (get_next_area_info(tid.team, &c, &mem) == B_OK) { + *peak_rss += mem.ram_size; + } + *page_faults = 0; +#elif defined(__APPLE__) + *peak_rss = rusage.ru_maxrss; // BSD reports in bytes + struct mach_task_basic_info info; + mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { + *current_rss = (size_t)info.resident_size; + } +#else + *peak_rss = rusage.ru_maxrss * 1024; // Linux reports in KiB +#endif +} + +#else + +#ifndef __wasi__ +// WebAssembly instances are not processes +#pragma message("define a way to get process info") +#endif + +void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +{ + *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); + *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); + *peak_rss = *peak_commit; + *current_rss = *current_commit; + *page_faults = 0; + *utime = 0; + *stime = 0; +} + +#endif + diff --git a/src/prim/prim-wasi.c b/src/prim/prim-wasi.c index 431113e3..964a7ede 100644 --- a/src/prim/prim-wasi.c +++ b/src/prim/prim-wasi.c @@ -152,3 +152,49 @@ size_t _mi_prim_numa_node(void) { size_t _mi_prim_numa_node_count(void) { return 1; } + + +//---------------------------------------------------------------- +// Clock +//---------------------------------------------------------------- + +#include + +#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) + +mi_msecs_t _mi_prim_clock_now(void) { + struct timespec t; + #ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &t); + #else + clock_gettime(CLOCK_REALTIME, &t); + #endif + return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000); +} + +#else + +// low resolution timer +mi_msecs_t _mi_prim_clock_now(void) { + return ((mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000)); +} + +#endif + + +//---------------------------------------------------------------- +// Process info +//---------------------------------------------------------------- + +void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +{ + *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); + *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); + *peak_rss = *peak_commit; + *current_rss = *current_commit; + *page_faults = 0; + *utime = 0; + *stime = 0; +} + + diff --git a/src/prim/prim-windows.c b/src/prim/prim-windows.c index 44ce9e8e..b83082e3 100644 --- a/src/prim/prim-windows.c +++ b/src/prim/prim-windows.c @@ -383,3 +383,75 @@ size_t _mi_prim_numa_node_count(void) { } return ((size_t)numa_max + 1); } + + +//---------------------------------------------------------------- +// Clock +//---------------------------------------------------------------- + +static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { + static LARGE_INTEGER mfreq; // = 0 + if (mfreq.QuadPart == 0LL) { + LARGE_INTEGER f; + QueryPerformanceFrequency(&f); + mfreq.QuadPart = f.QuadPart/1000LL; + if (mfreq.QuadPart == 0) mfreq.QuadPart = 1; + } + return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart); +} + +mi_msecs_t _mi_prim_clock_now(void) { + LARGE_INTEGER t; + QueryPerformanceCounter(&t); + return mi_to_msecs(t); +} + + +//---------------------------------------------------------------- +// Process info +//---------------------------------------------------------------- + +#include +#include + +static mi_msecs_t filetime_msecs(const FILETIME* ftime) { + ULARGE_INTEGER i; + i.LowPart = ftime->dwLowDateTime; + i.HighPart = ftime->dwHighDateTime; + mi_msecs_t msecs = (i.QuadPart / 10000); // FILETIME is in 100 nano seconds + return msecs; +} + +typedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD); +static PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL; + +void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +{ + FILETIME ct; + FILETIME ut; + FILETIME st; + FILETIME et; + GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); + *utime = filetime_msecs(&ut); + *stime = filetime_msecs(&st); + + // load psapi on demand + if (pGetProcessMemoryInfo == NULL) { + HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll")); + if (hDll != NULL) { + pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo"); + } + } + + // get process info + PROCESS_MEMORY_COUNTERS info; + memset(&info, 0, sizeof(info)); + if (pGetProcessMemoryInfo != NULL) { + pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); + } + *current_rss = (size_t)info.WorkingSetSize; + *peak_rss = (size_t)info.PeakWorkingSetSize; + *current_commit = (size_t)info.PagefileUsage; + *peak_commit = (size_t)info.PeakPagefileUsage; + *page_faults = (size_t)info.PageFaultCount; +} diff --git a/src/prim/prim.h b/src/prim/prim.h index ec001a9e..14f75156 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -58,6 +58,15 @@ size_t _mi_prim_numa_node(void); // Return the number of logical NUMA nodes size_t _mi_prim_numa_node_count(void); +// High resolution clock +mi_msecs_t _mi_prim_clock_now(void); + +// Return process information +void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, + size_t* current_rss, size_t* peak_rss, + size_t* current_commit, size_t* peak_commit, size_t* page_faults); + + #endif // MIMALLOC_PRIM_H diff --git a/src/stats.c b/src/stats.c index 84d677fa..c9b3bb95 100644 --- a/src/stats.c +++ b/src/stats.c @@ -7,8 +7,9 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" +#include "prim/prim.h" -#include // fputs, stderr +#include // snprintf #include // memset #if defined(_MSC_VER) && (_MSC_VER < 1920) @@ -291,8 +292,6 @@ static void mi_cdecl mi_buffered_out(const char* msg, void* arg) { // Print statistics //------------------------------------------------------------ -static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults); - static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept { // wrap the output function to be line buffered char buf[256]; @@ -337,15 +336,15 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); _mi_fprintf(out, arg, "%10s: %7zu\n", "numa nodes", _mi_os_numa_node_count()); - mi_msecs_t elapsed; - mi_msecs_t user_time; - mi_msecs_t sys_time; + size_t elapsed; + size_t user_time; + size_t sys_time; size_t current_rss; size_t peak_rss; size_t current_commit; size_t peak_commit; size_t page_faults; - mi_stat_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); + mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); _mi_fprintf(out, arg, "%10s: %7ld.%03ld s\n", "elapsed", elapsed/1000, elapsed%1000); _mi_fprintf(out, arg, "%10s: user: %ld.%03ld s, system: %ld.%03ld s, faults: %lu, rss: ", "process", user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, (unsigned long)page_faults ); @@ -404,47 +403,13 @@ void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { // ---------------------------------------------------------------- // Basic timer for convenience; use milli-seconds to avoid doubles // ---------------------------------------------------------------- -#ifdef _WIN32 -#include -static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { - static LARGE_INTEGER mfreq; // = 0 - if (mfreq.QuadPart == 0LL) { - LARGE_INTEGER f; - QueryPerformanceFrequency(&f); - mfreq.QuadPart = f.QuadPart/1000LL; - if (mfreq.QuadPart == 0) mfreq.QuadPart = 1; - } - return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart); -} - -mi_msecs_t _mi_clock_now(void) { - LARGE_INTEGER t; - QueryPerformanceCounter(&t); - return mi_to_msecs(t); -} -#else -#include -#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) -mi_msecs_t _mi_clock_now(void) { - struct timespec t; - #ifdef CLOCK_MONOTONIC - clock_gettime(CLOCK_MONOTONIC, &t); - #else - clock_gettime(CLOCK_REALTIME, &t); - #endif - return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000); -} -#else -// low resolution timer -mi_msecs_t _mi_clock_now(void) { - return ((mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000)); -} -#endif -#endif - static mi_msecs_t mi_clock_diff; +mi_msecs_t _mi_clock_now(void) { + return _mi_prim_clock_now(); +} + mi_msecs_t _mi_clock_start(void) { if (mi_clock_diff == 0.0) { mi_msecs_t t0 = _mi_clock_now(); @@ -463,130 +428,9 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) { // Basic process statistics // -------------------------------------------------------- -#if defined(_WIN32) -#include -#include - -static mi_msecs_t filetime_msecs(const FILETIME* ftime) { - ULARGE_INTEGER i; - i.LowPart = ftime->dwLowDateTime; - i.HighPart = ftime->dwHighDateTime; - mi_msecs_t msecs = (i.QuadPart / 10000); // FILETIME is in 100 nano seconds - return msecs; -} - -typedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD); -static PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL; - -static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) -{ - *elapsed = _mi_clock_end(mi_process_start); - FILETIME ct; - FILETIME ut; - FILETIME st; - FILETIME et; - GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); - *utime = filetime_msecs(&ut); - *stime = filetime_msecs(&st); - - // load psapi on demand - if (pGetProcessMemoryInfo == NULL) { - HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll")); - if (hDll != NULL) { - pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo"); - } - } - - // get process info - PROCESS_MEMORY_COUNTERS info; - memset(&info, 0, sizeof(info)); - if (pGetProcessMemoryInfo != NULL) { - pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); - } - *current_rss = (size_t)info.WorkingSetSize; - *peak_rss = (size_t)info.PeakWorkingSetSize; - *current_commit = (size_t)info.PagefileUsage; - *peak_commit = (size_t)info.PeakPagefileUsage; - *page_faults = (size_t)info.PageFaultCount; -} - -#elif !defined(__wasi__) && (defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)) -#include -#include -#include - -#if defined(__APPLE__) -#include -#endif - -#if defined(__HAIKU__) -#include -#endif - -static mi_msecs_t timeval_secs(const struct timeval* tv) { - return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L); -} - -static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) -{ - *elapsed = _mi_clock_end(mi_process_start); - struct rusage rusage; - getrusage(RUSAGE_SELF, &rusage); - *utime = timeval_secs(&rusage.ru_utime); - *stime = timeval_secs(&rusage.ru_stime); -#if !defined(__HAIKU__) - *page_faults = rusage.ru_majflt; -#endif - // estimate commit using our stats - *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); - *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); - *current_rss = *current_commit; // estimate -#if defined(__HAIKU__) - // Haiku does not have (yet?) a way to - // get these stats per process - thread_info tid; - area_info mem; - ssize_t c; - get_thread_info(find_thread(0), &tid); - while (get_next_area_info(tid.team, &c, &mem) == B_OK) { - *peak_rss += mem.ram_size; - } - *page_faults = 0; -#elif defined(__APPLE__) - *peak_rss = rusage.ru_maxrss; // BSD reports in bytes - struct mach_task_basic_info info; - mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; - if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { - *current_rss = (size_t)info.resident_size; - } -#else - *peak_rss = rusage.ru_maxrss * 1024; // Linux reports in KiB -#endif -} - -#else -#ifndef __wasi__ -// WebAssembly instances are not processes -#pragma message("define a way to get process info") -#endif - -static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) -{ - *elapsed = _mi_clock_end(mi_process_start); - *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); - *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); - *peak_rss = *peak_commit; - *current_rss = *current_commit; - *page_faults = 0; - *utime = 0; - *stime = 0; -} -#endif - - mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept { - mi_msecs_t elapsed = 0; + mi_msecs_t elapsed = _mi_clock_end(mi_process_start); mi_msecs_t utime = 0; mi_msecs_t stime = 0; size_t current_rss0 = 0; @@ -594,8 +438,8 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s size_t current_commit0 = 0; size_t peak_commit0 = 0; size_t page_faults0 = 0; - mi_stat_process_info(&elapsed,&utime, &stime, ¤t_rss0, &peak_rss0, ¤t_commit0, &peak_commit0, &page_faults0); - if (elapsed_msecs!=NULL) *elapsed_msecs = (elapsed < 0 ? 0 : (elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)elapsed : PTRDIFF_MAX)); + _mi_prim_process_info(&utime, &stime, ¤t_rss0, &peak_rss0, ¤t_commit0, &peak_commit0, &page_faults0); + if (elapsed_msecs!=NULL) *elapsed_msecs = (elapsed < 0 ? 0 : (elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)elapsed : PTRDIFF_MAX)); if (user_msecs!=NULL) *user_msecs = (utime < 0 ? 0 : (utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)utime : PTRDIFF_MAX)); if (system_msecs!=NULL) *system_msecs = (stime < 0 ? 0 : (stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)stime : PTRDIFF_MAX)); if (current_rss!=NULL) *current_rss = current_rss0; From 10f62eb5a19e412c8f0691cad64fe3b59b9a346d Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 14 Mar 2023 18:10:00 -0700 Subject: [PATCH 028/102] add c primitives, move getenv into primitives --- include/mimalloc-internal.h | 9 ++ src/alloc-posix.c | 2 +- src/options.c | 166 ++++++++++-------------------------- src/prim/prim-unix.c | 66 ++++++++++++++ src/prim/prim-wasi.c | 30 +++++++ src/prim/prim-windows.c | 55 +++++++++++- src/prim/prim.h | 7 ++ 7 files changed, 209 insertions(+), 126 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index ee26bfb8..ce6e6a6b 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -166,6 +166,15 @@ bool _mi_free_delayed_block(mi_block_t* block); void _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept; // for runtime integration void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); +// option.c, c primitives +char _mi_toupper(char c); +int _mi_strnicmp(const char* s, const char* t, size_t n); +void _mi_strlcpy(char* dest, const char* src, size_t dest_size); +void _mi_strlcat(char* dest, const char* src, size_t dest_size); +size_t _mi_strlen(const char* s); +size_t _mi_strnlen(const char* s, size_t max_len); + + #if MI_DEBUG>1 bool _mi_page_is_valid(mi_page_t* page); #endif diff --git a/src/alloc-posix.c b/src/alloc-posix.c index e6505f29..f0cfe629 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -149,7 +149,7 @@ int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept { else { *buf = mi_strdup(p); if (*buf==NULL) return ENOMEM; - if (size != NULL) *size = strlen(p); + if (size != NULL) *size = _mi_strlen(p); } return 0; } diff --git a/src/options.c b/src/options.c index 44319a42..a93ea89e 100644 --- a/src/options.c +++ b/src/options.c @@ -7,17 +7,11 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" +#include "prim/prim.h" // mi_prim_out_stderr #include -#include // strtol -#include // strncpy, strncat, strlen, strstr -#include // toupper #include -#ifdef _MSC_VER -#pragma warning(disable:4996) // strncpy, strncat -#endif - static long mi_max_error_count = 16; // stop outputting errors after this (use < 0 for no limit) static long mi_max_warning_count = 16; // stop outputting warnings after this (use < 0 for no limit) @@ -170,41 +164,11 @@ void mi_option_disable(mi_option_t option) { mi_option_set_enabled(option,false); } - static void mi_cdecl mi_out_stderr(const char* msg, void* arg) { MI_UNUSED(arg); - if (msg == NULL) return; - #ifdef _WIN32 - // on windows with redirection, the C runtime cannot handle locale dependent output - // after the main thread closes so we use direct console output. - if (!_mi_preloading()) { - // _cputs(msg); // _cputs cannot be used at is aborts if it fails to lock the console - static HANDLE hcon = INVALID_HANDLE_VALUE; - static bool hconIsConsole; - if (hcon == INVALID_HANDLE_VALUE) { - CONSOLE_SCREEN_BUFFER_INFO sbi; - hcon = GetStdHandle(STD_ERROR_HANDLE); - hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); - } - const size_t len = strlen(msg); - if (len > 0 && len < UINT32_MAX) { - DWORD written = 0; - if (hconIsConsole) { - WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL); - } - else if (hcon != INVALID_HANDLE_VALUE) { - // use direct write if stderr was redirected - WriteFile(hcon, msg, (DWORD)len, &written, NULL); - } - else { - // finally fall back to fputs after all - fputs(msg, stderr); - } - } + if (msg != NULL && msg[0] != 0) { + _mi_prim_out_stderr(msg); } - #else - fputs(msg, stderr); - #endif } // Since an output function can be registered earliest in the `main` @@ -221,7 +185,7 @@ static void mi_cdecl mi_out_buf(const char* msg, void* arg) { MI_UNUSED(arg); if (msg==NULL) return; if (mi_atomic_load_relaxed(&out_len)>=MI_MAX_DELAY_OUTPUT) return; - size_t n = strlen(msg); + size_t n = _mi_strlen(msg); if (n==0) return; // claim space size_t start = mi_atomic_add_acq_rel(&out_len, n); @@ -358,7 +322,7 @@ void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) { } static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) { - if (prefix != NULL && strlen(prefix) <= 32 && !_mi_is_main_thread()) { + if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) { char tprefix[64]; snprintf(tprefix, sizeof(tprefix), "%sthread 0x%llx: ", prefix, (unsigned long long)_mi_thread_id()); mi_vfprintf(out, arg, tprefix, fmt, args); @@ -463,8 +427,20 @@ void _mi_error_message(int err, const char* fmt, ...) { // -------------------------------------------------------- // Initialize options by checking the environment // -------------------------------------------------------- +char _mi_toupper(char c) { + if (c >= 'a' && c <= 'z') return (c - 'a' + 'A'); + else return c; +} -static void mi_strlcpy(char* dest, const char* src, size_t dest_size) { +int _mi_strnicmp(const char* s, const char* t, size_t n) { + if (n == 0) return 0; + for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) { + if (_mi_toupper(*s) != _mi_toupper(*t)) break; + } + return (n == 0 ? 0 : *s - *t); +} + +void _mi_strlcpy(char* dest, const char* src, size_t dest_size) { if (dest==NULL || src==NULL || dest_size == 0) return; // copy until end of src, or when dest is (almost) full while (*src != 0 && dest_size > 1) { @@ -475,7 +451,7 @@ static void mi_strlcpy(char* dest, const char* src, size_t dest_size) { *dest = 0; } -static void mi_strlcat(char* dest, const char* src, size_t dest_size) { +void _mi_strlcat(char* dest, const char* src, size_t dest_size) { if (dest==NULL || src==NULL || dest_size == 0) return; // find end of string in the dest buffer while (*dest != 0 && dest_size > 1) { @@ -483,7 +459,21 @@ static void mi_strlcat(char* dest, const char* src, size_t dest_size) { dest_size--; } // and catenate - mi_strlcpy(dest, src, dest_size); + _mi_strlcpy(dest, src, dest_size); +} + +size_t _mi_strlen(const char* s) { + if (s==NULL) return 0; + size_t len = 0; + while(s[len] != 0) { len++; } + return len; +} + +size_t _mi_strnlen(const char* s, size_t max_len) { + if (s==NULL) return 0; + size_t len = 0; + while(s[len] != 0 && len < max_len) { len++; } + return len; } #ifdef MI_NO_GETENV @@ -494,94 +484,28 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { return false; } #else -#if defined _WIN32 -// On Windows use GetEnvironmentVariable instead of getenv to work -// reliably even when this is invoked before the C runtime is initialized. -// i.e. when `_mi_preloading() == true`. -// Note: on windows, environment names are not case sensitive. -#include static bool mi_getenv(const char* name, char* result, size_t result_size) { - result[0] = 0; - size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size); - return (len > 0 && len < result_size); -} -#elif !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0) -// On Posix systemsr use `environ` to acces environment variables -// even before the C runtime is initialized. -#if defined(__APPLE__) && defined(__has_include) && __has_include() -#include -static char** mi_get_environ(void) { - return (*_NSGetEnviron()); -} -#else -extern char** environ; -static char** mi_get_environ(void) { - return environ; + if (name==NULL || result == NULL || result_size < 64) return false; + return _mi_prim_getenv(name,result,result_size); } #endif -static int mi_strnicmp(const char* s, const char* t, size_t n) { - if (n == 0) return 0; - for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) { - if (toupper(*s) != toupper(*t)) break; - } - return (n == 0 ? 0 : *s - *t); -} -static bool mi_getenv(const char* name, char* result, size_t result_size) { - if (name==NULL) return false; - const size_t len = strlen(name); - if (len == 0) return false; - char** env = mi_get_environ(); - if (env == NULL) return false; - // compare up to 256 entries - for (int i = 0; i < 256 && env[i] != NULL; i++) { - const char* s = env[i]; - if (mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive - // found it - mi_strlcpy(result, s + len + 1, result_size); - return true; - } - } - return false; -} -#else -// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime -static bool mi_getenv(const char* name, char* result, size_t result_size) { - // cannot call getenv() when still initializing the C runtime. - if (_mi_preloading()) return false; - const char* s = getenv(name); - if (s == NULL) { - // we check the upper case name too. - char buf[64+1]; - size_t len = strlen(name); - if (len >= sizeof(buf)) len = sizeof(buf) - 1; - for (size_t i = 0; i < len; i++) { - buf[i] = toupper(name[i]); - } - buf[len] = 0; - s = getenv(buf); - } - if (s != NULL && strlen(s) < result_size) { - mi_strlcpy(result, s, result_size); - return true; - } - else { - return false; - } -} -#endif // !MI_USE_ENVIRON -#endif // !MI_NO_GETENV + +// TODO: implement ourselves to reduce dependencies on the C runtime +#include // strtol +#include // strstr + static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment char buf[64+1]; - mi_strlcpy(buf, "mimalloc_", sizeof(buf)); - mi_strlcat(buf, desc->name, sizeof(buf)); + _mi_strlcpy(buf, "mimalloc_", sizeof(buf)); + _mi_strlcat(buf, desc->name, sizeof(buf)); char s[64+1]; if (mi_getenv(buf, s, sizeof(s))) { - size_t len = strlen(s); + size_t len = _mi_strnlen(s,64); if (len >= sizeof(buf)) len = sizeof(buf) - 1; for (size_t i = 0; i < len; i++) { - buf[i] = (char)toupper(s[i]); + buf[i] = _mi_toupper(s[i]); } buf[len] = 0; if (buf[0]==0 || strstr("1;TRUE;YES;ON", buf) != NULL) { diff --git a/src/prim/prim-unix.c b/src/prim/prim-unix.c index 297dc2a9..495b274e 100644 --- a/src/prim/prim-unix.c +++ b/src/prim/prim-unix.c @@ -589,3 +589,69 @@ void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current #endif + +//---------------------------------------------------------------- +// Output +//---------------------------------------------------------------- + +void _mi_prim_out_stderr( const char* msg ) { + fputs(msg,stderr); +} + + +//---------------------------------------------------------------- +// Environment +//---------------------------------------------------------------- + +#if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0) +// On Posix systemsr use `environ` to acces environment variables +// even before the C runtime is initialized. +#if defined(__APPLE__) && defined(__has_include) && __has_include() +#include +static char** mi_get_environ(void) { + return (*_NSGetEnviron()); +} +#else +extern char** environ; +static char** mi_get_environ(void) { + return environ; +} +#endif +bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { + if (name==NULL) return false; + const size_t len = _mi_strlen(name); + if (len == 0) return false; + char** env = mi_get_environ(); + if (env == NULL) return false; + // compare up to 256 entries + for (int i = 0; i < 256 && env[i] != NULL; i++) { + const char* s = env[i]; + if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive + // found it + _mi_strlcpy(result, s + len + 1, result_size); + return true; + } + } + return false; +} +#else +// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime +bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { + // cannot call getenv() when still initializing the C runtime. + if (_mi_preloading()) return false; + const char* s = getenv(name); + if (s == NULL) { + // we check the upper case name too. + char buf[64+1]; + size_t len = _mi_strnlen(name,sizeof(buf)-1); + for (size_t i = 0; i < len; i++) { + buf[i] = _mi_toupper(name[i]); + } + buf[len] = 0; + s = getenv(buf); + } + if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false; + _mi_strlcpy(result, s, result_size); + return true; +} +#endif // !MI_USE_ENVIRON diff --git a/src/prim/prim-wasi.c b/src/prim/prim-wasi.c index 964a7ede..7de491e4 100644 --- a/src/prim/prim-wasi.c +++ b/src/prim/prim-wasi.c @@ -197,4 +197,34 @@ void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current *stime = 0; } +//---------------------------------------------------------------- +// Output +//---------------------------------------------------------------- +void _mi_prim_out_stderr( const char* msg ) { + fputs(msg,stderr); +} + + +//---------------------------------------------------------------- +// Environment +//---------------------------------------------------------------- + +bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { + // cannot call getenv() when still initializing the C runtime. + if (_mi_preloading()) return false; + const char* s = getenv(name); + if (s == NULL) { + // we check the upper case name too. + char buf[64+1]; + size_t len = _mi_strnlen(name,sizeof(buf)-1); + for (size_t i = 0; i < len; i++) { + buf[i] = _mi_toupper(name[i]); + } + buf[len] = 0; + s = getenv(buf); + } + if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false; + _mi_strlcpy(result, s, result_size); + return true; +} \ No newline at end of file diff --git a/src/prim/prim-windows.c b/src/prim/prim-windows.c index b83082e3..97b1936f 100644 --- a/src/prim/prim-windows.c +++ b/src/prim/prim-windows.c @@ -10,6 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-atomic.h" #include "prim.h" #include // strerror +#include // fputs, stderr #ifdef _MSC_VER #pragma warning(disable:4996) // strerror @@ -407,10 +408,6 @@ mi_msecs_t _mi_prim_clock_now(void) { } -//---------------------------------------------------------------- -// Process info -//---------------------------------------------------------------- - #include #include @@ -455,3 +452,53 @@ void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current *peak_commit = (size_t)info.PeakPagefileUsage; *page_faults = (size_t)info.PageFaultCount; } + +//---------------------------------------------------------------- +// Output +//---------------------------------------------------------------- + +void _mi_prim_out_stderr( const char* msg ) +{ + // on windows with redirection, the C runtime cannot handle locale dependent output + // after the main thread closes so we use direct console output. + if (!_mi_preloading()) { + // _cputs(msg); // _cputs cannot be used at is aborts if it fails to lock the console + static HANDLE hcon = INVALID_HANDLE_VALUE; + static bool hconIsConsole; + if (hcon == INVALID_HANDLE_VALUE) { + CONSOLE_SCREEN_BUFFER_INFO sbi; + hcon = GetStdHandle(STD_ERROR_HANDLE); + hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); + } + const size_t len = strlen(msg); + if (len > 0 && len < UINT32_MAX) { + DWORD written = 0; + if (hconIsConsole) { + WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL); + } + else if (hcon != INVALID_HANDLE_VALUE) { + // use direct write if stderr was redirected + WriteFile(hcon, msg, (DWORD)len, &written, NULL); + } + else { + // finally fall back to fputs after all + fputs(msg, stderr); + } + } + } +} + + +//---------------------------------------------------------------- +// Environment +//---------------------------------------------------------------- + +// On Windows use GetEnvironmentVariable instead of getenv to work +// reliably even when this is invoked before the C runtime is initialized. +// i.e. when `_mi_preloading() == true`. +// Note: on windows, environment names are not case sensitive. +bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { + result[0] = 0; + size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size); + return (len > 0 && len < result_size); +} \ No newline at end of file diff --git a/src/prim/prim.h b/src/prim/prim.h index 14f75156..734d02bf 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -66,6 +66,13 @@ void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults); +// Default stderr output. +// msg != NULL && strlen(msg) > 0 +void _mi_prim_out_stderr( const char* msg ); + +// Get an environment variable. +// name != NULL, result != NULL, result_size >= 64 +bool _mi_prim_getenv(const char* name, char* result, size_t result_size); #endif // MIMALLOC_PRIM_H From 4348a05d0f92d3197cbd26acdf641551d125121f Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 14 Mar 2023 18:24:38 -0700 Subject: [PATCH 029/102] small fixes --- src/options.c | 3 ++- src/prim/prim-unix.c | 8 +++++++- src/prim/prim-wasi.c | 10 ++++++++-- src/prim/prim-windows.c | 8 ++++++-- src/prim/prim.h | 23 +++++++++-------------- 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/options.c b/src/options.c index a93ea89e..4c76ae41 100644 --- a/src/options.c +++ b/src/options.c @@ -9,7 +9,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-atomic.h" #include "prim/prim.h" // mi_prim_out_stderr -#include +#include // FILE +#include // abort #include diff --git a/src/prim/prim-unix.c b/src/prim/prim-unix.c index 495b274e..ce24c24a 100644 --- a/src/prim/prim-unix.c +++ b/src/prim/prim-unix.c @@ -504,7 +504,13 @@ mi_msecs_t _mi_prim_clock_now(void) { // low resolution timer mi_msecs_t _mi_prim_clock_now(void) { - return ((mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000)); + #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) + return (mi_msecs_t)clock(); + #elif (CLOCKS_PER_SEC < 1000) + return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + #else + return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); + #endif } #endif diff --git a/src/prim/prim-wasi.c b/src/prim/prim-wasi.c index 7de491e4..c37b0847 100644 --- a/src/prim/prim-wasi.c +++ b/src/prim/prim-wasi.c @@ -110,7 +110,7 @@ static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) { - MI_UNUSED(allow_large); + MI_UNUSED(allow_large); MI_UNUSED(commit); *is_large = false; return mi_prim_mem_grow(size, try_alignment); } @@ -176,7 +176,13 @@ mi_msecs_t _mi_prim_clock_now(void) { // low resolution timer mi_msecs_t _mi_prim_clock_now(void) { - return ((mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000)); + #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) + return (mi_msecs_t)clock(); + #elif (CLOCKS_PER_SEC < 1000) + return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + #else + return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); + #endif } #endif diff --git a/src/prim/prim-windows.c b/src/prim/prim-windows.c index 97b1936f..f6f1a9db 100644 --- a/src/prim/prim-windows.c +++ b/src/prim/prim-windows.c @@ -408,6 +408,10 @@ mi_msecs_t _mi_prim_clock_now(void) { } +//---------------------------------------------------------------- +// Process Info +//---------------------------------------------------------------- + #include #include @@ -470,7 +474,7 @@ void _mi_prim_out_stderr( const char* msg ) hcon = GetStdHandle(STD_ERROR_HANDLE); hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); } - const size_t len = strlen(msg); + const size_t len = _mi_strlen(msg); if (len > 0 && len < UINT32_MAX) { DWORD written = 0; if (hconIsConsole) { @@ -501,4 +505,4 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { result[0] = 0; size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size); return (len > 0 && len < result_size); -} \ No newline at end of file +} diff --git a/src/prim/prim.h b/src/prim/prim.h index 734d02bf..2cbbc85c 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -9,9 +9,9 @@ terms of the MIT license. A copy of the license can be found in the file #define MIMALLOC_PRIM_H // note: on all primitive functions, we always get: -// addr != NULL and page aligned -// size > 0 and page aligned -// +// addr != NULL and page aligned +// size > 0 and page aligned + // OS memory configuration typedef struct mi_os_mem_config_s { @@ -26,12 +26,10 @@ typedef struct mi_os_mem_config_s { void _mi_prim_mem_init( mi_os_mem_config_t* config ); // Free OS memory -// pre: addr != NULL, size > 0 void _mi_prim_free(void* addr, size_t size ); -// Allocate OS memory. +// Allocate OS memory. Return NULL on error. // The `try_alignment` is just a hint and the returned pointer does not have to be aligned. -// return NULL on error. // pre: !commit => !allow_large // try_alignment >= _mi_os_page_size() and a power of 2 void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large); @@ -58,23 +56,20 @@ size_t _mi_prim_numa_node(void); // Return the number of logical NUMA nodes size_t _mi_prim_numa_node_count(void); -// High resolution clock +// Clock ticks mi_msecs_t _mi_prim_clock_now(void); -// Return process information +// Return process information (only for statistics) void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults); -// Default stderr output. -// msg != NULL && strlen(msg) > 0 +// Default stderr output. (only for warnings etc. with verbose enabled) +// msg != NULL && _mi_strlen(msg) > 0 void _mi_prim_out_stderr( const char* msg ); -// Get an environment variable. +// Get an environment variable. (only for options) // name != NULL, result != NULL, result_size >= 64 bool _mi_prim_getenv(const char* name, char* result, size_t result_size); - #endif // MIMALLOC_PRIM_H - - From 3579d3b8617c858a3d2f9a6c25b0c13d34cf811a Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 14 Mar 2023 19:38:45 -0700 Subject: [PATCH 030/102] move mi_thread_id to primitives --- include/mimalloc-internal.h | 102 +-------------------------------- src/alloc.c | 8 +-- src/init.c | 5 ++ src/prim/prim.h | 110 +++++++++++++++++++++++++++++++++++- 4 files changed, 119 insertions(+), 106 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index ce6e6a6b..6e6da9f3 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -74,6 +74,7 @@ extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); size_t _mi_current_thread_count(void); bool _mi_preloading(void); // true while the C runtime is not ready +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; // os.c size_t _mi_os_page_size(void); @@ -754,107 +755,6 @@ static inline size_t _mi_os_numa_node_count(void) { } -// ------------------------------------------------------------------- -// Getting the thread id should be performant as it is called in the -// fast path of `_mi_free` and we specialize for various platforms. -// We only require _mi_threadid() to return a unique id for each thread. -// ------------------------------------------------------------------- -#if defined(_WIN32) - -#define WIN32_LEAN_AND_MEAN -#include -static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { - // Windows: works on Intel and ARM in both 32- and 64-bit - return (uintptr_t)NtCurrentTeb(); -} - -// We use assembly for a fast thread id on the main platforms. The TLS layout depends on -// both the OS and libc implementation so we use specific tests for each main platform. -// If you test on another platform and it works please send a PR :-) -// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. -#elif defined(__GNUC__) && ( \ - (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ - || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \ - || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ - || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ - || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ - ) - -static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { - void* res; - const size_t ofs = (slot*sizeof(void*)); - #if defined(__i386__) - __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS - #elif defined(__APPLE__) && defined(__x86_64__) - __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS - #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) - __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI - #elif defined(__x86_64__) - __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS - #elif defined(__arm__) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); - res = tcb[slot]; - #elif defined(__aarch64__) - void** tcb; MI_UNUSED(ofs); - #if defined(__APPLE__) // M1, issue #343 - __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); - #else - __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); - #endif - res = tcb[slot]; - #endif - return res; -} - -// setting a tls slot is only used on macOS for now -static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { - const size_t ofs = (slot*sizeof(void*)); - #if defined(__i386__) - __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS - #elif defined(__APPLE__) && defined(__x86_64__) - __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS - #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) - __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI - #elif defined(__x86_64__) - __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS - #elif defined(__arm__) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); - tcb[slot] = value; - #elif defined(__aarch64__) - void** tcb; MI_UNUSED(ofs); - #if defined(__APPLE__) // M1, issue #343 - __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); - #else - __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); - #endif - tcb[slot] = value; - #endif -} - -static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { - #if defined(__BIONIC__) - // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id - // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86 - return (uintptr_t)mi_tls_slot(1); - #else - // in all our other targets, slot 0 is the thread id - // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h - // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36 - return (uintptr_t)mi_tls_slot(0); - #endif -} - -#else - -// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). -static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { - return (uintptr_t)&_mi_heap_default; -} - -#endif - // ----------------------------------------------------------------------- // Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero) diff --git a/src/alloc.c b/src/alloc.c index 04c4c48c..d8276df9 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -11,10 +11,10 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" +#include "prim/prim.h" // _mi_prim_thread_id() - -#include // memset, strlen -#include // malloc, exit +#include // memset, strlen (for mi_strdup) +#include // malloc, abort #define MI_IN_ALLOC_C #include "alloc-override.c" @@ -536,7 +536,7 @@ void mi_free(void* p) mi_attr_noexcept { if mi_unlikely(p == NULL) return; mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); - const bool is_local= (_mi_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); + const bool is_local= (_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); mi_page_t* const page = _mi_segment_page_of(segment, p); if mi_likely(is_local) { // thread-local free? diff --git a/src/init.c b/src/init.c index a2a6be75..dd555030 100644 --- a/src/init.c +++ b/src/init.c @@ -6,6 +6,7 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" #include "mimalloc-internal.h" +#include "prim/prim.h" #include // memcpy, memset #include // atexit @@ -103,6 +104,10 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { }; +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { + return _mi_prim_thread_id(); +} + // the thread-local default heap for allocation mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; diff --git a/src/prim/prim.h b/src/prim/prim.h index 2cbbc85c..2d951930 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -12,7 +12,6 @@ terms of the MIT license. A copy of the license can be found in the file // addr != NULL and page aligned // size > 0 and page aligned - // OS memory configuration typedef struct mi_os_mem_config_s { size_t page_size; // 4KiB @@ -72,4 +71,113 @@ void _mi_prim_out_stderr( const char* msg ); // name != NULL, result != NULL, result_size >= 64 bool _mi_prim_getenv(const char* name, char* result, size_t result_size); + +//------------------------------------------------------------------- +// Thread id +// +// Getting the thread id should be performant as it is called in the +// fast path of `_mi_free` and we specialize for various platforms as +// inlined definitions. Regular code should call `init.c:_mi_thread_id()`. +// We only require _mi_prim_thread_id() to return a unique id for each thread. +//------------------------------------------------------------------- + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; + +#if defined(_WIN32) + +#define WIN32_LEAN_AND_MEAN +#include +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + // Windows: works on Intel and ARM in both 32- and 64-bit + return (uintptr_t)NtCurrentTeb(); +} + +// We use assembly for a fast thread id on the main platforms. The TLS layout depends on +// both the OS and libc implementation so we use specific tests for each main platform. +// If you test on another platform and it works please send a PR :-) +// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. +#elif defined(__GNUC__) && ( \ + (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \ + || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ + || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ + ) + +static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { + void* res; + const size_t ofs = (slot*sizeof(void*)); + #if defined(__i386__) + __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS + #elif defined(__APPLE__) && defined(__x86_64__) + __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS + #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI + #elif defined(__x86_64__) + __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS + #elif defined(__arm__) + void** tcb; MI_UNUSED(ofs); + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + res = tcb[slot]; + #elif defined(__aarch64__) + void** tcb; MI_UNUSED(ofs); + #if defined(__APPLE__) // M1, issue #343 + __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + res = tcb[slot]; + #endif + return res; +} + +// setting a tls slot is only used on macOS for now +static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { + const size_t ofs = (slot*sizeof(void*)); + #if defined(__i386__) + __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS + #elif defined(__APPLE__) && defined(__x86_64__) + __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS + #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI + #elif defined(__x86_64__) + __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS + #elif defined(__arm__) + void** tcb; MI_UNUSED(ofs); + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + tcb[slot] = value; + #elif defined(__aarch64__) + void** tcb; MI_UNUSED(ofs); + #if defined(__APPLE__) // M1, issue #343 + __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + tcb[slot] = value; + #endif +} + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + #if defined(__BIONIC__) + // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id + // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86 + return (uintptr_t)mi_tls_slot(1); + #else + // in all our other targets, slot 0 is the thread id + // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h + // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36 + return (uintptr_t)mi_tls_slot(0); + #endif +} + +#else + +// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + return (uintptr_t)&_mi_heap_default; +} + +#endif + + #endif // MIMALLOC_PRIM_H From 9b110090b2f79ab202c01bd806c867122f9d729f Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 14 Mar 2023 20:35:00 -0700 Subject: [PATCH 031/102] move threadid and mi_get_default_heap to primitives --- include/mimalloc-internal.h | 94 ++--------------------------------- src/alloc-aligned.c | 27 +++++----- src/alloc.c | 38 +++++++------- src/heap.c | 14 ++++-- src/init.c | 6 +-- src/os.c | 4 +- src/page.c | 5 +- src/prim/prim.h | 99 +++++++++++++++++++++++++++++++++++-- src/random.c | 2 + 9 files changed, 152 insertions(+), 137 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 6e6da9f3..5843f60f 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -73,8 +73,9 @@ extern mi_decl_cache_align mi_stats_t _mi_stats_main; extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); size_t _mi_current_thread_count(void); -bool _mi_preloading(void); // true while the C runtime is not ready +bool _mi_preloading(void); // true while the C runtime is not ready mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; +mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap // os.c size_t _mi_os_page_size(void); @@ -329,93 +330,11 @@ static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* tot } -/* ---------------------------------------------------------------------------------------- -The thread local default heap: `_mi_get_default_heap` returns the thread local heap. -On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a -__thread local variable (`_mi_heap_default`). With the initial-exec TLS model this ensures -that the storage will always be available (allocated on the thread stacks). -On some platforms though we cannot use that when overriding `malloc` since the underlying -TLS implementation (or the loader) will call itself `malloc` on a first access and recurse. -We try to circumvent this in an efficient way: -- macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the - loader itself calls `malloc` even before the modules are initialized. -- OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS). -- DragonFly: defaults are working but seem slow compared to freeBSD (see PR #323) +/*---------------------------------------------------------------------------------------- + Heap functions ------------------------------------------------------------------------------------------- */ extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap -extern bool _mi_process_is_initialized; -mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap - -#if defined(MI_MALLOC_OVERRIDE) -#if defined(__APPLE__) // macOS -#define MI_TLS_SLOT 89 // seems unused? -// #define MI_TLS_RECURSE_GUARD 1 -// other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) -// see -#elif defined(__OpenBSD__) -// use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16) -// see -#define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) -// #elif defined(__DragonFly__) -// #warning "mimalloc is not working correctly on DragonFly yet." -// #define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) -#elif defined(__ANDROID__) -// See issue #381 -#define MI_TLS_PTHREAD -#endif -#endif - -#if defined(MI_TLS_SLOT) -static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept; // forward declaration -#elif defined(MI_TLS_PTHREAD_SLOT_OFS) -static inline mi_heap_t** mi_tls_pthread_heap_slot(void) { - pthread_t self = pthread_self(); - #if defined(__DragonFly__) - if (self==NULL) { - mi_heap_t* pheap_main = _mi_heap_main_get(); - return &pheap_main; - } - #endif - return (mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS); -} -#elif defined(MI_TLS_PTHREAD) -extern pthread_key_t _mi_heap_default_key; -#endif - -// Default heap to allocate from (if not using TLS- or pthread slots). -// Do not use this directly but use through `mi_heap_get_default()` (or the unchecked `mi_get_default_heap`). -// This thread local variable is only used when neither MI_TLS_SLOT, MI_TLS_PTHREAD, or MI_TLS_PTHREAD_SLOT_OFS are defined. -// However, on the Apple M1 we do use the address of this variable as the unique thread-id (issue #356). -extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from - -static inline mi_heap_t* mi_get_default_heap(void) { -#if defined(MI_TLS_SLOT) - mi_heap_t* heap = (mi_heap_t*)mi_tls_slot(MI_TLS_SLOT); - if mi_unlikely(heap == NULL) { - #ifdef __GNUC__ - __asm(""); // prevent conditional load of the address of _mi_heap_empty - #endif - heap = (mi_heap_t*)&_mi_heap_empty; - } - return heap; -#elif defined(MI_TLS_PTHREAD_SLOT_OFS) - mi_heap_t* heap = *mi_tls_pthread_heap_slot(); - return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); -#elif defined(MI_TLS_PTHREAD) - mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key)); - return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); -#else - #if defined(MI_TLS_RECURSE_GUARD) - if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get(); - #endif - return _mi_heap_default; -#endif -} - -static inline bool mi_heap_is_default(const mi_heap_t* heap) { - return (heap == mi_get_default_heap()); -} static inline bool mi_heap_is_backing(const mi_heap_t* heap) { return (heap->tld->heap_backing == heap); @@ -443,11 +362,6 @@ static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t si return heap->pages_free_direct[idx]; } -// Get the page belonging to a certain size class -static inline mi_page_t* _mi_get_free_small_page(size_t size) { - return _mi_heap_get_free_small_page(mi_get_default_heap(), size); -} - // Segment that contains the pointer // Large aligned blocks may be aligned at N*MI_SEGMENT_SIZE (inside a huge segment > MI_SEGMENT_SIZE), // and we need align "down" to the segment info which is `MI_SEGMENT_SIZE` bytes before it; diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 08ad9814..aa9b7bd0 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -7,8 +7,9 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" +#include "prim/prim.h" // mi_prim_get_default_heap -#include // memset +#include // memset // ------------------------------------------------------ // Aligned Allocation @@ -187,27 +188,27 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, } mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_heap_malloc_aligned_at(mi_get_default_heap(), size, alignment, offset); + return mi_heap_malloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { - return mi_heap_malloc_aligned(mi_get_default_heap(), size, alignment); + return mi_heap_malloc_aligned(mi_prim_get_default_heap(), size, alignment); } mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_heap_zalloc_aligned_at(mi_get_default_heap(), size, alignment, offset); + return mi_heap_zalloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset); } mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { - return mi_heap_zalloc_aligned(mi_get_default_heap(), size, alignment); + return mi_heap_zalloc_aligned(mi_prim_get_default_heap(), size, alignment); } mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_heap_calloc_aligned_at(mi_get_default_heap(), count, size, alignment, offset); + return mi_heap_calloc_aligned_at(mi_prim_get_default_heap(), count, size, alignment, offset); } mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept { - return mi_heap_calloc_aligned(mi_get_default_heap(), count, size, alignment); + return mi_heap_calloc_aligned(mi_prim_get_default_heap(), count, size, alignment); } @@ -282,25 +283,25 @@ mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_ } mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_heap_realloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset); + return mi_heap_realloc_aligned_at(mi_prim_get_default_heap(), p, newsize, alignment, offset); } mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_heap_realloc_aligned(mi_get_default_heap(), p, newsize, alignment); + return mi_heap_realloc_aligned(mi_prim_get_default_heap(), p, newsize, alignment); } mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_heap_rezalloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset); + return mi_heap_rezalloc_aligned_at(mi_prim_get_default_heap(), p, newsize, alignment, offset); } mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { - return mi_heap_rezalloc_aligned(mi_get_default_heap(), p, newsize, alignment); + return mi_heap_rezalloc_aligned(mi_prim_get_default_heap(), p, newsize, alignment); } mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_heap_recalloc_aligned_at(mi_get_default_heap(), p, newcount, size, alignment, offset); + return mi_heap_recalloc_aligned_at(mi_prim_get_default_heap(), p, newcount, size, alignment, offset); } mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { - return mi_heap_recalloc_aligned(mi_get_default_heap(), p, newcount, size, alignment); + return mi_heap_recalloc_aligned(mi_prim_get_default_heap(), p, newcount, size, alignment); } diff --git a/src/alloc.c b/src/alloc.c index d8276df9..0bda4db8 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -106,7 +106,7 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, mi_track_malloc(p,size,zero); #if MI_STAT>1 if (p != NULL) { - if (!mi_heap_is_initialized(heap)) { heap = mi_get_default_heap(); } + if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); } #endif @@ -119,7 +119,7 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc_small(mi_h } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { - return mi_heap_malloc_small(mi_get_default_heap(), size); + return mi_heap_malloc_small(mi_prim_get_default_heap(), size); } // The main allocation function @@ -135,7 +135,7 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z mi_track_malloc(p,size,zero); #if MI_STAT>1 if (p != NULL) { - if (!mi_heap_is_initialized(heap)) { heap = mi_get_default_heap(); } + if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); } #endif @@ -152,12 +152,12 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { - return mi_heap_malloc(mi_get_default_heap(), size); + return mi_heap_malloc(mi_prim_get_default_heap(), size); } // zero initialized small block mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { - return mi_heap_malloc_small_zero(mi_get_default_heap(), size, true); + return mi_heap_malloc_small_zero(mi_prim_get_default_heap(), size, true); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -165,7 +165,7 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* } mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { - return mi_heap_zalloc(mi_get_default_heap(),size); + return mi_heap_zalloc(mi_prim_get_default_heap(),size); } @@ -649,7 +649,7 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_calloc(mi_heap_t* } mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { - return mi_heap_calloc(mi_get_default_heap(),count,size); + return mi_heap_calloc(mi_prim_get_default_heap(),count,size); } // Uninitialized `calloc` @@ -660,7 +660,7 @@ mi_decl_nodiscard extern mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, } mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { - return mi_heap_mallocn(mi_get_default_heap(),count,size); + return mi_heap_mallocn(mi_prim_get_default_heap(),count,size); } // Expand (or shrink) in place (or fail) @@ -737,24 +737,24 @@ mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { - return mi_heap_realloc(mi_get_default_heap(),p,newsize); + return mi_heap_realloc(mi_prim_get_default_heap(),p,newsize); } mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { - return mi_heap_reallocn(mi_get_default_heap(),p,count,size); + return mi_heap_reallocn(mi_prim_get_default_heap(),p,count,size); } // Reallocate but free `p` on errors mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { - return mi_heap_reallocf(mi_get_default_heap(),p,newsize); + return mi_heap_reallocf(mi_prim_get_default_heap(),p,newsize); } mi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept { - return mi_heap_rezalloc(mi_get_default_heap(), p, newsize); + return mi_heap_rezalloc(mi_prim_get_default_heap(), p, newsize); } mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { - return mi_heap_recalloc(mi_get_default_heap(), p, count, size); + return mi_heap_recalloc(mi_prim_get_default_heap(), p, count, size); } @@ -775,7 +775,7 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const c } mi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept { - return mi_heap_strdup(mi_get_default_heap(), s); + return mi_heap_strdup(mi_prim_get_default_heap(), s); } // `strndup` using mi_malloc @@ -792,7 +792,7 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const } mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept { - return mi_heap_strndup(mi_get_default_heap(),s,n); + return mi_heap_strndup(mi_prim_get_default_heap(),s,n); } #ifndef __wasi__ @@ -861,7 +861,7 @@ char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) #endif mi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept { - return mi_heap_realpath(mi_get_default_heap(),fname,resolved_name); + return mi_heap_realpath(mi_prim_get_default_heap(),fname,resolved_name); } #endif @@ -937,7 +937,7 @@ mi_decl_export mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t si } static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) { - return mi_heap_try_new(mi_get_default_heap(), size, nothrow); + return mi_heap_try_new(mi_prim_get_default_heap(), size, nothrow); } @@ -948,7 +948,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size } mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) { - return mi_heap_alloc_new(mi_get_default_heap(), size); + return mi_heap_alloc_new(mi_prim_get_default_heap(), size); } @@ -964,7 +964,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si } mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { - return mi_heap_alloc_new_n(mi_get_default_heap(), size, count); + return mi_heap_alloc_new_n(mi_prim_get_default_heap(), size, count); } diff --git a/src/heap.c b/src/heap.c index 94fed2b5..b12a9962 100644 --- a/src/heap.c +++ b/src/heap.c @@ -9,6 +9,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-internal.h" #include "mimalloc-atomic.h" #include "mimalloc-track.h" +#include "prim/prim.h" // mi_prim_get_default_heap #include // memset, memcpy @@ -168,7 +169,7 @@ void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept { } void mi_collect(bool force) mi_attr_noexcept { - mi_heap_collect(mi_get_default_heap(), force); + mi_heap_collect(mi_prim_get_default_heap(), force); } @@ -178,9 +179,14 @@ void mi_collect(bool force) mi_attr_noexcept { mi_heap_t* mi_heap_get_default(void) { mi_thread_init(); - return mi_get_default_heap(); + return mi_prim_get_default_heap(); } +static bool mi_heap_is_default(const mi_heap_t* heap) { + return (heap == mi_prim_get_default_heap()); +} + + mi_heap_t* mi_heap_get_backing(void) { mi_heap_t* heap = mi_heap_get_default(); mi_assert_internal(heap!=NULL); @@ -418,7 +424,7 @@ mi_heap_t* mi_heap_set_default(mi_heap_t* heap) { mi_assert(mi_heap_is_initialized(heap)); if (heap==NULL || !mi_heap_is_initialized(heap)) return NULL; mi_assert_expensive(mi_heap_is_valid(heap)); - mi_heap_t* old = mi_get_default_heap(); + mi_heap_t* old = mi_prim_get_default_heap(); _mi_heap_set_default_direct(heap); return old; } @@ -468,7 +474,7 @@ bool mi_heap_check_owned(mi_heap_t* heap, const void* p) { } bool mi_check_owned(const void* p) { - return mi_heap_check_owned(mi_get_default_heap(), p); + return mi_heap_check_owned(mi_prim_get_default_heap(), p); } /* ----------------------------------------------------------- diff --git a/src/init.c b/src/init.c index dd555030..2d8f0df2 100644 --- a/src/init.c +++ b/src/init.c @@ -239,13 +239,13 @@ static void mi_thread_data_collect(void) { // Initialize the thread local default heap, called from `mi_thread_init` static bool _mi_heap_init(void) { - if (mi_heap_is_initialized(mi_get_default_heap())) return true; + if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true; if (_mi_is_main_thread()) { // mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization // the main heap is statically allocated mi_heap_main_init(); _mi_heap_set_default_direct(&_mi_heap_main); - //mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_get_default_heap()); + //mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_prim_get_default_heap()); } else { // use `_mi_os_alloc` to allocate directly from the OS @@ -418,7 +418,7 @@ void mi_thread_init(void) mi_attr_noexcept } void mi_thread_done(void) mi_attr_noexcept { - _mi_thread_done(mi_get_default_heap()); + _mi_thread_done(mi_prim_get_default_heap()); } static void _mi_thread_done(mi_heap_t* heap) { diff --git a/src/os.c b/src/os.c index 8ef72e04..8fb7b781 100644 --- a/src/os.c +++ b/src/os.c @@ -120,7 +120,7 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode - uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); + uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB #endif uintptr_t expected = hint + size; @@ -483,7 +483,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { // Initialize the start address after the 32TiB area start = ((uintptr_t)32 << 40); // 32TiB virtual start address #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode - uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); + uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB #endif } diff --git a/src/page.c b/src/page.c index 46dfd26f..1347c845 100644 --- a/src/page.c +++ b/src/page.c @@ -103,6 +103,8 @@ static bool mi_page_is_valid_init(mi_page_t* page) { return true; } +extern bool _mi_process_is_initialized; // has mi_process_init been called? + bool _mi_page_is_valid(mi_page_t* page) { mi_assert_internal(mi_page_is_valid_init(page)); #if MI_SECURE @@ -873,8 +875,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_al // initialize if necessary if mi_unlikely(!mi_heap_is_initialized(heap)) { - mi_thread_init(); // calls `_mi_heap_init` in turn - heap = mi_get_default_heap(); + heap = mi_heap_get_default(); // calls mi_thread_init if mi_unlikely(!mi_heap_is_initialized(heap)) { return NULL; } } mi_assert_internal(mi_heap_is_initialized(heap)); diff --git a/src/prim/prim.h b/src/prim/prim.h index 2d951930..272fd72f 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -104,7 +104,7 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ ) -static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { +static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept { void* res; const size_t ofs = (slot*sizeof(void*)); #if defined(__i386__) @@ -132,7 +132,7 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { } // setting a tls slot is only used on macOS for now -static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { +static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { const size_t ofs = (slot*sizeof(void*)); #if defined(__i386__) __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS @@ -161,12 +161,12 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { #if defined(__BIONIC__) // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86 - return (uintptr_t)mi_tls_slot(1); + return (uintptr_t)mi_prim_tls_slot(1); #else // in all our other targets, slot 0 is the thread id // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36 - return (uintptr_t)mi_tls_slot(0); + return (uintptr_t)mi_prim_tls_slot(0); #endif } @@ -180,4 +180,95 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { #endif + +/* ---------------------------------------------------------------------------------------- +The thread local default heap: `_mi_prim_get_default_heap()` +This is inlined here as it is on the fast path for allocation functions. + +On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a +__thread local variable (`_mi_heap_default`). With the initial-exec TLS model this ensures +that the storage will always be available (allocated on the thread stacks). +On some platforms though we cannot use that when overriding `malloc` since the underlying +TLS implementation (or the loader) will call itself `malloc` on a first access and recurse. +We try to circumvent this in an efficient way: +- macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the + loader itself calls `malloc` even before the modules are initialized. +- OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS). +- DragonFly: defaults are working but seem slow compared to freeBSD (see PR #323) +------------------------------------------------------------------------------------------- */ + +// defined in `init.c`; do not use these directly +extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from +extern bool _mi_process_is_initialized; // has mi_process_init been called? + +static inline mi_heap_t* mi_prim_get_default_heap(void); + +#if defined(MI_MALLOC_OVERRIDE) +#if defined(__APPLE__) // macOS + #define MI_TLS_SLOT 89 // seems unused? + // #define MI_TLS_RECURSE_GUARD 1 + // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) + // see +#elif defined(__OpenBSD__) + // use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16) + // see + #define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) + // #elif defined(__DragonFly__) + // #warning "mimalloc is not working correctly on DragonFly yet." + // #define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) +#elif defined(__ANDROID__) + // See issue #381 + #define MI_TLS_PTHREAD +#endif +#endif + + +#if defined(MI_TLS_SLOT) + +static inline mi_heap_t* mi_prim_get_default_heap(void) { + mi_heap_t* heap = (mi_heap_t*)mi_prim_tls_slot(MI_TLS_SLOT); + if mi_unlikely(heap == NULL) { + #ifdef __GNUC__ + __asm(""); // prevent conditional load of the address of _mi_heap_empty + #endif + heap = (mi_heap_t*)&_mi_heap_empty; + } + return heap; +} + +#elif defined(MI_TLS_PTHREAD_SLOT_OFS) + +static inline mi_heap_t* mi_prim_get_default_heap(void) { + mi_heap_t* heap; + pthread_t self = pthread_self(); + #if defined(__DragonFly__) + if (self==NULL) { heap = _mi_heap_main_get(); } else + #endif + { + heap = *((mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS)); + } + return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); +} + +#elif defined(MI_TLS_PTHREAD) + +extern pthread_key_t _mi_heap_default_key; +static inline mi_heap_t* mi_prim_get_default_heap(void) { + mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key)); + return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); +} + +#else // default using a thread local variable; used on most platforms. + +static inline mi_heap_t* mi_prim_get_default_heap(void) { + #if defined(MI_TLS_RECURSE_GUARD) + if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get(); + #endif + return _mi_heap_default; +} + +#endif // mi_prim_get_default_heap() + + + #endif // MIMALLOC_PRIM_H diff --git a/src/random.c b/src/random.c index 06d4ba4a..ad14bf1f 100644 --- a/src/random.c +++ b/src/random.c @@ -168,6 +168,8 @@ If we cannot get good randomness, we fall back to weak randomness based on a tim #if defined(_WIN32) +#include + #if defined(MI_USE_RTLGENRANDOM) // || defined(__cplusplus) // We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using // dynamic overriding, we observed it can raise an exception when compiled with C++, and From 973268bf1ed92713b413defd5de27d7d3309fd7f Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 15 Mar 2023 12:40:18 -0700 Subject: [PATCH 032/102] move random initialization to primitives --- src/prim/prim-unix.c | 90 +++++++++++++++++++++- src/prim/prim-wasi.c | 11 ++- src/prim/prim-windows.c | 42 +++++++++++ src/prim/prim.h | 4 + src/random.c | 162 ++-------------------------------------- 5 files changed, 150 insertions(+), 159 deletions(-) diff --git a/src/prim/prim-unix.c b/src/prim/prim-unix.c index ce24c24a..6b593fd3 100644 --- a/src/prim/prim-unix.c +++ b/src/prim/prim-unix.c @@ -6,7 +6,7 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #ifndef _DEFAULT_SOURCE -#define _DEFAULT_SOURCE // ensure mmap flags are defined +#define _DEFAULT_SOURCE // ensure mmap flags and syscall are defined #endif #if defined(__sun) @@ -661,3 +661,91 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { return true; } #endif // !MI_USE_ENVIRON + + +//---------------------------------------------------------------- +// Random +//---------------------------------------------------------------- + +#if defined(__APPLE__) + +#include +#if defined(MAC_OS_X_VERSION_10_10) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_10 +#include +#include +#endif +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + #if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 + // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf + // may fail silently on macOS. See PR #390, and + return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); + #else + // fall back on older macOS + arc4random_buf(buf, buf_len); + return true; + #endif +} + +#elif defined(__ANDROID__) || defined(__DragonFly__) || \ + defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__sun) + +#include +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + arc4random_buf(buf, buf_len); + return true; +} + +#elif defined(__linux__) || defined(__HAIKU__) + +#if defined(__linux__) +#include +#endif +#include +#include +#include +#include +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + // Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h` + // and for the latter the actual `getrandom` call is not always defined. + // (see ) + // We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed. + #ifdef SYS_getrandom + #ifndef GRND_NONBLOCK + #define GRND_NONBLOCK (1) + #endif + static _Atomic(uintptr_t) no_getrandom; // = 0 + if (mi_atomic_load_acquire(&no_getrandom)==0) { + ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK); + if (ret >= 0) return (buf_len == (size_t)ret); + if (errno != ENOSYS) return false; + mi_atomic_store_release(&no_getrandom, 1UL); // don't call again, and fall back to /dev/urandom + } + #endif + int flags = O_RDONLY; + #if defined(O_CLOEXEC) + flags |= O_CLOEXEC; + #endif + int fd = open("/dev/urandom", flags, 0); + if (fd < 0) return false; + size_t count = 0; + while(count < buf_len) { + ssize_t ret = read(fd, (char*)buf + count, buf_len - count); + if (ret<=0) { + if (errno!=EAGAIN && errno!=EINTR) break; + } + else { + count += ret; + } + } + close(fd); + return (count==buf_len); +} + +#else + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + return false; +} + +#endif \ No newline at end of file diff --git a/src/prim/prim-wasi.c b/src/prim/prim-wasi.c index c37b0847..38efece0 100644 --- a/src/prim/prim-wasi.c +++ b/src/prim/prim-wasi.c @@ -233,4 +233,13 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false; _mi_strlcpy(result, s, result_size); return true; -} \ No newline at end of file +} + + +//---------------------------------------------------------------- +// Random +//---------------------------------------------------------------- + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + return false; +} diff --git a/src/prim/prim-windows.c b/src/prim/prim-windows.c index f6f1a9db..3b791de7 100644 --- a/src/prim/prim-windows.c +++ b/src/prim/prim-windows.c @@ -506,3 +506,45 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size); return (len > 0 && len < result_size); } + + + +//---------------------------------------------------------------- +// Random +//---------------------------------------------------------------- + +#if defined(MI_USE_RTLGENRANDOM) // || defined(__cplusplus) +// We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using +// dynamic overriding, we observed it can raise an exception when compiled with C++, and +// sometimes deadlocks when also running under the VS debugger. +// In contrast, issue #623 implies that on Windows Server 2019 we need to use BCryptGenRandom. +// To be continued.. +#pragma comment (lib,"advapi32.lib") +#define RtlGenRandom SystemFunction036 +mi_decl_externc BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength); + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + return (RtlGenRandom(buf, (ULONG)buf_len) != 0); +} + +#else + +#ifndef BCRYPT_USE_SYSTEM_PREFERRED_RNG +#define BCRYPT_USE_SYSTEM_PREFERRED_RNG 0x00000002 +#endif + +typedef LONG (NTAPI *PBCryptGenRandom)(HANDLE, PUCHAR, ULONG, ULONG); +static PBCryptGenRandom pBCryptGenRandom = NULL; + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + if (pBCryptGenRandom == NULL) { + HINSTANCE hDll = LoadLibrary(TEXT("bcrypt.dll")); + if (hDll != NULL) { + pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom"); + } + if (pBCryptGenRandom == NULL) return false; + } + return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0); +} + +#endif // MI_USE_RTLGENRANDOM diff --git a/src/prim/prim.h b/src/prim/prim.h index 272fd72f..c067046d 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -72,6 +72,10 @@ void _mi_prim_out_stderr( const char* msg ); bool _mi_prim_getenv(const char* name, char* result, size_t result_size); +// Fill a buffer with strong randomness; return `false` on error or if +// there is no strong randomization available. +bool _mi_prim_random_buf(void* buf, size_t buf_len); + //------------------------------------------------------------------- // Thread id // diff --git a/src/random.c b/src/random.c index ad14bf1f..3c8372c8 100644 --- a/src/random.c +++ b/src/random.c @@ -4,14 +4,10 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#ifndef _DEFAULT_SOURCE -#define _DEFAULT_SOURCE // for syscall() on Linux -#endif - #include "mimalloc.h" #include "mimalloc-internal.h" - -#include // memset +#include "prim/prim.h" // _mi_prim_random_buf +#include // memset /* ---------------------------------------------------------------------------- We use our own PRNG to keep predictable performance of random number generation @@ -158,161 +154,13 @@ uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { /* ---------------------------------------------------------------------------- -To initialize a fresh random context we rely on the OS: -- Windows : BCryptGenRandom (or RtlGenRandom) -- macOS : CCRandomGenerateBytes, arc4random_buf -- bsd,wasi : arc4random_buf -- Linux : getrandom,/dev/urandom +To initialize a fresh random context. If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. -----------------------------------------------------------------------------*/ -#if defined(_WIN32) - -#include - -#if defined(MI_USE_RTLGENRANDOM) // || defined(__cplusplus) -// We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using -// dynamic overriding, we observed it can raise an exception when compiled with C++, and -// sometimes deadlocks when also running under the VS debugger. -// In contrast, issue #623 implies that on Windows Server 2019 we need to use BCryptGenRandom. -// To be continued.. -#pragma comment (lib,"advapi32.lib") -#define RtlGenRandom SystemFunction036 -#ifdef __cplusplus -extern "C" { -#endif -BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength); -#ifdef __cplusplus -} -#endif -static bool os_random_buf(void* buf, size_t buf_len) { - return (RtlGenRandom(buf, (ULONG)buf_len) != 0); -} -#else - -#ifndef BCRYPT_USE_SYSTEM_PREFERRED_RNG -#define BCRYPT_USE_SYSTEM_PREFERRED_RNG 0x00000002 -#endif - -typedef LONG (NTAPI *PBCryptGenRandom)(HANDLE, PUCHAR, ULONG, ULONG); -static PBCryptGenRandom pBCryptGenRandom = NULL; - -static bool os_random_buf(void* buf, size_t buf_len) { - if (pBCryptGenRandom == NULL) { - HINSTANCE hDll = LoadLibrary(TEXT("bcrypt.dll")); - if (hDll != NULL) { - pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom"); - } - } - if (pBCryptGenRandom == NULL) { - return false; - } - else { - return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0); - } -} -#endif - -#elif defined(__APPLE__) -#include -#if defined(MAC_OS_X_VERSION_10_10) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_10 -#include -#include -#endif -static bool os_random_buf(void* buf, size_t buf_len) { - #if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 - // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf - // may fail silently on macOS. See PR #390, and - return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); - #else - // fall back on older macOS - arc4random_buf(buf, buf_len); - return true; - #endif -} - -#elif defined(__ANDROID__) || defined(__DragonFly__) || \ - defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__sun) // todo: what to use with __wasi__? -#include -static bool os_random_buf(void* buf, size_t buf_len) { - arc4random_buf(buf, buf_len); - return true; -} -#elif defined(__linux__) || defined(__HAIKU__) -#if defined(__linux__) -#include -#endif -#include -#include -#include -#include -#include -static bool os_random_buf(void* buf, size_t buf_len) { - // Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h` - // and for the latter the actual `getrandom` call is not always defined. - // (see ) - // We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed. -#ifdef SYS_getrandom - #ifndef GRND_NONBLOCK - #define GRND_NONBLOCK (1) - #endif - static _Atomic(uintptr_t) no_getrandom; // = 0 - if (mi_atomic_load_acquire(&no_getrandom)==0) { - ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK); - if (ret >= 0) return (buf_len == (size_t)ret); - if (errno != ENOSYS) return false; - mi_atomic_store_release(&no_getrandom, 1UL); // don't call again, and fall back to /dev/urandom - } -#endif - int flags = O_RDONLY; - #if defined(O_CLOEXEC) - flags |= O_CLOEXEC; - #endif - int fd = open("/dev/urandom", flags, 0); - if (fd < 0) return false; - size_t count = 0; - while(count < buf_len) { - ssize_t ret = read(fd, (char*)buf + count, buf_len - count); - if (ret<=0) { - if (errno!=EAGAIN && errno!=EINTR) break; - } - else { - count += ret; - } - } - close(fd); - return (count==buf_len); -} -#else -static bool os_random_buf(void* buf, size_t buf_len) { - return false; -} -#endif - -#if defined(_WIN32) -#include -#elif defined(__APPLE__) -#include -#else -#include -#endif - uintptr_t _mi_os_random_weak(uintptr_t extra_seed) { uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random - - #if defined(_WIN32) - LARGE_INTEGER pcount; - QueryPerformanceCounter(&pcount); - x ^= (uintptr_t)(pcount.QuadPart); - #elif defined(__APPLE__) - x ^= (uintptr_t)mach_absolute_time(); - #else - struct timespec time; - clock_gettime(CLOCK_MONOTONIC, &time); - x ^= (uintptr_t)time.tv_sec; - x ^= (uintptr_t)time.tv_nsec; - #endif + x ^= _mi_prim_clock_now(); // and do a few randomization steps uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1; for (uintptr_t i = 0; i < max; i++) { @@ -324,7 +172,7 @@ uintptr_t _mi_os_random_weak(uintptr_t extra_seed) { static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { uint8_t key[32]; - if (use_weak || !os_random_buf(key, sizeof(key))) { + if (use_weak || !_mi_prim_random_buf(key, sizeof(key))) { // if we fail to get random data from the OS, we fall back to a // weak random source based on the current time #if !defined(__wasi__) From 9a2dbf373e2f8665c88950cf65bc11469d9fa94d Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 15 Mar 2023 13:35:23 -0700 Subject: [PATCH 033/102] move thread init to primitives --- include/mimalloc-internal.h | 3 +- src/init.c | 78 ++++++++----------------------------- src/prim/prim-unix.c | 48 +++++++++++++++++++++++ src/prim/prim-wasi.c | 17 ++++++++ src/prim/prim-windows.c | 56 ++++++++++++++++++++++++++ src/prim/prim.h | 10 +++++ 6 files changed, 149 insertions(+), 63 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 5843f60f..36de1342 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -75,7 +75,8 @@ bool _mi_is_main_thread(void); size_t _mi_current_thread_count(void); bool _mi_preloading(void); // true while the C runtime is not ready mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; -mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap +mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap +void _mi_thread_done(mi_heap_t* heap); // os.c size_t _mi_os_page_size(void); diff --git a/src/init.c b/src/init.c index 2d8f0df2..31ec87cb 100644 --- a/src/init.c +++ b/src/init.c @@ -339,54 +339,12 @@ static bool _mi_heap_done(mi_heap_t* heap) { // to set up the thread local keys. // -------------------------------------------------------- -static void _mi_thread_done(mi_heap_t* default_heap); - -#if defined(_WIN32) && defined(MI_SHARED_LIB) - // nothing to do as it is done in DllMain -#elif defined(_WIN32) && !defined(MI_SHARED_LIB) - // use thread local storage keys to detect thread ending - #include - #include - #if (_WIN32_WINNT < 0x600) // before Windows Vista - WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); - WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); - WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); - WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); - #endif - static DWORD mi_fls_key = (DWORD)(-1); - static void NTAPI mi_fls_done(PVOID value) { - mi_heap_t* heap = (mi_heap_t*)value; - if (heap != NULL) { - _mi_thread_done(heap); - FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main heap, issue #672 - } - } -#elif defined(MI_USE_PTHREADS) - // use pthread local storage keys to detect thread ending - // (and used with MI_TLS_PTHREADS for the default heap) - pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1); - static void mi_pthread_done(void* value) { - if (value!=NULL) _mi_thread_done((mi_heap_t*)value); - } -#elif defined(__wasi__) -// no pthreads in the WebAssembly Standard Interface -#else - #pragma message("define a way to call mi_thread_done when a thread is done") -#endif - // Set up handlers so `mi_thread_done` is called automatically static void mi_process_setup_auto_thread_done(void) { static bool tls_initialized = false; // fine if it races if (tls_initialized) return; tls_initialized = true; - #if defined(_WIN32) && defined(MI_SHARED_LIB) - // nothing to do as it is done in DllMain - #elif defined(_WIN32) && !defined(MI_SHARED_LIB) - mi_fls_key = FlsAlloc(&mi_fls_done); - #elif defined(MI_USE_PTHREADS) - mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1)); - pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); - #endif + _mi_prim_thread_init_auto_done(); _mi_heap_set_default_direct(&_mi_heap_main); } @@ -418,13 +376,19 @@ void mi_thread_init(void) mi_attr_noexcept } void mi_thread_done(void) mi_attr_noexcept { - _mi_thread_done(mi_prim_get_default_heap()); + _mi_thread_done(NULL); } -static void _mi_thread_done(mi_heap_t* heap) { +void _mi_thread_done(mi_heap_t* heap) +{ mi_atomic_decrement_relaxed(&thread_count); _mi_stat_decrease(&_mi_stats_main.threads, 1); + if (heap == NULL) { + heap = mi_prim_get_default_heap(); + if (heap == NULL) return; + } + // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... if (heap->thread_id != _mi_thread_id()) return; @@ -446,16 +410,7 @@ void _mi_heap_set_default_direct(mi_heap_t* heap) { // ensure the default heap is passed to `_mi_thread_done` // setting to a non-NULL value also ensures `mi_thread_done` is called. - #if defined(_WIN32) && defined(MI_SHARED_LIB) - // nothing to do as it is done in DllMain - #elif defined(_WIN32) && !defined(MI_SHARED_LIB) - mi_assert_internal(mi_fls_key != 0); - FlsSetValue(mi_fls_key, heap); - #elif defined(MI_USE_PTHREADS) - if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD - pthread_setspecific(_mi_heap_default_key, heap); - } - #endif + _mi_prim_thread_associate_default_heap(heap); } @@ -570,11 +525,11 @@ void mi_process_init(void) mi_attr_noexcept { _mi_verbose_message("mem tracking: %s\n", MI_TRACK_TOOL); mi_thread_init(); - #if defined(_WIN32) && !defined(MI_SHARED_LIB) - // When building as a static lib the FLS cleanup happens to early for the main thread. + #if defined(_WIN32) + // On windows, when building as a static lib the FLS cleanup happens to early for the main thread. // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup // will not call _mi_thread_done on the (still executing) main thread. See issue #508. - FlsSetValue(mi_fls_key, NULL); + _mi_prim_thread_associate_default_heap(NULL); #endif mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL) @@ -605,10 +560,9 @@ static void mi_cdecl mi_process_done(void) { if (process_done) return; process_done = true; - #if defined(_WIN32) && !defined(MI_SHARED_LIB) - FlsFree(mi_fls_key); // call thread-done on all threads (except the main thread) to prevent dangling callback pointer if statically linked with a DLL; Issue #208 - #endif - + // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread + _mi_prim_thread_done_auto_done(); + #ifndef MI_SKIP_COLLECT_ON_EXIT #if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB) // free all memory if possible on process exit. This is not needed for a stand-alone process diff --git a/src/prim/prim-unix.c b/src/prim/prim-unix.c index 6b593fd3..9270e088 100644 --- a/src/prim/prim-unix.c +++ b/src/prim/prim-unix.c @@ -748,4 +748,52 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { return false; } +#endif + + +//---------------------------------------------------------------- +// Thread init/done +//---------------------------------------------------------------- + +#if defined(MI_USE_PTHREADS) + +// use pthread local storage keys to detect thread ending +// (and used with MI_TLS_PTHREADS for the default heap) +pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1); + +static void mi_pthread_done(void* value) { + if (value!=NULL) { + _mi_thread_done((mi_heap_t*)value); + } +} + +void _mi_prim_thread_init_auto_done(void) { + mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1)); + pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); +} + +void _mi_prim_thread_done_auto_done(void) { + // nothing to do +} + +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD + pthread_setspecific(_mi_heap_default_key, heap); + } +} + +#else + +void _mi_prim_thread_init_auto_done(void) { + // nothing +} + +void _mi_prim_thread_done_auto_done(void) { + // nothing +} + +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + MI_UNUSED(heap); +} + #endif \ No newline at end of file diff --git a/src/prim/prim-wasi.c b/src/prim/prim-wasi.c index 38efece0..f4d3be58 100644 --- a/src/prim/prim-wasi.c +++ b/src/prim/prim-wasi.c @@ -243,3 +243,20 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { bool _mi_prim_random_buf(void* buf, size_t buf_len) { return false; } + + +//---------------------------------------------------------------- +// Thread init/done +//---------------------------------------------------------------- + +void _mi_prim_thread_init_auto_done(void) { + // nothing +} + +void _mi_prim_thread_done_auto_done(void) { + // nothing +} + +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + MI_UNUSED(heap); +} diff --git a/src/prim/prim-windows.c b/src/prim/prim-windows.c index 3b791de7..008b9fa4 100644 --- a/src/prim/prim-windows.c +++ b/src/prim/prim-windows.c @@ -548,3 +548,59 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { } #endif // MI_USE_RTLGENRANDOM + +//---------------------------------------------------------------- +// Thread init/done +//---------------------------------------------------------------- + +#if !defined(MI_SHARED_LIB) + +// use thread local storage keys to detect thread ending +#include +#if (_WIN32_WINNT < 0x600) // before Windows Vista +WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); +WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); +WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); +WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); +#endif + +static DWORD mi_fls_key = (DWORD)(-1); + +static void NTAPI mi_fls_done(PVOID value) { + mi_heap_t* heap = (mi_heap_t*)value; + if (heap != NULL) { + _mi_thread_done(heap); + FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main heap, issue #672 + } +} + +void _mi_prim_thread_init_auto_done(void) { + mi_fls_key = FlsAlloc(&mi_fls_done); +} + +void _mi_prim_thread_done_auto_done(void) { + // call thread-done on all threads (except the main thread) to prevent + // dangling callback pointer if statically linked with a DLL; Issue #208 + FlsFree(mi_fls_key); +} + +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + mi_assert_internal(mi_fls_key != (DWORD)(-1)); + FlsSetValue(mi_fls_key, heap); +} + +#else + +// Dll; nothing to do as in that case thread_done is handled through the DLL_THREAD_DETACH event. + +void _mi_prim_thread_init_auto_done(void) { +} + +void _mi_prim_thread_done_auto_done(void) { +} + +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + MI_UNUSED(heap); +} + +#endif diff --git a/src/prim/prim.h b/src/prim/prim.h index c067046d..5ed0f2e0 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -76,6 +76,16 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size); // there is no strong randomization available. bool _mi_prim_random_buf(void* buf, size_t buf_len); +// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination. +void _mi_prim_thread_init_auto_done(void); + +// Called on process exit and may take action to clean up resources associated with the thread auto done. +void _mi_prim_thread_done_auto_done(void); + +// Called when the default heap for a thread changes +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); + + //------------------------------------------------------------------- // Thread id // From 84ef963a479eba52a016ae9eed938f450e77094f Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 15 Mar 2023 14:43:35 -0700 Subject: [PATCH 034/102] remove conioinclude --- src/options.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/options.c b/src/options.c index 4c76ae41..f0c52c84 100644 --- a/src/options.c +++ b/src/options.c @@ -23,9 +23,6 @@ int mi_version(void) mi_attr_noexcept { return MI_MALLOC_VERSION; } -#ifdef _WIN32 -#include -#endif // -------------------------------------------------------- // Options From 479ef4bf4c8cdfad0f5be274275337a813a23012 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 15 Mar 2023 19:07:35 -0700 Subject: [PATCH 035/102] fix precise free size in aligned allocation --- src/os.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/os.c b/src/os.c index 8fb7b781..a91bbb91 100644 --- a/src/os.c +++ b/src/os.c @@ -142,17 +142,22 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { Free memory -------------------------------------------------------------- */ -void _mi_os_free_ex(void* addr, size_t size, bool was_committed, mi_stats_t* tld_stats) -{ +static void mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats_t* tld_stats) { MI_UNUSED(tld_stats); - mi_stats_t* stats = &_mi_stats_main; + mi_assert_internal((size % _mi_os_page_size()) == 0); if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) - const size_t csize = _mi_os_good_alloc_size(size); - _mi_prim_free(addr, csize); + _mi_prim_free(addr, size); + mi_stats_t* stats = &_mi_stats_main; if (was_committed) { _mi_stat_decrease(&stats->committed, size); } _mi_stat_decrease(&stats->reserved, size); } + +void _mi_os_free_ex(void* addr, size_t size, bool was_committed, mi_stats_t* tld_stats) { + const size_t csize = _mi_os_good_alloc_size(size); + mi_os_mem_free(addr,csize,was_committed,tld_stats); +} + void _mi_os_free(void* p, size_t size, mi_stats_t* tld_stats) { _mi_os_free_ex(p, size, true, tld_stats); } @@ -205,7 +210,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, // if not aligned, free it, overallocate, and unmap around it if (((uintptr_t)p % alignment != 0)) { - _mi_os_free_ex(p, size, commit, stats); + mi_os_mem_free(p, size, commit, stats); _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (%zu bytes, address: %p, alignment: %zu, commit: %d)\n", size, p, alignment, commit); if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; @@ -235,8 +240,8 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t mid_size = _mi_align_up(size, _mi_os_page_size()); size_t post_size = over_size - pre_size - mid_size; mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); - if (pre_size > 0) _mi_os_free_ex(p, pre_size, commit, stats); - if (post_size > 0) _mi_os_free_ex((uint8_t*)aligned_p + mid_size, post_size, commit, stats); + if (pre_size > 0) mi_os_mem_free(p, pre_size, commit, stats); + if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); // we can return the aligned pointer on `mmap` (and sbrk) systems p = aligned_p; } From cfe3d04299be2436d515f9c448db3e461f40228a Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 15 Mar 2023 19:15:53 -0700 Subject: [PATCH 036/102] cleanup --- src/bitmap.h | 2 +- src/prim/prim.h | 14 +++++++------- src/static.c | 21 +++++++++++---------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/bitmap.h b/src/bitmap.h index 39ca55b2..435c5f0a 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019-2020 Microsoft Research, Daan Leijen +Copyright (c) 2019-2023 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/prim/prim.h b/src/prim/prim.h index 5ed0f2e0..5a58a79f 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -22,10 +22,10 @@ typedef struct mi_os_mem_config_s { } mi_os_mem_config_t; // Initialize -void _mi_prim_mem_init( mi_os_mem_config_t* config ); +void _mi_prim_mem_init( mi_os_mem_config_t* config ); // Free OS memory -void _mi_prim_free(void* addr, size_t size ); +void _mi_prim_free(void* addr, size_t size ); // Allocate OS memory. Return NULL on error. // The `try_alignment` is just a hint and the returned pointer does not have to be aligned. @@ -34,14 +34,14 @@ void _mi_prim_free(void* addr, size_t size ); void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large); // Commit memory. Returns error code or 0 on success. -int _mi_prim_commit(void* addr, size_t size, bool commit); +int _mi_prim_commit(void* addr, size_t size, bool commit); // Reset memory. The range keeps being accessible but the content might be reset. // Returns error code or 0 on success. -int _mi_prim_reset(void* addr, size_t size); +int _mi_prim_reset(void* addr, size_t size); // Protect memory. Returns error code or 0 on success. -int _mi_prim_protect(void* addr, size_t size, bool protect); +int _mi_prim_protect(void* addr, size_t size, bool protect); // Allocate huge (1GiB) pages possibly associated with a NUMA node. // pre: size > 0 and a multiple of 1GiB. @@ -59,13 +59,13 @@ size_t _mi_prim_numa_node_count(void); mi_msecs_t _mi_prim_clock_now(void); // Return process information (only for statistics) -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, +void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults); // Default stderr output. (only for warnings etc. with verbose enabled) // msg != NULL && _mi_strlen(msg) > 0 -void _mi_prim_out_stderr( const char* msg ); +void _mi_prim_out_stderr( const char* msg ); // Get an environment variable. (only for options) // name != NULL, result != NULL, result_size >= 64 diff --git a/src/static.c b/src/static.c index 4b3abc28..0de72fe3 100644 --- a/src/static.c +++ b/src/static.c @@ -20,20 +20,21 @@ terms of the MIT license. A copy of the license can be found in the file // containing the whole library. If it is linked first // it will override all the standard library allocation // functions (on Unix's). -#include "stats.c" -#include "random.c" -#include "os.c" -#include "bitmap.c" -#include "arena.c" -#include "region.c" -#include "segment.c" -#include "page.c" -#include "heap.c" #include "alloc.c" #include "alloc-aligned.c" -#include "alloc-posix.c" #if MI_OSX_ZONE #include "alloc-override-osx.c" #endif +#include "alloc-posix.c" +#include "arena.c" +#include "bitmap.c" +#include "heap.c" #include "init.c" #include "options.c" +#include "os.c" +#include "page.c" +#include "prim/prim.c" +#include "random.c" +#include "region.c" +#include "segment.c" +#include "stats.c" From 9fb4f2a501450beb394af85d476d9f412be0a680 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 15 Mar 2023 19:25:18 -0700 Subject: [PATCH 037/102] update vs2019 ide --- ide/vs2019/mimalloc-override.vcxproj | 1 + ide/vs2019/mimalloc-override.vcxproj.filters | 1 + ide/vs2019/mimalloc.vcxproj | 1 + ide/vs2019/mimalloc.vcxproj.filters | 3 +++ 4 files changed, 6 insertions(+) diff --git a/ide/vs2019/mimalloc-override.vcxproj b/ide/vs2019/mimalloc-override.vcxproj index 5fa59569..5fb809c8 100644 --- a/ide/vs2019/mimalloc-override.vcxproj +++ b/ide/vs2019/mimalloc-override.vcxproj @@ -236,6 +236,7 @@ + diff --git a/ide/vs2019/mimalloc-override.vcxproj.filters b/ide/vs2019/mimalloc-override.vcxproj.filters index c06fd1de..737e0600 100644 --- a/ide/vs2019/mimalloc-override.vcxproj.filters +++ b/ide/vs2019/mimalloc-override.vcxproj.filters @@ -49,6 +49,7 @@ Source Files + diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj index c12955c3..9e372ca4 100644 --- a/ide/vs2019/mimalloc.vcxproj +++ b/ide/vs2019/mimalloc.vcxproj @@ -225,6 +225,7 @@ + diff --git a/ide/vs2019/mimalloc.vcxproj.filters b/ide/vs2019/mimalloc.vcxproj.filters index 4cd0eb2e..5ab88914 100644 --- a/ide/vs2019/mimalloc.vcxproj.filters +++ b/ide/vs2019/mimalloc.vcxproj.filters @@ -52,6 +52,9 @@ Source Files + + Source Files + From 824fd8a7b100be3c3c6db0fe0b8591ad0e334719 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 15 Mar 2023 20:31:52 -0700 Subject: [PATCH 038/102] fix issue #707; rename a local template parameter (destroy) to work around two-phase template resolve in msvc 2019 --- ide/vs2017/mimalloc.vcxproj | 2 +- ide/vs2019/mimalloc.vcxproj | 2 +- ide/vs2022/mimalloc.vcxproj | 12 ++++++------ include/mimalloc.h | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index d29c2c7f..70bf4366 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -129,7 +129,7 @@ true ../../include _CRT_SECURE_NO_WARNINGS;MI_DEBUG=3;%(PreprocessorDefinitions); - CompileAsC + CompileAsCpp false stdcpp14 diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj index c12955c3..729a5c63 100644 --- a/ide/vs2019/mimalloc.vcxproj +++ b/ide/vs2019/mimalloc.vcxproj @@ -114,7 +114,7 @@ Level4 Disabled true - true + Default ../../include MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index 9081881c..49eb5180 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -95,12 +95,12 @@ Level4 Disabled true - true + Default ../../include MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false - Default + stdcpp20 @@ -114,7 +114,7 @@ Level4 Disabled true - true + Default ../../include MI_DEBUG=4;%(PreprocessorDefinitions); CompileAsCpp @@ -141,7 +141,7 @@ Level4 MaxSpeed true - true + Default ../../include %(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode @@ -151,7 +151,7 @@ Default CompileAsCpp true - Default + stdcpp20 true @@ -169,7 +169,7 @@ Level4 MaxSpeed true - true + Default ../../include %(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode diff --git a/include/mimalloc.h b/include/mimalloc.h index 27f6b331..4fc7a752 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -470,13 +470,13 @@ template bool operator==(const mi_stl_allocator& , const template bool operator!=(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return false; } -#if (__cplusplus >= 201103L) || (_MSC_VER >= 1920) // C++11, at least vs2019 +#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11 #define MI_HAS_HEAP_STL_ALLOCATOR 1 #include // std::shared_ptr // Common base class for STL allocators in a specific heap -template struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common { +template struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common { using typename _mi_stl_allocator_common::size_type; using typename _mi_stl_allocator_common::value_type; using typename _mi_stl_allocator_common::pointer; @@ -495,7 +495,7 @@ template struct _mi_heap_stl_allocator_common : public _m #endif void collect(bool force) { mi_heap_collect(this->heap.get(), force); } - template bool is_equal(const _mi_heap_stl_allocator_common& x) const { return (this->heap == x.heap); } + template bool is_equal(const _mi_heap_stl_allocator_common& x) const { return (this->heap == x.heap); } protected: std::shared_ptr heap; @@ -503,10 +503,10 @@ protected: _mi_heap_stl_allocator_common() { mi_heap_t* hp = mi_heap_new(); - this->heap.reset(hp, (destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */ + this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */ } _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { } - template _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { } + template _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { } private: static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } } From c4c96d2f8d9f62f6a5149be1f7aea2c457108ddf Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 15 Mar 2023 20:38:10 -0700 Subject: [PATCH 039/102] update older vs ide projects --- ide/vs2017/mimalloc-override.vcxproj | 1 + ide/vs2017/mimalloc-override.vcxproj.filters | 3 +++ ide/vs2017/mimalloc.vcxproj | 1 + ide/vs2017/mimalloc.vcxproj.filters | 3 +++ include/mimalloc-internal.h | 1 + 5 files changed, 9 insertions(+) diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index f3b7dd1e..f308225b 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -236,6 +236,7 @@ + diff --git a/ide/vs2017/mimalloc-override.vcxproj.filters b/ide/vs2017/mimalloc-override.vcxproj.filters index e045ed8c..a67dddad 100644 --- a/ide/vs2017/mimalloc-override.vcxproj.filters +++ b/ide/vs2017/mimalloc-override.vcxproj.filters @@ -82,5 +82,8 @@ Source Files + + Source Files + \ No newline at end of file diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index 70bf4366..e7be785c 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -233,6 +233,7 @@ + diff --git a/ide/vs2017/mimalloc.vcxproj.filters b/ide/vs2017/mimalloc.vcxproj.filters index 500292c5..27cd4b5e 100644 --- a/ide/vs2017/mimalloc.vcxproj.filters +++ b/ide/vs2017/mimalloc.vcxproj.filters @@ -62,6 +62,9 @@ Source Files + + Source Files + diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 36de1342..33aafa70 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -699,6 +699,7 @@ static inline size_t mi_ctz(uintptr_t x) { #elif defined(_MSC_VER) #include // LONG_MAX +#include // BitScanReverse64 #define MI_HAVE_FAST_BITSCAN static inline size_t mi_clz(uintptr_t x) { if (x==0) return MI_INTPTR_BITS; From 7d834864bb59c31f8ced90d27530d51f5e05ae64 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 16 Mar 2023 11:35:11 -0700 Subject: [PATCH 040/102] fix macOSX compilation --- src/init.c | 2 +- src/prim/prim-unix.c | 2 +- src/prim/prim.h | 18 +++++++++++------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/init.c b/src/init.c index 31ec87cb..d73c1e1e 100644 --- a/src/init.c +++ b/src/init.c @@ -399,7 +399,7 @@ void _mi_thread_done(mi_heap_t* heap) void _mi_heap_set_default_direct(mi_heap_t* heap) { mi_assert_internal(heap != NULL); #if defined(MI_TLS_SLOT) - mi_tls_slot_set(MI_TLS_SLOT,heap); + mi_prim_tls_slot_set(MI_TLS_SLOT,heap); #elif defined(MI_TLS_PTHREAD_SLOT_OFS) *mi_tls_pthread_heap_slot() = heap; #elif defined(MI_TLS_PTHREAD) diff --git a/src/prim/prim-unix.c b/src/prim/prim-unix.c index 9270e088..997c0356 100644 --- a/src/prim/prim-unix.c +++ b/src/prim/prim-unix.c @@ -796,4 +796,4 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { MI_UNUSED(heap); } -#endif \ No newline at end of file +#endif diff --git a/src/prim/prim.h b/src/prim/prim.h index 5a58a79f..967c6698 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -252,16 +252,20 @@ static inline mi_heap_t* mi_prim_get_default_heap(void) { #elif defined(MI_TLS_PTHREAD_SLOT_OFS) -static inline mi_heap_t* mi_prim_get_default_heap(void) { - mi_heap_t* heap; +static inline mi_heap_t** mi_prim_tls_pthread_heap_slot(void) { pthread_t self = pthread_self(); #if defined(__DragonFly__) - if (self==NULL) { heap = _mi_heap_main_get(); } else + if (self==NULL) return NULL; #endif - { - heap = *((mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS)); - } - return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); + return (mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS); +} + +static inline mi_heap_t* mi_prim_get_default_heap(void) { + mi_heap_t** pheap = mi_prim_tls_pthread_heap_slot(); + if mi_unlikely(pheap == NULL) return _mi_heap_main_get(); + mi_heap_t* heap = *pheap; + if mi_unlikely(heap == NULL) return (mi_heap_t*)&_mi_heap_empty; + return heap; } #elif defined(MI_TLS_PTHREAD) From 134b23b9210795734b4cd3127616ca02e3461150 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 16 Mar 2023 17:42:00 -0700 Subject: [PATCH 041/102] fix asan/valgrind api fill test --- test/test-api-fill.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test-api-fill.c b/test/test-api-fill.c index a32dfa27..2ad06808 100644 --- a/test/test-api-fill.c +++ b/test/test-api-fill.c @@ -271,7 +271,7 @@ int main(void) { mi_free(p); }; - + #if !(MI_TRACK_VALGRIND || MI_TRACK_ASAN) CHECK_BODY("fill-freed-small") { size_t malloc_size = MI_SMALL_SIZE_MAX / 2; uint8_t* p = (uint8_t*)mi_malloc(malloc_size); @@ -286,6 +286,7 @@ int main(void) { // First sizeof(void*) bytes will contain housekeeping data, skip these result = check_debug_fill_freed(p + sizeof(void*), malloc_size - sizeof(void*)); }; + #endif #endif // --------------------------------------------------- @@ -309,7 +310,7 @@ bool check_zero_init(uint8_t* p, size_t size) { #if MI_DEBUG >= 2 bool check_debug_fill_uninit(uint8_t* p, size_t size) { -#if MI_TRACK_VALGRIND +#if MI_TRACK_VALGRIND || MI_TRACK_ASAN (void)p; (void)size; return true; // when compiled with valgrind we don't init on purpose #else From 8a1f6c82b238b62b9489ff252d97d3c419306a30 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 16 Mar 2023 17:47:00 -0700 Subject: [PATCH 042/102] move prim files in subdirectories --- src/prim/prim.c | 6 +++--- src/prim/{prim-unix.c => unix/prim.c} | 4 +++- src/prim/{prim-wasi.c => wasi/prim.c} | 4 +++- src/prim/{prim-windows.c => windows/prim.c} | 4 +++- 4 files changed, 12 insertions(+), 6 deletions(-) rename src/prim/{prim-unix.c => unix/prim.c} (99%) rename src/prim/{prim-wasi.c => wasi/prim.c} (99%) rename src/prim/{prim-windows.c => windows/prim.c} (99%) diff --git a/src/prim/prim.c b/src/prim/prim.c index 83b7abc1..eec13c48 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -9,10 +9,10 @@ terms of the MIT license. A copy of the license can be found in the file // depending on the OS. #if defined(_WIN32) -#include "prim-windows.c" // VirtualAlloc (Windows) +#include "windows/prim.c" // VirtualAlloc (Windows) #elif defined(__wasi__) #define MI_USE_SBRK -#include "prim-wasi.h" // memory-grow or sbrk (Wasm) +#include "wasi/prim.h" // memory-grow or sbrk (Wasm) #else -#include "prim-unix.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) +#include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) #endif diff --git a/src/prim/prim-unix.c b/src/prim/unix/prim.c similarity index 99% rename from src/prim/prim-unix.c rename to src/prim/unix/prim.c index 997c0356..d1cd4301 100644 --- a/src/prim/prim-unix.c +++ b/src/prim/unix/prim.c @@ -5,6 +5,8 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +// This file is included in `src/prim/prim.c` + #ifndef _DEFAULT_SOURCE #define _DEFAULT_SOURCE // ensure mmap flags and syscall are defined #endif @@ -21,7 +23,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" -#include "prim.h" +#include "../prim.h" #include // mmap #include // sysconf diff --git a/src/prim/prim-wasi.c b/src/prim/wasi/prim.c similarity index 99% rename from src/prim/prim-wasi.c rename to src/prim/wasi/prim.c index f4d3be58..b8ac1a1b 100644 --- a/src/prim/prim-wasi.c +++ b/src/prim/wasi/prim.c @@ -5,10 +5,12 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +// This file is included in `src/prim/prim.c` + #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" -#include "prim.h" +#include "../prim.h" //--------------------------------------------- // Initialize diff --git a/src/prim/prim-windows.c b/src/prim/windows/prim.c similarity index 99% rename from src/prim/prim-windows.c rename to src/prim/windows/prim.c index 008b9fa4..2fa445a1 100644 --- a/src/prim/prim-windows.c +++ b/src/prim/windows/prim.c @@ -5,10 +5,12 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +// This file is included in `src/prim/prim.c` + #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" -#include "prim.h" +#include "../prim.h" #include // strerror #include // fputs, stderr From 072316bd33e84453b133fe1749a3b757040cb1f8 Mon Sep 17 00:00:00 2001 From: Xinglong He Date: Sat, 11 Mar 2023 17:30:21 -0800 Subject: [PATCH 043/102] add etw support --- ide/vs2022/mimalloc-override.vcxproj | 5 + ide/vs2022/mimalloc.vcxproj | 1 + include/mimalloc-etw-gen.h | 905 +++++++++++++++++++++++++++ include/mimalloc-etw-gen.man | Bin 0 -> 3950 bytes include/mimalloc-etw.h | 4 + include/mimalloc-track.h | 12 + include/mimalloc-types.h | 4 +- include/mimalloc.wprp | 56 ++ src/init.c | 5 + 9 files changed, 991 insertions(+), 1 deletion(-) create mode 100644 include/mimalloc-etw-gen.h create mode 100644 include/mimalloc-etw-gen.man create mode 100644 include/mimalloc-etw.h create mode 100644 include/mimalloc.wprp diff --git a/ide/vs2022/mimalloc-override.vcxproj b/ide/vs2022/mimalloc-override.vcxproj index a1d25d28..1eb72952 100644 --- a/ide/vs2022/mimalloc-override.vcxproj +++ b/ide/vs2022/mimalloc-override.vcxproj @@ -212,6 +212,8 @@ + + @@ -252,6 +254,9 @@ + + + diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index 15c7e039..9e0ffc85 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -246,6 +246,7 @@ + diff --git a/include/mimalloc-etw-gen.h b/include/mimalloc-etw-gen.h new file mode 100644 index 00000000..4e0a092a --- /dev/null +++ b/include/mimalloc-etw-gen.h @@ -0,0 +1,905 @@ +//**********************************************************************` +//* This is an include file generated by Message Compiler. *` +//* *` +//* Copyright (c) Microsoft Corporation. All Rights Reserved. *` +//**********************************************************************` +#pragma once + +//***************************************************************************** +// +// Notes on the ETW event code generated by MC: +// +// - Structures and arrays of structures are treated as an opaque binary blob. +// The caller is responsible for packing the data for the structure into a +// single region of memory, with no padding between values. The macro will +// have an extra parameter for the length of the blob. +// - Arrays of nul-terminated strings must be packed by the caller into a +// single binary blob containing the correct number of strings, with a nul +// after each string. The size of the blob is specified in characters, and +// includes the final nul. +// - Arrays of SID are treated as a single binary blob. The caller is +// responsible for packing the SID values into a single region of memory with +// no padding. +// - The length attribute on the data element in the manifest is significant +// for values with intype win:UnicodeString, win:AnsiString, or win:Binary. +// The length attribute must be specified for win:Binary, and is optional for +// win:UnicodeString and win:AnsiString (if no length is given, the strings +// are assumed to be nul-terminated). For win:UnicodeString, the length is +// measured in characters, not bytes. +// - For an array of win:UnicodeString, win:AnsiString, or win:Binary, the +// length attribute applies to every value in the array, so every value in +// the array must have the same length. The values in the array are provided +// to the macro via a single pointer -- the caller is responsible for packing +// all of the values into a single region of memory with no padding between +// values. +// - Values of type win:CountedUnicodeString, win:CountedAnsiString, and +// win:CountedBinary can be generated and collected on Vista or later. +// However, they may not decode properly without the Windows 10 2018 Fall +// Update. +// - Arrays of type win:CountedUnicodeString, win:CountedAnsiString, and +// win:CountedBinary must be packed by the caller into a single region of +// memory. The format for each item is a UINT16 byte-count followed by that +// many bytes of data. When providing the array to the generated macro, you +// must provide the total size of the packed array data, including the UINT16 +// sizes for each item. In the case of win:CountedUnicodeString, the data +// size is specified in WCHAR (16-bit) units. In the case of +// win:CountedAnsiString and win:CountedBinary, the data size is specified in +// bytes. +// +//***************************************************************************** + +#include +#include +#include + +#ifndef ETW_INLINE + #ifdef _ETW_KM_ + // In kernel mode, save stack space by never inlining templates. + #define ETW_INLINE DECLSPEC_NOINLINE __inline + #else + // In user mode, save code size by inlining templates as appropriate. + #define ETW_INLINE __inline + #endif +#endif // ETW_INLINE + +#if defined(__cplusplus) +extern "C" { +#endif + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// MCGEN_USE_KERNEL_MODE_APIS macro: +// Controls whether the generated code uses kernel-mode or user-mode APIs. +// - Set to 0 to use Windows user-mode APIs such as EventRegister. +// - Set to 1 to use Windows kernel-mode APIs such as EtwRegister. +// Default is based on whether the _ETW_KM_ macro is defined (i.e. by wdm.h). +// Note that the APIs can also be overridden directly, e.g. by setting the +// MCGEN_EVENTWRITETRANSFER or MCGEN_EVENTREGISTER macros. +// +#ifndef MCGEN_USE_KERNEL_MODE_APIS + #ifdef _ETW_KM_ + #define MCGEN_USE_KERNEL_MODE_APIS 1 + #else + #define MCGEN_USE_KERNEL_MODE_APIS 0 + #endif +#endif // MCGEN_USE_KERNEL_MODE_APIS + +// +// MCGEN_HAVE_EVENTSETINFORMATION macro: +// Controls how McGenEventSetInformation uses the EventSetInformation API. +// - Set to 0 to disable the use of EventSetInformation +// (McGenEventSetInformation will always return an error). +// - Set to 1 to directly invoke MCGEN_EVENTSETINFORMATION. +// - Set to 2 to to locate EventSetInformation at runtime via GetProcAddress +// (user-mode) or MmGetSystemRoutineAddress (kernel-mode). +// Default is determined as follows: +// - If MCGEN_EVENTSETINFORMATION has been customized, set to 1 +// (i.e. use MCGEN_EVENTSETINFORMATION). +// - Else if the target OS version has EventSetInformation, set to 1 +// (i.e. use MCGEN_EVENTSETINFORMATION). +// - Else set to 2 (i.e. try to dynamically locate EventSetInformation). +// Note that an McGenEventSetInformation function will only be generated if one +// or more provider in a manifest has provider traits. +// +#ifndef MCGEN_HAVE_EVENTSETINFORMATION + #ifdef MCGEN_EVENTSETINFORMATION // if MCGEN_EVENTSETINFORMATION has been customized, + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #elif MCGEN_USE_KERNEL_MODE_APIS // else if using kernel-mode APIs, + #if NTDDI_VERSION >= 0x06040000 // if target OS is Windows 10 or later, + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #else // else + #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find "EtwSetInformation" via MmGetSystemRoutineAddress. + #endif // else (using user-mode APIs) + #else // if target OS and SDK is Windows 8 or later, + #if WINVER >= 0x0602 && defined(EVENT_FILTER_TYPE_SCHEMATIZED) + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #else // else + #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find "EventSetInformation" via GetModuleHandleExW/GetProcAddress. + #endif + #endif +#endif // MCGEN_HAVE_EVENTSETINFORMATION + +// +// MCGEN Override Macros +// +// The following override macros may be defined before including this header +// to control the APIs used by this header: +// +// - MCGEN_EVENTREGISTER +// - MCGEN_EVENTUNREGISTER +// - MCGEN_EVENTSETINFORMATION +// - MCGEN_EVENTWRITETRANSFER +// +// If the the macro is undefined, the MC implementation will default to the +// corresponding ETW APIs. For example, if the MCGEN_EVENTREGISTER macro is +// undefined, the EventRegister[MyProviderName] macro will use EventRegister +// in user mode and will use EtwRegister in kernel mode. +// +// To prevent issues from conflicting definitions of these macros, the value +// of the override macro will be used as a suffix in certain internal function +// names. Because of this, the override macros must follow certain rules: +// +// - The macro must be defined before any MC-generated header is included and +// must not be undefined or redefined after any MC-generated header is +// included. Different translation units (i.e. different .c or .cpp files) +// may set the macros to different values, but within a translation unit +// (within a single .c or .cpp file), the macro must be set once and not +// changed. +// - The override must be an object-like macro, not a function-like macro +// (i.e. the override macro must not have a parameter list). +// - The override macro's value must be a simple identifier, i.e. must be +// something that starts with a letter or '_' and contains only letters, +// numbers, and '_' characters. +// - If the override macro's value is the name of a second object-like macro, +// the second object-like macro must follow the same rules. (The override +// macro's value can also be the name of a function-like macro, in which +// case the function-like macro does not need to follow the same rules.) +// +// For example, the following will cause compile errors: +// +// #define MCGEN_EVENTWRITETRANSFER MyNamespace::MyClass::MyFunction // Value has non-identifier characters (colon). +// #define MCGEN_EVENTWRITETRANSFER GetEventWriteFunctionPointer(7) // Value has non-identifier characters (parentheses). +// #define MCGEN_EVENTWRITETRANSFER(h,e,a,r,c,d) EventWrite(h,e,c,d) // Override is defined as a function-like macro. +// #define MY_OBJECT_LIKE_MACRO MyNamespace::MyClass::MyEventWriteFunction +// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // Evaluates to something with non-identifier characters (colon). +// +// The following would be ok: +// +// #define MCGEN_EVENTWRITETRANSFER MyEventWriteFunction1 // OK, suffix will be "MyEventWriteFunction1". +// #define MY_OBJECT_LIKE_MACRO MyEventWriteFunction2 +// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // OK, suffix will be "MyEventWriteFunction2". +// #define MY_FUNCTION_LIKE_MACRO(h,e,a,r,c,d) MyNamespace::MyClass::MyEventWriteFunction3(h,e,c,d) +// #define MCGEN_EVENTWRITETRANSFER MY_FUNCTION_LIKE_MACRO // OK, suffix will be "MY_FUNCTION_LIKE_MACRO". +// +#ifndef MCGEN_EVENTREGISTER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTREGISTER EtwRegister + #else + #define MCGEN_EVENTREGISTER EventRegister + #endif +#endif // MCGEN_EVENTREGISTER +#ifndef MCGEN_EVENTUNREGISTER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTUNREGISTER EtwUnregister + #else + #define MCGEN_EVENTUNREGISTER EventUnregister + #endif +#endif // MCGEN_EVENTUNREGISTER +#ifndef MCGEN_EVENTSETINFORMATION + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTSETINFORMATION EtwSetInformation + #else + #define MCGEN_EVENTSETINFORMATION EventSetInformation + #endif +#endif // MCGEN_EVENTSETINFORMATION +#ifndef MCGEN_EVENTWRITETRANSFER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTWRITETRANSFER EtwWriteTransfer + #else + #define MCGEN_EVENTWRITETRANSFER EventWriteTransfer + #endif +#endif // MCGEN_EVENTWRITETRANSFER + +// +// MCGEN_EVENT_ENABLED macro: +// Override to control how the EventWrite[EventName] macros determine whether +// an event is enabled. The default behavior is for EventWrite[EventName] to +// use the EventEnabled[EventName] macros. +// +#ifndef MCGEN_EVENT_ENABLED +#define MCGEN_EVENT_ENABLED(EventName) EventEnabled##EventName() +#endif + +// +// MCGEN_EVENT_ENABLED_FORCONTEXT macro: +// Override to control how the EventWrite[EventName]_ForContext macros +// determine whether an event is enabled. The default behavior is for +// EventWrite[EventName]_ForContext to use the +// EventEnabled[EventName]_ForContext macros. +// +#ifndef MCGEN_EVENT_ENABLED_FORCONTEXT +#define MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, EventName) EventEnabled##EventName##_ForContext(pContext) +#endif + +// +// MCGEN_ENABLE_CHECK macro: +// Determines whether the specified event would be considered as enabled +// based on the state of the specified context. Slightly faster than calling +// McGenEventEnabled directly. +// +#ifndef MCGEN_ENABLE_CHECK +#define MCGEN_ENABLE_CHECK(Context, Descriptor) (Context.IsEnabled && McGenEventEnabled(&Context, &Descriptor)) +#endif + +#if !defined(MCGEN_TRACE_CONTEXT_DEF) +#define MCGEN_TRACE_CONTEXT_DEF +// This structure is for use by MC-generated code and should not be used directly. +typedef struct _MCGEN_TRACE_CONTEXT +{ + TRACEHANDLE RegistrationHandle; + TRACEHANDLE Logger; // Used as pointer to provider traits. + ULONGLONG MatchAnyKeyword; + ULONGLONG MatchAllKeyword; + ULONG Flags; + ULONG IsEnabled; + UCHAR Level; + UCHAR Reserve; + USHORT EnableBitsCount; + PULONG EnableBitMask; + const ULONGLONG* EnableKeyWords; + const UCHAR* EnableLevel; +} MCGEN_TRACE_CONTEXT, *PMCGEN_TRACE_CONTEXT; +#endif // MCGEN_TRACE_CONTEXT_DEF + +#if !defined(MCGEN_LEVEL_KEYWORD_ENABLED_DEF) +#define MCGEN_LEVEL_KEYWORD_ENABLED_DEF +// +// Determines whether an event with a given Level and Keyword would be +// considered as enabled based on the state of the specified context. +// Note that you may want to use MCGEN_ENABLE_CHECK instead of calling this +// function directly. +// +FORCEINLINE +BOOLEAN +McGenLevelKeywordEnabled( + _In_ PMCGEN_TRACE_CONTEXT EnableInfo, + _In_ UCHAR Level, + _In_ ULONGLONG Keyword + ) +{ + // + // Check if the event Level is lower than the level at which + // the channel is enabled. + // If the event Level is 0 or the channel is enabled at level 0, + // all levels are enabled. + // + + if ((Level <= EnableInfo->Level) || // This also covers the case of Level == 0. + (EnableInfo->Level == 0)) { + + // + // Check if Keyword is enabled + // + + if ((Keyword == (ULONGLONG)0) || + ((Keyword & EnableInfo->MatchAnyKeyword) && + ((Keyword & EnableInfo->MatchAllKeyword) == EnableInfo->MatchAllKeyword))) { + return TRUE; + } + } + + return FALSE; +} +#endif // MCGEN_LEVEL_KEYWORD_ENABLED_DEF + +#if !defined(MCGEN_EVENT_ENABLED_DEF) +#define MCGEN_EVENT_ENABLED_DEF +// +// Determines whether the specified event would be considered as enabled based +// on the state of the specified context. Note that you may want to use +// MCGEN_ENABLE_CHECK instead of calling this function directly. +// +FORCEINLINE +BOOLEAN +McGenEventEnabled( + _In_ PMCGEN_TRACE_CONTEXT EnableInfo, + _In_ PCEVENT_DESCRIPTOR EventDescriptor + ) +{ + return McGenLevelKeywordEnabled(EnableInfo, EventDescriptor->Level, EventDescriptor->Keyword); +} +#endif // MCGEN_EVENT_ENABLED_DEF + +#if !defined(MCGEN_CONTROL_CALLBACK) +#define MCGEN_CONTROL_CALLBACK + +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +VOID +__stdcall +McGenControlCallbackV2( + _In_ LPCGUID SourceId, + _In_ ULONG ControlCode, + _In_ UCHAR Level, + _In_ ULONGLONG MatchAnyKeyword, + _In_ ULONGLONG MatchAllKeyword, + _In_opt_ PEVENT_FILTER_DESCRIPTOR FilterData, + _Inout_opt_ PVOID CallbackContext + ) +/*++ + +Routine Description: + + This is the notification callback for Windows Vista and later. + +Arguments: + + SourceId - The GUID that identifies the session that enabled the provider. + + ControlCode - The parameter indicates whether the provider + is being enabled or disabled. + + Level - The level at which the event is enabled. + + MatchAnyKeyword - The bitmask of keywords that the provider uses to + determine the category of events that it writes. + + MatchAllKeyword - This bitmask additionally restricts the category + of events that the provider writes. + + FilterData - The provider-defined data. + + CallbackContext - The context of the callback that is defined when the provider + called EtwRegister to register itself. + +Remarks: + + ETW calls this function to notify provider of enable/disable + +--*/ +{ + PMCGEN_TRACE_CONTEXT Ctx = (PMCGEN_TRACE_CONTEXT)CallbackContext; + ULONG Ix; +#ifndef MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + UNREFERENCED_PARAMETER(SourceId); + UNREFERENCED_PARAMETER(FilterData); +#endif + + if (Ctx == NULL) { + return; + } + + switch (ControlCode) { + + case EVENT_CONTROL_CODE_ENABLE_PROVIDER: + Ctx->Level = Level; + Ctx->MatchAnyKeyword = MatchAnyKeyword; + Ctx->MatchAllKeyword = MatchAllKeyword; + Ctx->IsEnabled = EVENT_CONTROL_CODE_ENABLE_PROVIDER; + + for (Ix = 0; Ix < Ctx->EnableBitsCount; Ix += 1) { + if (McGenLevelKeywordEnabled(Ctx, Ctx->EnableLevel[Ix], Ctx->EnableKeyWords[Ix]) != FALSE) { + Ctx->EnableBitMask[Ix >> 5] |= (1 << (Ix % 32)); + } else { + Ctx->EnableBitMask[Ix >> 5] &= ~(1 << (Ix % 32)); + } + } + break; + + case EVENT_CONTROL_CODE_DISABLE_PROVIDER: + Ctx->IsEnabled = EVENT_CONTROL_CODE_DISABLE_PROVIDER; + Ctx->Level = 0; + Ctx->MatchAnyKeyword = 0; + Ctx->MatchAllKeyword = 0; + if (Ctx->EnableBitsCount > 0) { +#pragma warning(suppress: 26451) // Arithmetic overflow cannot occur, no matter the value of EnableBitCount + RtlZeroMemory(Ctx->EnableBitMask, (((Ctx->EnableBitsCount - 1) / 32) + 1) * sizeof(ULONG)); + } + break; + + default: + break; + } + +#ifdef MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + // + // Call user defined callback + // + MCGEN_PRIVATE_ENABLE_CALLBACK_V2( + SourceId, + ControlCode, + Level, + MatchAnyKeyword, + MatchAllKeyword, + FilterData, + CallbackContext + ); +#endif // MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + + return; +} + +#endif // MCGEN_CONTROL_CALLBACK + +#ifndef _mcgen_PENABLECALLBACK + #if MCGEN_USE_KERNEL_MODE_APIS + #define _mcgen_PENABLECALLBACK PETWENABLECALLBACK + #else + #define _mcgen_PENABLECALLBACK PENABLECALLBACK + #endif +#endif // _mcgen_PENABLECALLBACK + +#if !defined(_mcgen_PASTE2) +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_PASTE2(a, b) _mcgen_PASTE2_imp(a, b) +#define _mcgen_PASTE2_imp(a, b) a##b +#endif // _mcgen_PASTE2 + +#if !defined(_mcgen_PASTE3) +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_PASTE3(a, b, c) _mcgen_PASTE3_imp(a, b, c) +#define _mcgen_PASTE3_imp(a, b, c) a##b##_##c +#endif // _mcgen_PASTE3 + +// +// Macro validation +// + +// Validate MCGEN_EVENTREGISTER: + +// Trigger an error if MCGEN_EVENTREGISTER is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER); + +// Trigger an error if MCGEN_EVENTREGISTER is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER) + MCGEN_EVENTREGISTER_must_not_be_redefined_between_headers; + +// Trigger an error if MCGEN_EVENTREGISTER is defined as a function-like macro: +typedef void MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTREGISTER; +typedef int _mcgen_PASTE2(MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTREGISTER); + +// Validate MCGEN_EVENTUNREGISTER: + +// Trigger an error if MCGEN_EVENTUNREGISTER is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER); + +// Trigger an error if MCGEN_EVENTUNREGISTER is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER) + MCGEN_EVENTUNREGISTER_must_not_be_redefined_between_headers; + +// Trigger an error if MCGEN_EVENTUNREGISTER is defined as a function-like macro: +typedef void MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTUNREGISTER; +typedef int _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTUNREGISTER); + +// Validate MCGEN_EVENTSETINFORMATION: + +// Trigger an error if MCGEN_EVENTSETINFORMATION is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION); + +// Trigger an error if MCGEN_EVENTSETINFORMATION is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION) + MCGEN_EVENTSETINFORMATION_must_not_be_redefined_between_headers; + +// Trigger an error if MCGEN_EVENTSETINFORMATION is defined as a function-like macro: +typedef void MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_MCGEN_EVENTSETINFORMATION; +typedef int _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_, MCGEN_EVENTSETINFORMATION); + +// Validate MCGEN_EVENTWRITETRANSFER: + +// Trigger an error if MCGEN_EVENTWRITETRANSFER is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER); + +// Trigger an error if MCGEN_EVENTWRITETRANSFER is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER) + MCGEN_EVENTWRITETRANSFER_must_not_be_redefined_between_headers;; + +// Trigger an error if MCGEN_EVENTWRITETRANSFER is defined as a function-like macro: +typedef void MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_MCGEN_EVENTWRITETRANSFER; +typedef int _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_, MCGEN_EVENTWRITETRANSFER); + +#ifndef McGenEventWrite_def +#define McGenEventWrite_def + +// This macro is for use by MC-generated code and should not be used directly. +#define McGenEventWrite _mcgen_PASTE2(McGenEventWrite_, MCGEN_EVENTWRITETRANSFER) + +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventWrite( + _In_ PMCGEN_TRACE_CONTEXT Context, + _In_ PCEVENT_DESCRIPTOR Descriptor, + _In_opt_ LPCGUID ActivityId, + _In_range_(1, 128) ULONG EventDataCount, + _Pre_cap_(EventDataCount) EVENT_DATA_DESCRIPTOR* EventData + ) +{ + const USHORT UNALIGNED* Traits; + + // Some customized MCGEN_EVENTWRITETRANSFER macros might ignore ActivityId. + UNREFERENCED_PARAMETER(ActivityId); + + Traits = (const USHORT UNALIGNED*)(UINT_PTR)Context->Logger; + + if (Traits == NULL) { + EventData[0].Ptr = 0; + EventData[0].Size = 0; + EventData[0].Reserved = 0; + } else { + EventData[0].Ptr = (ULONG_PTR)Traits; + EventData[0].Size = *Traits; + EventData[0].Reserved = 2; // EVENT_DATA_DESCRIPTOR_TYPE_PROVIDER_METADATA + } + + return MCGEN_EVENTWRITETRANSFER( + Context->RegistrationHandle, + Descriptor, + ActivityId, + NULL, + EventDataCount, + EventData); +} +#endif // McGenEventWrite_def + +#if !defined(McGenEventRegisterUnregister) +#define McGenEventRegisterUnregister + +// This macro is for use by MC-generated code and should not be used directly. +#define McGenEventRegister _mcgen_PASTE2(McGenEventRegister_, MCGEN_EVENTREGISTER) + +#pragma warning(push) +#pragma warning(disable:6103) +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventRegister( + _In_ LPCGUID ProviderId, + _In_opt_ _mcgen_PENABLECALLBACK EnableCallback, + _In_opt_ PVOID CallbackContext, + _Inout_ PREGHANDLE RegHandle + ) +/*++ + +Routine Description: + + This function registers the provider with ETW. + +Arguments: + + ProviderId - Provider ID to register with ETW. + + EnableCallback - Callback to be used. + + CallbackContext - Context for the callback. + + RegHandle - Pointer to registration handle. + +Remarks: + + Should not be called if the provider is already registered (i.e. should not + be called if *RegHandle != 0). Repeatedly registering a provider is a bug + and may indicate a race condition. However, for compatibility with previous + behavior, this function will return SUCCESS in this case. + +--*/ +{ + ULONG Error; + + if (*RegHandle != 0) + { + Error = 0; // ERROR_SUCCESS + } + else + { + Error = MCGEN_EVENTREGISTER(ProviderId, EnableCallback, CallbackContext, RegHandle); + } + + return Error; +} +#pragma warning(pop) + +// This macro is for use by MC-generated code and should not be used directly. +#define McGenEventUnregister _mcgen_PASTE2(McGenEventUnregister_, MCGEN_EVENTUNREGISTER) + +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventUnregister(_Inout_ PREGHANDLE RegHandle) +/*++ + +Routine Description: + + Unregister from ETW and set *RegHandle = 0. + +Arguments: + + RegHandle - the pointer to the provider registration handle + +Remarks: + + If provider has not been registered (i.e. if *RegHandle == 0), + return SUCCESS. It is safe to call McGenEventUnregister even if the + call to McGenEventRegister returned an error. + +--*/ +{ + ULONG Error; + + if(*RegHandle == 0) + { + Error = 0; // ERROR_SUCCESS + } + else + { + Error = MCGEN_EVENTUNREGISTER(*RegHandle); + *RegHandle = (REGHANDLE)0; + } + + return Error; +} + +#endif // McGenEventRegisterUnregister + +#ifndef _mcgen_EVENT_BIT_SET + #if defined(_M_IX86) || defined(_M_X64) + // This macro is for use by MC-generated code and should not be used directly. + #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((((const unsigned char*)EnableBits)[BitPosition >> 3] & (1u << (BitPosition & 7))) != 0) + #else // CPU type + // This macro is for use by MC-generated code and should not be used directly. + #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((EnableBits[BitPosition >> 5] & (1u << (BitPosition & 31))) != 0) + #endif // CPU type +#endif // _mcgen_EVENT_BIT_SET + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +// Provider "microsoft-windows-mimalloc" event count 2 +//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +// Provider GUID = 138f4dbb-ee04-4899-aa0a-572ad4475779 +EXTERN_C __declspec(selectany) const GUID ETW_MI_Provider = {0x138f4dbb, 0xee04, 0x4899, {0xaa, 0x0a, 0x57, 0x2a, 0xd4, 0x47, 0x57, 0x79}}; + +#ifndef ETW_MI_Provider_Traits +#define ETW_MI_Provider_Traits NULL +#endif // ETW_MI_Provider_Traits + +// +// Event Descriptors +// +EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_ALLOC = {0x64, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0}; +#define ETW_MI_ALLOC_value 0x64 +EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_FREE = {0x65, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0}; +#define ETW_MI_FREE_value 0x65 + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// Event Enablement Bits +// These variables are for use by MC-generated code and should not be used directly. +// +EXTERN_C __declspec(selectany) DECLSPEC_CACHEALIGN ULONG microsoft_windows_mimallocEnableBits[1]; +EXTERN_C __declspec(selectany) const ULONGLONG microsoft_windows_mimallocKeywords[1] = {0x0}; +EXTERN_C __declspec(selectany) const unsigned char microsoft_windows_mimallocLevels[1] = {4}; + +// +// Provider context +// +EXTERN_C __declspec(selectany) MCGEN_TRACE_CONTEXT ETW_MI_Provider_Context = {0, (ULONG_PTR)ETW_MI_Provider_Traits, 0, 0, 0, 0, 0, 0, 1, microsoft_windows_mimallocEnableBits, microsoft_windows_mimallocKeywords, microsoft_windows_mimallocLevels}; + +// +// Provider REGHANDLE +// +#define microsoft_windows_mimallocHandle (ETW_MI_Provider_Context.RegistrationHandle) + +// +// This macro is set to 0, indicating that the EventWrite[Name] macros do not +// have an Activity parameter. This is controlled by the -km and -um options. +// +#define ETW_MI_Provider_EventWriteActivity 0 + +// +// Register with ETW using the control GUID specified in the manifest. +// Invoke this macro during module initialization (i.e. program startup, +// DLL process attach, or driver load) to initialize the provider. +// Note that if this function returns an error, the error means that +// will not work, but no action needs to be taken -- even if EventRegister +// returns an error, it is generally safe to use EventWrite and +// EventUnregister macros (they will be no-ops if EventRegister failed). +// +#ifndef EventRegistermicrosoft_windows_mimalloc +#define EventRegistermicrosoft_windows_mimalloc() McGenEventRegister(&ETW_MI_Provider, McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle) +#endif + +// +// Register with ETW using a specific control GUID (i.e. a GUID other than what +// is specified in the manifest). Advanced scenarios only. +// +#ifndef EventRegisterByGuidmicrosoft_windows_mimalloc +#define EventRegisterByGuidmicrosoft_windows_mimalloc(Guid) McGenEventRegister(&(Guid), McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle) +#endif + +// +// Unregister with ETW and close the provider. +// Invoke this macro during module shutdown (i.e. program exit, DLL process +// detach, or driver unload) to unregister the provider. +// Note that you MUST call EventUnregister before DLL or driver unload +// (not optional): failure to unregister a provider before DLL or driver unload +// will result in crashes. +// +#ifndef EventUnregistermicrosoft_windows_mimalloc +#define EventUnregistermicrosoft_windows_mimalloc() McGenEventUnregister(µsoft_windows_mimallocHandle) +#endif + +// +// MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION macro: +// Define this macro to enable support for caller-allocated provider context. +// +#ifdef MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION + +// +// Advanced scenarios: Caller-allocated provider context. +// Use when multiple differently-configured provider handles are needed, +// e.g. for container-aware drivers, one context per container. +// +// Usage: +// +// - Caller enables the feature before including this header, e.g. +// #define MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION 1 +// - Caller allocates memory, e.g. pContext = malloc(sizeof(McGenContext_microsoft_windows_mimalloc)); +// - Caller registers the provider, e.g. EventRegistermicrosoft_windows_mimalloc_ForContext(pContext); +// - Caller writes events, e.g. EventWriteMyEvent_ForContext(pContext, ...); +// - Caller unregisters, e.g. EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext); +// - Caller frees memory, e.g. free(pContext); +// + +typedef struct tagMcGenContext_microsoft_windows_mimalloc { + // The fields of this structure are subject to change and should + // not be accessed directly. To access the provider's REGHANDLE, + // use microsoft_windows_mimallocHandle_ForContext(pContext). + MCGEN_TRACE_CONTEXT Context; + ULONG EnableBits[1]; +} McGenContext_microsoft_windows_mimalloc; + +#define EventRegistermicrosoft_windows_mimalloc_ForContext(pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&ETW_MI_Provider, pContext) +#define EventRegisterByGuidmicrosoft_windows_mimalloc_ForContext(Guid, pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&(Guid), pContext) +#define EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext) McGenEventUnregister(&(pContext)->Context.RegistrationHandle) + +// +// Provider REGHANDLE for caller-allocated context. +// +#define microsoft_windows_mimallocHandle_ForContext(pContext) ((pContext)->Context.RegistrationHandle) + +// This function is for use by MC-generated code and should not be used directly. +// Initialize and register the caller-allocated context. +__inline +ULONG __stdcall +_mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)( + _In_ LPCGUID pProviderId, + _Out_ McGenContext_microsoft_windows_mimalloc* pContext) +{ + RtlZeroMemory(pContext, sizeof(*pContext)); + pContext->Context.Logger = (ULONG_PTR)ETW_MI_Provider_Traits; + pContext->Context.EnableBitsCount = 1; + pContext->Context.EnableBitMask = pContext->EnableBits; + pContext->Context.EnableKeyWords = microsoft_windows_mimallocKeywords; + pContext->Context.EnableLevel = microsoft_windows_mimallocLevels; + return McGenEventRegister( + pProviderId, + McGenControlCallbackV2, + &pContext->Context, + &pContext->Context.RegistrationHandle); +} + +// This function is for use by MC-generated code and should not be used directly. +// Trigger a compile error if called with the wrong parameter type. +FORCEINLINE +_Ret_ McGenContext_microsoft_windows_mimalloc* +_mcgen_CheckContextType_microsoft_windows_mimalloc(_In_ McGenContext_microsoft_windows_mimalloc* pContext) +{ + return pContext; +} + +#endif // MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION + +// +// Enablement check macro for event "ETW_MI_ALLOC" +// +#define EventEnabledETW_MI_ALLOC() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0) +#define EventEnabledETW_MI_ALLOC_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0) + +// +// Event write macros for event "ETW_MI_ALLOC" +// +#define EventWriteETW_MI_ALLOC(Address, Size) \ + MCGEN_EVENT_ENABLED(ETW_MI_ALLOC) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) : 0 +#define EventWriteETW_MI_ALLOC_AssumeEnabled(Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) +#define EventWriteETW_MI_ALLOC_ForContext(pContext, Address, Size) \ + MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_ALLOC) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&(pContext)->Context, &ETW_MI_ALLOC, Address, Size) : 0 +#define EventWriteETW_MI_ALLOC_ForContextAssumeEnabled(pContext, Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_ALLOC, Address, Size) + +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER) + +// +// Enablement check macro for event "ETW_MI_FREE" +// +#define EventEnabledETW_MI_FREE() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0) +#define EventEnabledETW_MI_FREE_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0) + +// +// Event write macros for event "ETW_MI_FREE" +// +#define EventWriteETW_MI_FREE(Address, Size) \ + MCGEN_EVENT_ENABLED(ETW_MI_FREE) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) : 0 +#define EventWriteETW_MI_FREE_AssumeEnabled(Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) +#define EventWriteETW_MI_FREE_ForContext(pContext, Address, Size) \ + MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_FREE) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&(pContext)->Context, &ETW_MI_FREE, Address, Size) : 0 +#define EventWriteETW_MI_FREE_ForContextAssumeEnabled(pContext, Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_FREE, Address, Size) + +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_TEMPLATE_FOR_ETW_MI_FREE _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER) + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// Template Functions +// + +// +// Function for template "ETW_CUSTOM_HEAP_ALLOC_DATA" (and possibly others). +// This function is for use by MC-generated code and should not be used directly. +// +#ifndef McTemplateU0xx_def +#define McTemplateU0xx_def +ETW_INLINE +ULONG +_mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)( + _In_ PMCGEN_TRACE_CONTEXT Context, + _In_ PCEVENT_DESCRIPTOR Descriptor, + _In_ const unsigned __int64 _Arg0, + _In_ const unsigned __int64 _Arg1 + ) +{ +#define McTemplateU0xx_ARGCOUNT 2 + + EVENT_DATA_DESCRIPTOR EventData[McTemplateU0xx_ARGCOUNT + 1]; + + EventDataDescCreate(&EventData[1],&_Arg0, sizeof(const unsigned __int64) ); + + EventDataDescCreate(&EventData[2],&_Arg1, sizeof(const unsigned __int64) ); + + return McGenEventWrite(Context, Descriptor, NULL, McTemplateU0xx_ARGCOUNT + 1, EventData); +} +#endif // McTemplateU0xx_def + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +#if defined(__cplusplus) +} +#endif diff --git a/include/mimalloc-etw-gen.man b/include/mimalloc-etw-gen.man new file mode 100644 index 0000000000000000000000000000000000000000..9ef58029233f5296ce71f657e46e068ef2d16418 GIT binary patch literal 3950 zcmeH~TTc@~6vxlAiQi%6*=?zyV6#dHLL{MTq(mP~$kK(vBp$7`^o?Hg#Z5!5#ou-Dw<5J|RS>^SNDo1B-C{k&d z{*Ih*qErEXQD4+~w&avuMYw0$3NH21DbzV=>r!QmQfFXQq%E^3gZBi=3h!!P6}*b6 zD$nrpvaGUNmUW)T7K;?x3>?Lq^HIf+B^DPKJGJZIBhfr4^wo#x%X%Hk!jrf4g7Z#- zuLjsR?v+U_E>>kM19%5`d>|`4`^2Gr%){A<`Ru0|G7_oJ+dGyoc;C<}pMdIRl{5;AVfa^LBY*I&i@?N&gl1psp zVw;O+4G+6}WZE60maAX%%=y;Ex{ha4;-^)sk@lThB`Z%3z1|XCJ9C(;m_J|zkFve-J?G} z#{2c=cm(Eko5J^#8FN$4*%CAheaE?v+$)eImh~~Q9_^f)bgyLCc{0lo?V3$0_iR{; zzP1jmx^lkVR*g>kR6a(jcWRwo{krJTky0ci)Q^50w&7VU^9$rvq-ZtAx0c}f$8FyS z<##Y@{{KH#Ypze`?P8vfO8kpt?cY~;5&7wHd&t>`oGo&sobB_vv{s1w&scnaO|Ova a?+rcGzA^hhqLVtGu0d~0k>&qOI=(+V`!f#! literal 0 HcmV?d00001 diff --git a/include/mimalloc-etw.h b/include/mimalloc-etw.h new file mode 100644 index 00000000..3c894056 --- /dev/null +++ b/include/mimalloc-etw.h @@ -0,0 +1,4 @@ +#pragma once +#include +#include "mimalloc-etw-gen.h" + diff --git a/include/mimalloc-track.h b/include/mimalloc-track.h index 272ca1b8..5e050512 100644 --- a/include/mimalloc-track.h +++ b/include/mimalloc-track.h @@ -71,6 +71,18 @@ defined, undefined, or not accessible at all: #define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) #define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size) +#elif MI_ETW +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_TOOL "ETW" + +#include "mimalloc-etw.h" + +#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)p, size) +#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)p, size) +#define mi_track_mem_defined(p,size) +#define mi_track_mem_undefined(p,size) +#define mi_track_mem_noaccess(p,size) + #else #define MI_TRACK_ENABLED 0 diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 9b3f5972..f8d891cb 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -32,6 +32,8 @@ terms of the MIT license. A copy of the license can be found in the file // Define MI_TRACK_ to enable tracking support // #define MI_TRACK_VALGRIND 1 // #define MI_TRACK_ASAN 1 +// Define MI_ETW to enable ETW provider + #define MI_ETW 1 // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). // #define MI_STAT 1 @@ -60,7 +62,7 @@ terms of the MIT license. A copy of the license can be found in the file // Reserve extra padding at the end of each block to be more resilient against heap block overflows. // The padding can detect buffer overflow on free. -#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || MI_TRACK_VALGRIND || MI_TRACK_ASAN) +#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_ETW) #define MI_PADDING 1 #endif diff --git a/include/mimalloc.wprp b/include/mimalloc.wprp new file mode 100644 index 00000000..7f284e9d --- /dev/null +++ b/include/mimalloc.wprp @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/init.c b/src/init.c index d73c1e1e..5574548e 100644 --- a/src/init.c +++ b/src/init.c @@ -11,6 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // memcpy, memset #include // atexit + // Empty page used to initialize the small free pages array const mi_page_t _mi_page_empty = { 0, false, false, false, false, @@ -549,6 +550,10 @@ void mi_process_init(void) mi_attr_noexcept { mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true); } } + +#ifdef MI_ETW + EventRegistermicrosoft_windows_mimalloc(); +#endif } // Called when the process is done (through `at_exit`) From 1a99efc671bd040b40b558f15d9906b2d24aa653 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 16 Mar 2023 20:08:43 -0700 Subject: [PATCH 044/102] integrate ETW windows event tracing into mimalloc as another track tool --- CMakeLists.txt | 16 ++++++ include/mimalloc-etw.h | 4 -- include/mimalloc-track.h | 48 +++++++++++++----- include/mimalloc-types.h | 5 +- src/init.c | 5 +- src/prim/readme.md | 6 ++- .../prim/windows/etw.h | 0 .../prim/windows/etw.man | Bin 3950 -> 3926 bytes {include => src/prim/windows}/mimalloc.wprp | 7 ++- src/prim/windows/readme.md | 17 +++++++ 10 files changed, 80 insertions(+), 28 deletions(-) delete mode 100644 include/mimalloc-etw.h rename include/mimalloc-etw-gen.h => src/prim/windows/etw.h (100%) rename include/mimalloc-etw-gen.man => src/prim/windows/etw.man (97%) rename {include => src/prim/windows}/mimalloc.wprp (91%) create mode 100644 src/prim/windows/readme.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 28dfe830..7aba5553 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_TRACK_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) option(MI_TRACK_ASAN "Compile with address sanitizer support (adds a small overhead)" OFF) +option(MI_TRACK_ETW "Compile with Windows event tracing (ETW) support (adds a small overhead)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) @@ -165,6 +166,21 @@ if(MI_TRACK_ASAN) endif() endif() +if(MI_TRACK_ETW) + if NOT WIN32 + set(MI_TRACK_ETW OFF) + message(WARNING "Can only enable ETW support on Windows (MI_TRACK_ETW=OFF)") + endif() + if (MI_TRACK_VALGRIND OR MI_TRACK_ASAN) + set(MI_TRACK_ETW OFF) + message(WARNING "Cannot enable ETW support with also Valgrind or ASAN support enabled (MI_TRACK_ETW=OFF)") + endif() + if(MI_TRACK_ETW) + message(STATUS "Compile with Windows event tracing support (MI_TRACK_ETW=ON)") + list(APPEND mi_defines MI_TRACK_ETW=1) + endif() +endif() + if(MI_SEE_ASM) message(STATUS "Generate assembly listings (MI_SEE_ASM=ON)") list(APPEND mi_cflags -save-temps) diff --git a/include/mimalloc-etw.h b/include/mimalloc-etw.h deleted file mode 100644 index 3c894056..00000000 --- a/include/mimalloc-etw.h +++ /dev/null @@ -1,4 +0,0 @@ -#pragma once -#include -#include "mimalloc-etw-gen.h" - diff --git a/include/mimalloc-track.h b/include/mimalloc-track.h index 5e050512..f78e8daa 100644 --- a/include/mimalloc-track.h +++ b/include/mimalloc-track.h @@ -27,10 +27,12 @@ Optional: #define mi_track_align(p,alignedp,offset,size) #define mi_track_resize(p,oldsize,newsize) + #define mi_track_init() The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block. The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`). The `mi_track_resize` is currently unused but could be called on reallocations within a block. +`mi_track_init` is called at program start. The following macros are for tools like asan and valgrind to track whether memory is defined, undefined, or not accessible at all: @@ -42,6 +44,7 @@ defined, undefined, or not accessible at all: -------------------------------------------------------------------------------------------------------*/ #if MI_TRACK_VALGRIND +// valgrind tool #define MI_TRACK_ENABLED 1 #define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on heap_destroy @@ -58,6 +61,7 @@ defined, undefined, or not accessible at all: #define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) #elif MI_TRACK_ASAN +// address sanitizer #define MI_TRACK_ENABLED 1 #define MI_TRACK_HEAP_DESTROY 0 @@ -71,19 +75,23 @@ defined, undefined, or not accessible at all: #define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) #define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size) -#elif MI_ETW -#define MI_TRACK_ENABLED 1 -#define MI_TRACK_TOOL "ETW" +#elif MI_TRACK_ETW +// windows event tracing -#include "mimalloc-etw.h" +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_TOOL "ETW" -#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)p, size) -#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)p, size) -#define mi_track_mem_defined(p,size) -#define mi_track_mem_undefined(p,size) -#define mi_track_mem_noaccess(p,size) +#define WIN32_LEAN_AND_MEAN +#include +#include "../src/prim/windows/etw.h" + +#define mi_track_init() EventRegistermicrosoft_windows_mimalloc(); +#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size) +#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size) #else +// no tracking #define MI_TRACK_ENABLED 0 #define MI_TRACK_HEAP_DESTROY 0 @@ -91,11 +99,6 @@ defined, undefined, or not accessible at all: #define mi_track_malloc_size(p,reqsize,size,zero) #define mi_track_free_size(p,_size) -#define mi_track_align(p,alignedp,offset,size) -#define mi_track_resize(p,oldsize,newsize) -#define mi_track_mem_defined(p,size) -#define mi_track_mem_undefined(p,size) -#define mi_track_mem_noaccess(p,size) #endif @@ -110,6 +113,23 @@ defined, undefined, or not accessible at all: #define mi_track_align(p,alignedp,offset,size) mi_track_mem_noaccess(p,offset) #endif +#ifndef mi_track_init +#define mi_track_init() +#endif + +#ifndef mi_track_mem_defined +#define mi_track_mem_defined(p,size) +#endif + +#ifndef mi_track_mem_undefined +#define mi_track_mem_undefined(p,size) +#endif + +#ifndef mi_track_mem_noaccess +#define mi_track_mem_noaccess(p,size) +#endif + + #if MI_PADDING #define mi_track_malloc(p,reqsize,zero) \ if ((p)!=NULL) { \ diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index f8d891cb..ebf764ab 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -32,8 +32,7 @@ terms of the MIT license. A copy of the license can be found in the file // Define MI_TRACK_ to enable tracking support // #define MI_TRACK_VALGRIND 1 // #define MI_TRACK_ASAN 1 -// Define MI_ETW to enable ETW provider - #define MI_ETW 1 +#define MI_TRACK_ETW 1 // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). // #define MI_STAT 1 @@ -62,7 +61,7 @@ terms of the MIT license. A copy of the license can be found in the file // Reserve extra padding at the end of each block to be more resilient against heap block overflows. // The padding can detect buffer overflow on free. -#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_ETW) +#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) #define MI_PADDING 1 #endif diff --git a/src/init.c b/src/init.c index 5574548e..495d26fd 100644 --- a/src/init.c +++ b/src/init.c @@ -534,6 +534,7 @@ void mi_process_init(void) mi_attr_noexcept { #endif mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL) + mi_track_init(); if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024); @@ -550,10 +551,6 @@ void mi_process_init(void) mi_attr_noexcept { mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true); } } - -#ifdef MI_ETW - EventRegistermicrosoft_windows_mimalloc(); -#endif } // Called when the process is done (through `at_exit`) diff --git a/src/prim/readme.md b/src/prim/readme.md index 14248496..eb02f274 100644 --- a/src/prim/readme.md +++ b/src/prim/readme.md @@ -1,6 +1,8 @@ +## Portability Primitives + This is the portability layer where all primitives needed from the OS are defined. - `prim.h`: API definition -- `prim.c`: Selects one of `prim-unix.c`, `prim-wasi.c`, or `prim-windows.c` depending on the host platform. +- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform. -Note: still work in progress, there may be other places in the sources that still depend on OS ifdef's. \ No newline at end of file +Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's. \ No newline at end of file diff --git a/include/mimalloc-etw-gen.h b/src/prim/windows/etw.h similarity index 100% rename from include/mimalloc-etw-gen.h rename to src/prim/windows/etw.h diff --git a/include/mimalloc-etw-gen.man b/src/prim/windows/etw.man similarity index 97% rename from include/mimalloc-etw-gen.man rename to src/prim/windows/etw.man index 9ef58029233f5296ce71f657e46e068ef2d16418..cfd1f8a9eaacd50af63f1e28f9540aa88c20f90c 100644 GIT binary patch delta 26 hcmaDScTH}>F8)-85{7aHJ%(I{M20*Dg^dq;`2cy32w?yK delta 50 zcmca6_fBrYF7;f7Oom*BM1~w7%x6euh-XM;C}AjP&}B#mvho=8z_NK8PxkTw0BG(F ARR910 diff --git a/include/mimalloc.wprp b/src/prim/windows/mimalloc.wprp similarity index 91% rename from include/mimalloc.wprp rename to src/prim/windows/mimalloc.wprp index 7f284e9d..b00cd7ad 100644 --- a/include/mimalloc.wprp +++ b/src/prim/windows/mimalloc.wprp @@ -29,7 +29,12 @@ - + + + + + + diff --git a/src/prim/windows/readme.md b/src/prim/windows/readme.md new file mode 100644 index 00000000..70292231 --- /dev/null +++ b/src/prim/windows/readme.md @@ -0,0 +1,17 @@ +## Primitives: + +- `prim.c` contains Windows primitives for OS allocation. + +## Event Tracing for Windows (ETW) + +- `etw.h` is generated from `etw.man` which contains the manifest for mimalloc events. + (100 is an allocation, 101 is for a free) + +- `mimalloc.wprp` is a profile for the Windows Performance Recorder (WPR). + In an admin prompt, you can use: + ``` + > wpr -start src\prim\windows\mimalloc.wprp -filemode + > + > wpr -stop test.etl + ``` + and then open `test.etl` in the Windows Performance Analyzer (WPA). \ No newline at end of file From 63f88cb43d1b381222f8d371646e75f535e86733 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 16 Mar 2023 20:10:46 -0700 Subject: [PATCH 045/102] rename --- src/prim/windows/{mimalloc.wprp => etw-mimalloc.wprp} | 0 src/prim/windows/readme.md | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/prim/windows/{mimalloc.wprp => etw-mimalloc.wprp} (100%) diff --git a/src/prim/windows/mimalloc.wprp b/src/prim/windows/etw-mimalloc.wprp similarity index 100% rename from src/prim/windows/mimalloc.wprp rename to src/prim/windows/etw-mimalloc.wprp diff --git a/src/prim/windows/readme.md b/src/prim/windows/readme.md index 70292231..217c3d17 100644 --- a/src/prim/windows/readme.md +++ b/src/prim/windows/readme.md @@ -7,10 +7,10 @@ - `etw.h` is generated from `etw.man` which contains the manifest for mimalloc events. (100 is an allocation, 101 is for a free) -- `mimalloc.wprp` is a profile for the Windows Performance Recorder (WPR). +- `etw-mimalloc.wprp` is a profile for the Windows Performance Recorder (WPR). In an admin prompt, you can use: ``` - > wpr -start src\prim\windows\mimalloc.wprp -filemode + > wpr -start src\prim\windows\etw-mimalloc.wprp -filemode > > wpr -stop test.etl ``` From 3ebcc0bac45a1e270f1a1cc03e9af1b9d91fce33 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 16 Mar 2023 20:13:21 -0700 Subject: [PATCH 046/102] fix syntax in cmakelists --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7aba5553..68b7cab4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,7 +167,7 @@ if(MI_TRACK_ASAN) endif() if(MI_TRACK_ETW) - if NOT WIN32 + if(NOT WIN32) set(MI_TRACK_ETW OFF) message(WARNING "Can only enable ETW support on Windows (MI_TRACK_ETW=OFF)") endif() From 17a20f280b86d087f5ed84f5c23c1c5ab1d7a51e Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 16 Mar 2023 20:16:31 -0700 Subject: [PATCH 047/102] dont track ETW by default --- include/mimalloc-types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index ebf764ab..3577b23a 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -32,7 +32,7 @@ terms of the MIT license. A copy of the license can be found in the file // Define MI_TRACK_ to enable tracking support // #define MI_TRACK_VALGRIND 1 // #define MI_TRACK_ASAN 1 -#define MI_TRACK_ETW 1 +// #define MI_TRACK_ETW 1 // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). // #define MI_STAT 1 From cbccbbe9a4614ee25407a6b4a50cda5df6a2461e Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 18 Mar 2023 11:11:49 +0000 Subject: [PATCH 048/102] c++ override test new placement operator --- test/main-override.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index db96efb1..5e4eed6a 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -95,6 +95,10 @@ static void various_tests() { delete t; t = new (std::nothrow) Test(42); delete t; + auto tbuf = new unsigned char[sizeof(Test)]; + t = new (tbuf) Test(42); + t->~Test(); + delete tbuf; } class Static { @@ -298,4 +302,4 @@ static void tsan_numa_test() { auto t1 = std::thread(dummy_worker); dummy_worker(); t1.join(); -} \ No newline at end of file +} From 8fbe7aae50a959cbb5324f675dcfd8c4ff18312d Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 19 Mar 2023 19:11:43 -0700 Subject: [PATCH 049/102] update process info primitive api --- src/prim/prim.c | 2 +- src/prim/prim.h | 16 +++++++++++++--- src/prim/unix/prim.c | 36 ++++++++++++++---------------------- src/prim/wasi/prim.c | 12 ++++-------- src/prim/windows/prim.c | 16 ++++++++-------- src/stats.c | 36 +++++++++++++++++++----------------- 6 files changed, 59 insertions(+), 59 deletions(-) diff --git a/src/prim/prim.c b/src/prim/prim.c index eec13c48..109ab8e8 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "windows/prim.c" // VirtualAlloc (Windows) #elif defined(__wasi__) #define MI_USE_SBRK -#include "wasi/prim.h" // memory-grow or sbrk (Wasm) +#include "wasi/prim.c" // memory-grow or sbrk (Wasm) #else #include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) #endif diff --git a/src/prim/prim.h b/src/prim/prim.h index 967c6698..3130d489 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -59,9 +59,18 @@ size_t _mi_prim_numa_node_count(void); mi_msecs_t _mi_prim_clock_now(void); // Return process information (only for statistics) -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, - size_t* current_rss, size_t* peak_rss, - size_t* current_commit, size_t* peak_commit, size_t* page_faults); +typedef struct mi_process_info_s { + mi_msecs_t elapsed; + mi_msecs_t utime; + mi_msecs_t stime; + size_t current_rss; + size_t peak_rss; + size_t current_commit; + size_t peak_commit; + size_t page_faults; +} mi_process_info_t; + +void _mi_prim_process_info(mi_process_info_t* pinfo); // Default stderr output. (only for warnings etc. with verbose enabled) // msg != NULL && _mi_strlen(msg) > 0 @@ -202,6 +211,7 @@ This is inlined here as it is on the fast path for allocation functions. On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a __thread local variable (`_mi_heap_default`). With the initial-exec TLS model this ensures that the storage will always be available (allocated on the thread stacks). + On some platforms though we cannot use that when overriding `malloc` since the underlying TLS implementation (or the loader) will call itself `malloc` on a first access and recurse. We try to circumvent this in an efficient way: diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index d1cd4301..1040c791 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -541,19 +541,15 @@ static mi_msecs_t timeval_secs(const struct timeval* tv) { return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L); } -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +void _mi_prim_process_info(mi_process_info_t* pinfo) { struct rusage rusage; getrusage(RUSAGE_SELF, &rusage); - *utime = timeval_secs(&rusage.ru_utime); - *stime = timeval_secs(&rusage.ru_stime); + pinfo->utime = timeval_secs(&rusage.ru_utime); + pinfo->stime = timeval_secs(&rusage.ru_stime); #if !defined(__HAIKU__) - *page_faults = rusage.ru_majflt; -#endif - // estimate commit using our stats - *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); - *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); - *current_rss = *current_commit; // estimate + pinfo->page_faults = rusage.ru_majflt; +#endif #if defined(__HAIKU__) // Haiku does not have (yet?) a way to // get these stats per process @@ -562,19 +558,20 @@ void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current ssize_t c; get_thread_info(find_thread(0), &tid); while (get_next_area_info(tid.team, &c, &mem) == B_OK) { - *peak_rss += mem.ram_size; + pinfo->peak_rss += mem.ram_size; } - *page_faults = 0; + pinfo->page_faults = 0; #elif defined(__APPLE__) - *peak_rss = rusage.ru_maxrss; // BSD reports in bytes + pinfo->peak_rss = rusage.ru_maxrss; // BSD reports in bytes struct mach_task_basic_info info; mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { - *current_rss = (size_t)info.resident_size; + pinfo->current_rss = (size_t)info.resident_size; } #else - *peak_rss = rusage.ru_maxrss * 1024; // Linux reports in KiB + pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux reports in KiB #endif + // use defaults for commit } #else @@ -584,15 +581,10 @@ void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current #pragma message("define a way to get process info") #endif -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +void _mi_prim_process_info(mi_process_info_t* pinfo) { - *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); - *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); - *peak_rss = *peak_commit; - *current_rss = *current_commit; - *page_faults = 0; - *utime = 0; - *stime = 0; + // use defaults + MI_UNUSED(pinfo); } #endif diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index b8ac1a1b..89c04d78 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -194,17 +194,13 @@ mi_msecs_t _mi_prim_clock_now(void) { // Process info //---------------------------------------------------------------- -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +void _mi_prim_process_info(mi_process_info_t* pinfo) { - *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); - *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); - *peak_rss = *peak_commit; - *current_rss = *current_commit; - *page_faults = 0; - *utime = 0; - *stime = 0; + // use defaults + MI_UNUSED(pinfo); } + //---------------------------------------------------------------- // Output //---------------------------------------------------------------- diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 2fa445a1..1ce44a10 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -428,15 +428,15 @@ static mi_msecs_t filetime_msecs(const FILETIME* ftime) { typedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD); static PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL; -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +void _mi_prim_process_info(mi_process_info_t* pinfo) { FILETIME ct; FILETIME ut; FILETIME st; FILETIME et; GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); - *utime = filetime_msecs(&ut); - *stime = filetime_msecs(&st); + pinfo->utime = filetime_msecs(&ut); + pinfo->stime = filetime_msecs(&st); // load psapi on demand if (pGetProcessMemoryInfo == NULL) { @@ -452,11 +452,11 @@ void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current if (pGetProcessMemoryInfo != NULL) { pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); } - *current_rss = (size_t)info.WorkingSetSize; - *peak_rss = (size_t)info.PeakWorkingSetSize; - *current_commit = (size_t)info.PagefileUsage; - *peak_commit = (size_t)info.PeakPagefileUsage; - *page_faults = (size_t)info.PageFaultCount; + pinfo->current_rss = (size_t)info.WorkingSetSize; + pinfo->peak_rss = (size_t)info.PeakWorkingSetSize; + pinfo->current_commit = (size_t)info.PagefileUsage; + pinfo->peak_commit = (size_t)info.PeakPagefileUsage; + pinfo->page_faults = (size_t)info.PageFaultCount; } //---------------------------------------------------------------- diff --git a/src/stats.c b/src/stats.c index 357bebce..4bc8835c 100644 --- a/src/stats.c +++ b/src/stats.c @@ -430,21 +430,23 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) { mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept { - mi_msecs_t elapsed = _mi_clock_end(mi_process_start); - mi_msecs_t utime = 0; - mi_msecs_t stime = 0; - size_t current_rss0 = 0; - size_t peak_rss0 = 0; - size_t current_commit0 = 0; - size_t peak_commit0 = 0; - size_t page_faults0 = 0; - _mi_prim_process_info(&utime, &stime, ¤t_rss0, &peak_rss0, ¤t_commit0, &peak_commit0, &page_faults0); - if (elapsed_msecs!=NULL) *elapsed_msecs = (elapsed < 0 ? 0 : (elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)elapsed : PTRDIFF_MAX)); - if (user_msecs!=NULL) *user_msecs = (utime < 0 ? 0 : (utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)utime : PTRDIFF_MAX)); - if (system_msecs!=NULL) *system_msecs = (stime < 0 ? 0 : (stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)stime : PTRDIFF_MAX)); - if (current_rss!=NULL) *current_rss = current_rss0; - if (peak_rss!=NULL) *peak_rss = peak_rss0; - if (current_commit!=NULL) *current_commit = current_commit0; - if (peak_commit!=NULL) *peak_commit = peak_commit0; - if (page_faults!=NULL) *page_faults = page_faults0; + mi_process_info_t pinfo = { 0 }; + pinfo.elapsed = _mi_clock_end(mi_process_start); + pinfo.utime = 0; + pinfo.stime = 0; + pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); + pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); + pinfo.current_rss = pinfo.current_commit; + pinfo.peak_rss = pinfo.peak_commit; + pinfo.page_faults = 0; + + _mi_prim_process_info(&pinfo); + if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX)); + if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX)); + if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX)); + if (current_rss!=NULL) *current_rss = pinfo.current_rss; + if (peak_rss!=NULL) *peak_rss = pinfo.peak_rss; + if (current_commit!=NULL) *current_commit = pinfo.current_commit; + if (peak_commit!=NULL) *peak_commit = pinfo.peak_commit; + if (page_faults!=NULL) *page_faults = pinfo.page_faults; } From 99c9f55511ea62e80cf7dd28182799a940d4b6bd Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 19 Mar 2023 20:21:20 -0700 Subject: [PATCH 050/102] simplify primitives API --- src/os.c | 27 +++++++++++++------ src/prim/prim.h | 7 ++--- src/prim/unix/prim.c | 60 ++++++++++++++++++++++------------------- src/prim/wasi/prim.c | 19 ++++++++----- src/prim/windows/prim.c | 28 ++++++++++--------- 5 files changed, 83 insertions(+), 58 deletions(-) diff --git a/src/os.c b/src/os.c index 56f96bf2..5263af42 100644 --- a/src/os.c +++ b/src/os.c @@ -135,7 +135,10 @@ static void mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats MI_UNUSED(tld_stats); mi_assert_internal((size % _mi_os_page_size()) == 0); if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) - _mi_prim_free(addr, size); + int err = _mi_prim_free(addr, size); + if (err != 0) { + _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); + } mi_stats_t* stats = &_mi_stats_main; if (was_committed) { _mi_stat_decrease(&stats->committed, size); } _mi_stat_decrease(&stats->reserved, size); @@ -163,7 +166,11 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo if (!commit) allow_large = false; if (try_alignment == 0) try_alignment = 1; // avoid 0 to ensure there will be no divide by zero when aligning - void* p = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large); + void* p = NULL; + int err = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large, &p); + if (err != 0) { + _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, size, try_alignment, commit, allow_large); + } /* if (commit && allow_large) { p = _mi_os_try_alloc_from_huge_reserved(size, try_alignment); @@ -200,7 +207,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, // if not aligned, free it, overallocate, and unmap around it if (((uintptr_t)p % alignment != 0)) { mi_os_mem_free(p, size, commit, stats); - _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (%zu bytes, address: %p, alignment: %zu, commit: %d)\n", size, p, alignment, commit); + _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; @@ -357,7 +364,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ int err = _mi_prim_commit(start, csize, commit); if (err != 0) { - _mi_warning_message("%s error: start: %p, csize: 0x%zx, err: %i\n", commit ? "commit" : "decommit", start, csize, err); + _mi_warning_message("cannot %s OS memory (error: %d (0x%d), address: %p, size: 0x%zx bytes)\n", commit ? "commit" : "decommit", err, err, start, csize); } mi_assert_internal(err == 0); return (err == 0); @@ -404,7 +411,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) int err = _mi_prim_reset(start, csize); if (err != 0) { - _mi_warning_message("madvise reset error: start: %p, csize: 0x%zx, errno: %i\n", start, csize, err); + _mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); } return (err == 0); } @@ -441,7 +448,7 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) { */ int err = _mi_prim_protect(start,csize,protect); if (err != 0) { - _mi_warning_message("mprotect error: start: %p, csize: 0x%zx, err: %i\n", start, csize, err); + _mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (protect ? "protect" : "unprotect"), err, err, start, csize); } return (err == 0); } @@ -516,13 +523,17 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse for (page = 0; page < pages; page++) { // allocate a page void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE); - void* p = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node); + void* p = NULL; + int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &p); + if (err != 0) { + _mi_warning_message("unable to allocate huge OS page (error: %d (0x%d), address: %p, size: %zx bytes)", err, err, addr, MI_HUGE_OS_PAGE_SIZE); + } // Did we succeed at a contiguous address? if (p != addr) { // no success, issue a warning and break if (p != NULL) { - _mi_warning_message("could not allocate contiguous huge page %zu at %p\n", page, addr); + _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); _mi_os_free(p, MI_HUGE_OS_PAGE_SIZE, &_mi_stats_main); } break; diff --git a/src/prim/prim.h b/src/prim/prim.h index 3130d489..1a4fb5d8 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -11,6 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file // note: on all primitive functions, we always get: // addr != NULL and page aligned // size > 0 and page aligned +// return value is an error code an int where 0 is success. // OS memory configuration typedef struct mi_os_mem_config_s { @@ -25,13 +26,13 @@ typedef struct mi_os_mem_config_s { void _mi_prim_mem_init( mi_os_mem_config_t* config ); // Free OS memory -void _mi_prim_free(void* addr, size_t size ); +int _mi_prim_free(void* addr, size_t size ); // Allocate OS memory. Return NULL on error. // The `try_alignment` is just a hint and the returned pointer does not have to be aligned. // pre: !commit => !allow_large // try_alignment >= _mi_os_page_size() and a power of 2 -void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large); +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, void** addr); // Commit memory. Returns error code or 0 on success. int _mi_prim_commit(void* addr, size_t size, bool commit); @@ -47,7 +48,7 @@ int _mi_prim_protect(void* addr, size_t size, bool protect); // pre: size > 0 and a multiple of 1GiB. // addr is either NULL or an address hint. // numa_node is either negative (don't care), or a numa node number. -void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node); +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, void** addr); // Return the current NUMA node size_t _mi_prim_numa_node(void); diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 1040c791..5a3ca5ab 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -96,11 +96,9 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) { // free //--------------------------------------------- -void _mi_prim_free(void* addr, size_t size ) { +int _mi_prim_free(void* addr, size_t size ) { bool err = (munmap(addr, size) == -1); - if (err) { - _mi_warning_message("unable to release OS memory: %s, addr: %p, size: %zu\n", strerror(errno), addr, size); - } + return (err ? errno : 0); } @@ -118,19 +116,24 @@ static int unix_madvise(void* addr, size_t size, int advice) { static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { MI_UNUSED(try_alignment); + void* p = NULL; #if defined(MAP_ALIGNED) // BSD if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { size_t n = mi_bsr(try_alignment); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB flags |= MAP_ALIGNED(n); - void* p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + int err = errno; + _mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%d), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + } if (p!=MAP_FAILED) return p; - // fall back to regular mmap + // fall back to regular mmap } } #elif defined(MAP_ALIGN) // Solaris if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { - void* p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); // addr parameter is the required alignment + p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); // addr parameter is the required alignment if (p!=MAP_FAILED) return p; // fall back to regular mmap } @@ -140,14 +143,18 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p if (addr == NULL) { void* hint = _mi_os_get_aligned_hint(try_alignment, size); if (hint != NULL) { - void* p = mmap(hint, size, protect_flags, flags, fd, 0); + p = mmap(hint, size, protect_flags, flags, fd, 0); + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + int err = errno; + _mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%d), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + } if (p!=MAP_FAILED) return p; - // fall back to regular mmap + // fall back to regular mmap } } #endif // regular mmap - void* p = mmap(addr, size, protect_flags, flags, fd, 0); + p = mmap(addr, size, protect_flags, flags, fd, 0); if (p!=MAP_FAILED) return p; // failed to allocate return NULL; @@ -217,7 +224,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec #ifdef MAP_HUGE_1GB if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) { mi_huge_pages_available = false; // don't try huge 1GiB pages again - _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (error %i)\n", errno); + _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno); lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB); p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd); } @@ -258,20 +265,18 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec #endif } } - if (p == NULL) { - _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, errno, addr, large_only, allow_large); - } return p; } // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) { +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, void** addr) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(commit || !allow_large); mi_assert_internal(try_alignment > 0); int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); - return unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); + *addr = unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); + return (*addr != NULL ? 0 : errno); } @@ -379,28 +384,29 @@ static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, co } #endif -void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) { +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, void** addr) { bool is_large = true; - void* p = unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); - if (p == NULL) return NULL; - if (numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes + *addr = unix_mmap(hint_addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); + if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes unsigned long numa_mask = (1UL << numa_node); // TODO: does `mbind` work correctly for huge OS pages? should we // use `set_mempolicy` before calling mmap instead? // see: - long err = mi_prim_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); + long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); if (err != 0) { - _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d: %s\n", numa_node, strerror(errno)); - } + err = errno; + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%d))\n", numa_node, err, err); + } } - return p; + return (*addr != NULL ? 0 : errno); } #else -void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) { - MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node); - return NULL; +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, void** addr) { + MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); + *addr = NULL; + return ENOMEM; } #endif diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index 89c04d78..f995304f 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -27,9 +27,10 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) { // Free //--------------------------------------------- -void _mi_prim_free(void* addr, size_t size ) { +int _mi_prim_free(void* addr, size_t size ) { MI_UNUSED(addr); MI_UNUSED(size); // wasi heap cannot be shrunk + return 0; } @@ -101,20 +102,23 @@ static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { } } } + /* if (p == NULL) { _mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment); errno = ENOMEM; return NULL; } - mi_assert_internal( try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); + */ + mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); return p; } // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) { +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, void** addr) { MI_UNUSED(allow_large); MI_UNUSED(commit); *is_large = false; - return mi_prim_mem_grow(size, try_alignment); + *addr = mi_prim_mem_grow(size, try_alignment); + return (*addr != NULL ? 0 : ENOMEM); } @@ -142,9 +146,10 @@ int _mi_prim_protect(void* addr, size_t size, bool protect) { // Huge pages and NUMA nodes //--------------------------------------------- -void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) { - MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node); - return NULL; +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, void** addr) { + MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); + *addr = NULL; + return ENOSYS; } size_t _mi_prim_numa_node(void) { diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 1ce44a10..1e15273a 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -156,7 +156,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) // Free //--------------------------------------------- -void _mi_prim_free(void* addr, size_t size ) { +int _mi_prim_free(void* addr, size_t size ) { DWORD errcode = 0; bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); if (err) { errcode = GetLastError(); } @@ -172,9 +172,7 @@ void _mi_prim_free(void* addr, size_t size ) { if (err) { errcode = GetLastError(); } } } - if (errcode != 0) { - _mi_warning_message("unable to release OS memory: error code 0x%x, addr: %p, size: %zu\n", errcode, addr, size); - } + return (int)errcode; } @@ -240,19 +238,18 @@ static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DW *is_large = ((flags&MEM_LARGE_PAGES) != 0); p = win_virtual_alloc_prim(addr, size, try_alignment, flags); } - if (p == NULL) { - _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); - } + //if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); } return p; } -void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) { +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, void** addr) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(commit || !allow_large); mi_assert_internal(try_alignment > 0); int flags = MEM_RESERVE; if (commit) { flags |= MEM_COMMIT; } - return win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); + *addr = win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); + return (*addr != NULL ? 0 : (int)GetLastError()); } @@ -296,7 +293,7 @@ int _mi_prim_protect(void* addr, size_t size, bool protect) { // Huge page allocation //--------------------------------------------- -void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) +static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int numa_node) { const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE; @@ -315,7 +312,7 @@ void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) params[1].Arg.ULong = (unsigned)numa_node; } SIZE_T psize = size; - void* base = addr; + void* base = hint_addr; NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count); if (err == 0 && base != NULL) { return base; @@ -330,11 +327,16 @@ void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) if (pVirtualAlloc2 != NULL && numa_node >= 0) { params[0].Type.Type = MiMemExtendedParameterNumaNode; params[0].Arg.ULong = (unsigned)numa_node; - return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1); + return (*pVirtualAlloc2)(GetCurrentProcess(), hint_addr, size, flags, PAGE_READWRITE, params, 1); } // otherwise use regular virtual alloc on older windows - return VirtualAlloc(addr, size, flags, PAGE_READWRITE); + return VirtualAlloc(hint_addr, size, flags, PAGE_READWRITE); +} + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, void** addr) { + *addr = _mi_prim_alloc_huge_os_pagesx(hint_addr,size,numa_node); + return (*addr != NULL ? 0 : (int)GetLastError()); } From 85a2bb5c608a451b143dc3dc27a278924229f0e4 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 19 Mar 2023 19:11:43 -0700 Subject: [PATCH 051/102] update process info primitive api --- src/prim/prim.c | 2 +- src/prim/prim.h | 16 +++++++++++++--- src/prim/unix/prim.c | 36 ++++++++++++++---------------------- src/prim/wasi/prim.c | 12 ++++-------- src/prim/windows/prim.c | 16 ++++++++-------- src/stats.c | 36 +++++++++++++++++++----------------- 6 files changed, 59 insertions(+), 59 deletions(-) diff --git a/src/prim/prim.c b/src/prim/prim.c index eec13c48..109ab8e8 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "windows/prim.c" // VirtualAlloc (Windows) #elif defined(__wasi__) #define MI_USE_SBRK -#include "wasi/prim.h" // memory-grow or sbrk (Wasm) +#include "wasi/prim.c" // memory-grow or sbrk (Wasm) #else #include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) #endif diff --git a/src/prim/prim.h b/src/prim/prim.h index 967c6698..3130d489 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -59,9 +59,18 @@ size_t _mi_prim_numa_node_count(void); mi_msecs_t _mi_prim_clock_now(void); // Return process information (only for statistics) -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, - size_t* current_rss, size_t* peak_rss, - size_t* current_commit, size_t* peak_commit, size_t* page_faults); +typedef struct mi_process_info_s { + mi_msecs_t elapsed; + mi_msecs_t utime; + mi_msecs_t stime; + size_t current_rss; + size_t peak_rss; + size_t current_commit; + size_t peak_commit; + size_t page_faults; +} mi_process_info_t; + +void _mi_prim_process_info(mi_process_info_t* pinfo); // Default stderr output. (only for warnings etc. with verbose enabled) // msg != NULL && _mi_strlen(msg) > 0 @@ -202,6 +211,7 @@ This is inlined here as it is on the fast path for allocation functions. On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a __thread local variable (`_mi_heap_default`). With the initial-exec TLS model this ensures that the storage will always be available (allocated on the thread stacks). + On some platforms though we cannot use that when overriding `malloc` since the underlying TLS implementation (or the loader) will call itself `malloc` on a first access and recurse. We try to circumvent this in an efficient way: diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index d1cd4301..1040c791 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -541,19 +541,15 @@ static mi_msecs_t timeval_secs(const struct timeval* tv) { return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L); } -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +void _mi_prim_process_info(mi_process_info_t* pinfo) { struct rusage rusage; getrusage(RUSAGE_SELF, &rusage); - *utime = timeval_secs(&rusage.ru_utime); - *stime = timeval_secs(&rusage.ru_stime); + pinfo->utime = timeval_secs(&rusage.ru_utime); + pinfo->stime = timeval_secs(&rusage.ru_stime); #if !defined(__HAIKU__) - *page_faults = rusage.ru_majflt; -#endif - // estimate commit using our stats - *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); - *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); - *current_rss = *current_commit; // estimate + pinfo->page_faults = rusage.ru_majflt; +#endif #if defined(__HAIKU__) // Haiku does not have (yet?) a way to // get these stats per process @@ -562,19 +558,20 @@ void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current ssize_t c; get_thread_info(find_thread(0), &tid); while (get_next_area_info(tid.team, &c, &mem) == B_OK) { - *peak_rss += mem.ram_size; + pinfo->peak_rss += mem.ram_size; } - *page_faults = 0; + pinfo->page_faults = 0; #elif defined(__APPLE__) - *peak_rss = rusage.ru_maxrss; // BSD reports in bytes + pinfo->peak_rss = rusage.ru_maxrss; // BSD reports in bytes struct mach_task_basic_info info; mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { - *current_rss = (size_t)info.resident_size; + pinfo->current_rss = (size_t)info.resident_size; } #else - *peak_rss = rusage.ru_maxrss * 1024; // Linux reports in KiB + pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux reports in KiB #endif + // use defaults for commit } #else @@ -584,15 +581,10 @@ void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current #pragma message("define a way to get process info") #endif -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +void _mi_prim_process_info(mi_process_info_t* pinfo) { - *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); - *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); - *peak_rss = *peak_commit; - *current_rss = *current_commit; - *page_faults = 0; - *utime = 0; - *stime = 0; + // use defaults + MI_UNUSED(pinfo); } #endif diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index b8ac1a1b..89c04d78 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -194,17 +194,13 @@ mi_msecs_t _mi_prim_clock_now(void) { // Process info //---------------------------------------------------------------- -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +void _mi_prim_process_info(mi_process_info_t* pinfo) { - *peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); - *current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); - *peak_rss = *peak_commit; - *current_rss = *current_commit; - *page_faults = 0; - *utime = 0; - *stime = 0; + // use defaults + MI_UNUSED(pinfo); } + //---------------------------------------------------------------- // Output //---------------------------------------------------------------- diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 2fa445a1..1ce44a10 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -428,15 +428,15 @@ static mi_msecs_t filetime_msecs(const FILETIME* ftime) { typedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD); static PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL; -void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) +void _mi_prim_process_info(mi_process_info_t* pinfo) { FILETIME ct; FILETIME ut; FILETIME st; FILETIME et; GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); - *utime = filetime_msecs(&ut); - *stime = filetime_msecs(&st); + pinfo->utime = filetime_msecs(&ut); + pinfo->stime = filetime_msecs(&st); // load psapi on demand if (pGetProcessMemoryInfo == NULL) { @@ -452,11 +452,11 @@ void _mi_prim_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* current if (pGetProcessMemoryInfo != NULL) { pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); } - *current_rss = (size_t)info.WorkingSetSize; - *peak_rss = (size_t)info.PeakWorkingSetSize; - *current_commit = (size_t)info.PagefileUsage; - *peak_commit = (size_t)info.PeakPagefileUsage; - *page_faults = (size_t)info.PageFaultCount; + pinfo->current_rss = (size_t)info.WorkingSetSize; + pinfo->peak_rss = (size_t)info.PeakWorkingSetSize; + pinfo->current_commit = (size_t)info.PagefileUsage; + pinfo->peak_commit = (size_t)info.PeakPagefileUsage; + pinfo->page_faults = (size_t)info.PageFaultCount; } //---------------------------------------------------------------- diff --git a/src/stats.c b/src/stats.c index c9b3bb95..8273740f 100644 --- a/src/stats.c +++ b/src/stats.c @@ -430,21 +430,23 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) { mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept { - mi_msecs_t elapsed = _mi_clock_end(mi_process_start); - mi_msecs_t utime = 0; - mi_msecs_t stime = 0; - size_t current_rss0 = 0; - size_t peak_rss0 = 0; - size_t current_commit0 = 0; - size_t peak_commit0 = 0; - size_t page_faults0 = 0; - _mi_prim_process_info(&utime, &stime, ¤t_rss0, &peak_rss0, ¤t_commit0, &peak_commit0, &page_faults0); - if (elapsed_msecs!=NULL) *elapsed_msecs = (elapsed < 0 ? 0 : (elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)elapsed : PTRDIFF_MAX)); - if (user_msecs!=NULL) *user_msecs = (utime < 0 ? 0 : (utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)utime : PTRDIFF_MAX)); - if (system_msecs!=NULL) *system_msecs = (stime < 0 ? 0 : (stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)stime : PTRDIFF_MAX)); - if (current_rss!=NULL) *current_rss = current_rss0; - if (peak_rss!=NULL) *peak_rss = peak_rss0; - if (current_commit!=NULL) *current_commit = current_commit0; - if (peak_commit!=NULL) *peak_commit = peak_commit0; - if (page_faults!=NULL) *page_faults = page_faults0; + mi_process_info_t pinfo = { 0 }; + pinfo.elapsed = _mi_clock_end(mi_process_start); + pinfo.utime = 0; + pinfo.stime = 0; + pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); + pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); + pinfo.current_rss = pinfo.current_commit; + pinfo.peak_rss = pinfo.peak_commit; + pinfo.page_faults = 0; + + _mi_prim_process_info(&pinfo); + if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX)); + if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX)); + if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX)); + if (current_rss!=NULL) *current_rss = pinfo.current_rss; + if (peak_rss!=NULL) *peak_rss = pinfo.peak_rss; + if (current_commit!=NULL) *current_commit = pinfo.current_commit; + if (peak_commit!=NULL) *peak_commit = pinfo.peak_commit; + if (page_faults!=NULL) *page_faults = pinfo.page_faults; } From 6ae6c427001e5b01f4f62d97c0b8c1cdca8c2888 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 19 Mar 2023 20:21:20 -0700 Subject: [PATCH 052/102] simplify primitives API --- src/os.c | 27 +++++++++++++------ src/prim/prim.h | 7 ++--- src/prim/unix/prim.c | 60 ++++++++++++++++++++++------------------- src/prim/wasi/prim.c | 19 ++++++++----- src/prim/windows/prim.c | 28 ++++++++++--------- 5 files changed, 83 insertions(+), 58 deletions(-) diff --git a/src/os.c b/src/os.c index a91bbb91..85c3652f 100644 --- a/src/os.c +++ b/src/os.c @@ -146,7 +146,10 @@ static void mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats MI_UNUSED(tld_stats); mi_assert_internal((size % _mi_os_page_size()) == 0); if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) - _mi_prim_free(addr, size); + int err = _mi_prim_free(addr, size); + if (err != 0) { + _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); + } mi_stats_t* stats = &_mi_stats_main; if (was_committed) { _mi_stat_decrease(&stats->committed, size); } _mi_stat_decrease(&stats->reserved, size); @@ -174,7 +177,11 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo if (!commit) allow_large = false; if (try_alignment == 0) try_alignment = 1; // avoid 0 to ensure there will be no divide by zero when aligning - void* p = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large); + void* p = NULL; + int err = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large, &p); + if (err != 0) { + _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, size, try_alignment, commit, allow_large); + } /* if (commit && allow_large) { p = _mi_os_try_alloc_from_huge_reserved(size, try_alignment); @@ -211,7 +218,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, // if not aligned, free it, overallocate, and unmap around it if (((uintptr_t)p % alignment != 0)) { mi_os_mem_free(p, size, commit, stats); - _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (%zu bytes, address: %p, alignment: %zu, commit: %d)\n", size, p, alignment, commit); + _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; @@ -368,7 +375,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ int err = _mi_prim_commit(start, csize, commit); if (err != 0) { - _mi_warning_message("%s error: start: %p, csize: 0x%zx, err: %i\n", commit ? "commit" : "decommit", start, csize, err); + _mi_warning_message("cannot %s OS memory (error: %d (0x%d), address: %p, size: 0x%zx bytes)\n", commit ? "commit" : "decommit", err, err, start, csize); } mi_assert_internal(err == 0); return (err == 0); @@ -412,7 +419,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) int err = _mi_prim_reset(start, csize); if (err != 0) { - _mi_warning_message("madvise reset error: start: %p, csize: 0x%zx, errno: %i\n", start, csize, err); + _mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); } return (err == 0); } @@ -448,7 +455,7 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) { */ int err = _mi_prim_protect(start,csize,protect); if (err != 0) { - _mi_warning_message("mprotect error: start: %p, csize: 0x%zx, err: %i\n", start, csize, err); + _mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (protect ? "protect" : "unprotect"), err, err, start, csize); } return (err == 0); } @@ -523,13 +530,17 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse for (page = 0; page < pages; page++) { // allocate a page void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE); - void* p = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node); + void* p = NULL; + int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &p); + if (err != 0) { + _mi_warning_message("unable to allocate huge OS page (error: %d (0x%d), address: %p, size: %zx bytes)", err, err, addr, MI_HUGE_OS_PAGE_SIZE); + } // Did we succeed at a contiguous address? if (p != addr) { // no success, issue a warning and break if (p != NULL) { - _mi_warning_message("could not allocate contiguous huge page %zu at %p\n", page, addr); + _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); _mi_os_free(p, MI_HUGE_OS_PAGE_SIZE, &_mi_stats_main); } break; diff --git a/src/prim/prim.h b/src/prim/prim.h index 3130d489..1a4fb5d8 100644 --- a/src/prim/prim.h +++ b/src/prim/prim.h @@ -11,6 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file // note: on all primitive functions, we always get: // addr != NULL and page aligned // size > 0 and page aligned +// return value is an error code an int where 0 is success. // OS memory configuration typedef struct mi_os_mem_config_s { @@ -25,13 +26,13 @@ typedef struct mi_os_mem_config_s { void _mi_prim_mem_init( mi_os_mem_config_t* config ); // Free OS memory -void _mi_prim_free(void* addr, size_t size ); +int _mi_prim_free(void* addr, size_t size ); // Allocate OS memory. Return NULL on error. // The `try_alignment` is just a hint and the returned pointer does not have to be aligned. // pre: !commit => !allow_large // try_alignment >= _mi_os_page_size() and a power of 2 -void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large); +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, void** addr); // Commit memory. Returns error code or 0 on success. int _mi_prim_commit(void* addr, size_t size, bool commit); @@ -47,7 +48,7 @@ int _mi_prim_protect(void* addr, size_t size, bool protect); // pre: size > 0 and a multiple of 1GiB. // addr is either NULL or an address hint. // numa_node is either negative (don't care), or a numa node number. -void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node); +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, void** addr); // Return the current NUMA node size_t _mi_prim_numa_node(void); diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 1040c791..5a3ca5ab 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -96,11 +96,9 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) { // free //--------------------------------------------- -void _mi_prim_free(void* addr, size_t size ) { +int _mi_prim_free(void* addr, size_t size ) { bool err = (munmap(addr, size) == -1); - if (err) { - _mi_warning_message("unable to release OS memory: %s, addr: %p, size: %zu\n", strerror(errno), addr, size); - } + return (err ? errno : 0); } @@ -118,19 +116,24 @@ static int unix_madvise(void* addr, size_t size, int advice) { static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { MI_UNUSED(try_alignment); + void* p = NULL; #if defined(MAP_ALIGNED) // BSD if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { size_t n = mi_bsr(try_alignment); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB flags |= MAP_ALIGNED(n); - void* p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + int err = errno; + _mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%d), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + } if (p!=MAP_FAILED) return p; - // fall back to regular mmap + // fall back to regular mmap } } #elif defined(MAP_ALIGN) // Solaris if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { - void* p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); // addr parameter is the required alignment + p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); // addr parameter is the required alignment if (p!=MAP_FAILED) return p; // fall back to regular mmap } @@ -140,14 +143,18 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p if (addr == NULL) { void* hint = _mi_os_get_aligned_hint(try_alignment, size); if (hint != NULL) { - void* p = mmap(hint, size, protect_flags, flags, fd, 0); + p = mmap(hint, size, protect_flags, flags, fd, 0); + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + int err = errno; + _mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%d), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + } if (p!=MAP_FAILED) return p; - // fall back to regular mmap + // fall back to regular mmap } } #endif // regular mmap - void* p = mmap(addr, size, protect_flags, flags, fd, 0); + p = mmap(addr, size, protect_flags, flags, fd, 0); if (p!=MAP_FAILED) return p; // failed to allocate return NULL; @@ -217,7 +224,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec #ifdef MAP_HUGE_1GB if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) { mi_huge_pages_available = false; // don't try huge 1GiB pages again - _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (error %i)\n", errno); + _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno); lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB); p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd); } @@ -258,20 +265,18 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec #endif } } - if (p == NULL) { - _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, errno, addr, large_only, allow_large); - } return p; } // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) { +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, void** addr) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(commit || !allow_large); mi_assert_internal(try_alignment > 0); int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); - return unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); + *addr = unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); + return (*addr != NULL ? 0 : errno); } @@ -379,28 +384,29 @@ static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, co } #endif -void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) { +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, void** addr) { bool is_large = true; - void* p = unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); - if (p == NULL) return NULL; - if (numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes + *addr = unix_mmap(hint_addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); + if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes unsigned long numa_mask = (1UL << numa_node); // TODO: does `mbind` work correctly for huge OS pages? should we // use `set_mempolicy` before calling mmap instead? // see: - long err = mi_prim_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); + long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); if (err != 0) { - _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d: %s\n", numa_node, strerror(errno)); - } + err = errno; + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%d))\n", numa_node, err, err); + } } - return p; + return (*addr != NULL ? 0 : errno); } #else -void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) { - MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node); - return NULL; +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, void** addr) { + MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); + *addr = NULL; + return ENOMEM; } #endif diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index 89c04d78..f995304f 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -27,9 +27,10 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) { // Free //--------------------------------------------- -void _mi_prim_free(void* addr, size_t size ) { +int _mi_prim_free(void* addr, size_t size ) { MI_UNUSED(addr); MI_UNUSED(size); // wasi heap cannot be shrunk + return 0; } @@ -101,20 +102,23 @@ static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { } } } + /* if (p == NULL) { _mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment); errno = ENOMEM; return NULL; } - mi_assert_internal( try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); + */ + mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); return p; } // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) { +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, void** addr) { MI_UNUSED(allow_large); MI_UNUSED(commit); *is_large = false; - return mi_prim_mem_grow(size, try_alignment); + *addr = mi_prim_mem_grow(size, try_alignment); + return (*addr != NULL ? 0 : ENOMEM); } @@ -142,9 +146,10 @@ int _mi_prim_protect(void* addr, size_t size, bool protect) { // Huge pages and NUMA nodes //--------------------------------------------- -void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) { - MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node); - return NULL; +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, void** addr) { + MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); + *addr = NULL; + return ENOSYS; } size_t _mi_prim_numa_node(void) { diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 1ce44a10..1e15273a 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -156,7 +156,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) // Free //--------------------------------------------- -void _mi_prim_free(void* addr, size_t size ) { +int _mi_prim_free(void* addr, size_t size ) { DWORD errcode = 0; bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); if (err) { errcode = GetLastError(); } @@ -172,9 +172,7 @@ void _mi_prim_free(void* addr, size_t size ) { if (err) { errcode = GetLastError(); } } } - if (errcode != 0) { - _mi_warning_message("unable to release OS memory: error code 0x%x, addr: %p, size: %zu\n", errcode, addr, size); - } + return (int)errcode; } @@ -240,19 +238,18 @@ static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DW *is_large = ((flags&MEM_LARGE_PAGES) != 0); p = win_virtual_alloc_prim(addr, size, try_alignment, flags); } - if (p == NULL) { - _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); - } + //if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); } return p; } -void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) { +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, void** addr) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(commit || !allow_large); mi_assert_internal(try_alignment > 0); int flags = MEM_RESERVE; if (commit) { flags |= MEM_COMMIT; } - return win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); + *addr = win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); + return (*addr != NULL ? 0 : (int)GetLastError()); } @@ -296,7 +293,7 @@ int _mi_prim_protect(void* addr, size_t size, bool protect) { // Huge page allocation //--------------------------------------------- -void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) +static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int numa_node) { const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE; @@ -315,7 +312,7 @@ void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) params[1].Arg.ULong = (unsigned)numa_node; } SIZE_T psize = size; - void* base = addr; + void* base = hint_addr; NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count); if (err == 0 && base != NULL) { return base; @@ -330,11 +327,16 @@ void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) if (pVirtualAlloc2 != NULL && numa_node >= 0) { params[0].Type.Type = MiMemExtendedParameterNumaNode; params[0].Arg.ULong = (unsigned)numa_node; - return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1); + return (*pVirtualAlloc2)(GetCurrentProcess(), hint_addr, size, flags, PAGE_READWRITE, params, 1); } // otherwise use regular virtual alloc on older windows - return VirtualAlloc(addr, size, flags, PAGE_READWRITE); + return VirtualAlloc(hint_addr, size, flags, PAGE_READWRITE); +} + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, void** addr) { + *addr = _mi_prim_alloc_huge_os_pagesx(hint_addr,size,numa_node); + return (*addr != NULL ? 0 : (int)GetLastError()); } From f58357548c9a2b377112174698e400c028cf1910 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 10:37:39 -0700 Subject: [PATCH 053/102] restructure header files --- ide/vs2022/mimalloc-override.vcxproj | 9 +++++---- ide/vs2022/mimalloc.vcxproj | 9 +++++---- include/{mimalloc-atomic.h => mimalloc/atomic.h} | 0 include/{mimalloc-internal.h => mimalloc/internal.h} | 4 ++-- {src/prim => include/mimalloc}/prim.h | 0 include/{mimalloc-track.h => mimalloc/track.h} | 0 include/{mimalloc-types.h => mimalloc/types.h} | 2 +- src/alloc-aligned.c | 4 ++-- src/alloc-override-osx.c | 2 +- src/alloc-posix.c | 2 +- src/alloc.c | 8 ++++---- src/arena.c | 4 ++-- src/bitmap.c | 2 +- src/heap.c | 7 +++---- src/init.c | 4 ++-- src/options.c | 6 +++--- src/os.c | 6 +++--- src/page.c | 4 ++-- src/prim/unix/prim.c | 6 +++--- src/prim/wasi/prim.c | 6 +++--- src/prim/windows/prim.c | 7 ++++--- src/random.c | 4 ++-- src/region.c | 4 ++-- src/segment.c | 4 ++-- src/static.c | 2 +- src/stats.c | 6 +++--- test/test-api-fill.c | 2 +- test/test-api.c | 4 ++-- 28 files changed, 60 insertions(+), 58 deletions(-) rename include/{mimalloc-atomic.h => mimalloc/atomic.h} (100%) rename include/{mimalloc-internal.h => mimalloc/internal.h} (99%) rename {src/prim => include/mimalloc}/prim.h (100%) rename include/{mimalloc-track.h => mimalloc/track.h} (100%) rename include/{mimalloc-types.h => mimalloc/types.h} (99%) diff --git a/ide/vs2022/mimalloc-override.vcxproj b/ide/vs2022/mimalloc-override.vcxproj index 1eb72952..6eac2fe0 100644 --- a/ide/vs2022/mimalloc-override.vcxproj +++ b/ide/vs2022/mimalloc-override.vcxproj @@ -209,15 +209,16 @@ - - - - + + + + + diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index 9e0ffc85..bf8f7d50 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -241,14 +241,15 @@ - - - - + + + + + diff --git a/include/mimalloc-atomic.h b/include/mimalloc/atomic.h similarity index 100% rename from include/mimalloc-atomic.h rename to include/mimalloc/atomic.h diff --git a/include/mimalloc-internal.h b/include/mimalloc/internal.h similarity index 99% rename from include/mimalloc-internal.h rename to include/mimalloc/internal.h index 33aafa70..51d8aa1e 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc/internal.h @@ -8,8 +8,8 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_INTERNAL_H #define MIMALLOC_INTERNAL_H -#include "mimalloc-types.h" -#include "mimalloc-track.h" +#include "mimalloc/types.h" +#include "mimalloc/track.h" #if (MI_DEBUG>0) #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) diff --git a/src/prim/prim.h b/include/mimalloc/prim.h similarity index 100% rename from src/prim/prim.h rename to include/mimalloc/prim.h diff --git a/include/mimalloc-track.h b/include/mimalloc/track.h similarity index 100% rename from include/mimalloc-track.h rename to include/mimalloc/track.h diff --git a/include/mimalloc-types.h b/include/mimalloc/types.h similarity index 99% rename from include/mimalloc-types.h rename to include/mimalloc/types.h index 3577b23a..da4b423f 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc/types.h @@ -10,7 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // ptrdiff_t #include // uintptr_t, uint16_t, etc -#include "mimalloc-atomic.h" // _Atomic +#include "mimalloc/atomic.h" // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index aa9b7bd0..674c74fe 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -6,8 +6,8 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "prim/prim.h" // mi_prim_get_default_heap +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" // mi_prim_get_default_heap #include // memset diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index a2819a8b..a517ddea 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -6,7 +6,7 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" +#include "mimalloc/internal.h" #if defined(MI_MALLOC_OVERRIDE) diff --git a/src/alloc-posix.c b/src/alloc-posix.c index f0cfe629..b6f09d1a 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -10,7 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file // for convenience and used when overriding these functions. // ------------------------------------------------------------------------ #include "mimalloc.h" -#include "mimalloc-internal.h" +#include "mimalloc/internal.h" // ------------------------------------------------------ // Posix & Unix functions definitions diff --git a/src/alloc.c b/src/alloc.c index 0bda4db8..301166eb 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -9,9 +9,9 @@ terms of the MIT license. A copy of the license can be found in the file #endif #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" -#include "prim/prim.h" // _mi_prim_thread_id() +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" // _mi_prim_thread_id() #include // memset, strlen (for mi_strdup) #include // malloc, abort @@ -40,7 +40,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz // allow use of the block internally // note: when tracking we need to avoid ever touching the MI_PADDING since - // that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc-track.h`) + // that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc/track.h`) mi_track_mem_undefined(block, mi_page_usable_block_size(page)); // zero the block? note: we need to zero the full block size (issue #63) diff --git a/src/arena.c b/src/arena.c index 57f48a73..d8178cc1 100644 --- a/src/arena.c +++ b/src/arena.c @@ -21,8 +21,8 @@ which is sometimes needed for embedded devices or shared memory for example. The arena allocation needs to be thread safe and we use an atomic bitmap to allocate. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" #include // memset #include // ENOMEM diff --git a/src/bitmap.c b/src/bitmap.c index 9ba994d7..8483de0b 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -18,7 +18,7 @@ between the fields. (This is used in arena allocation) ---------------------------------------------------------------------------- */ #include "mimalloc.h" -#include "mimalloc-internal.h" +#include "mimalloc/internal.h" #include "bitmap.h" /* ----------------------------------------------------------- diff --git a/src/heap.c b/src/heap.c index b12a9962..fea033dc 100644 --- a/src/heap.c +++ b/src/heap.c @@ -6,10 +6,9 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" -#include "mimalloc-track.h" -#include "prim/prim.h" // mi_prim_get_default_heap +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" // mi_prim_get_default_heap #include // memset, memcpy diff --git a/src/init.c b/src/init.c index 495d26fd..8c5c8049 100644 --- a/src/init.c +++ b/src/init.c @@ -5,8 +5,8 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "prim/prim.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" #include // memcpy, memset #include // atexit diff --git a/src/options.c b/src/options.c index f0c52c84..816a2919 100644 --- a/src/options.c +++ b/src/options.c @@ -5,9 +5,9 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" -#include "prim/prim.h" // mi_prim_out_stderr +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" // mi_prim_out_stderr #include // FILE #include // abort diff --git a/src/os.c b/src/os.c index 85c3652f..d5a3398f 100644 --- a/src/os.c +++ b/src/os.c @@ -5,9 +5,9 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" -#include "prim/prim.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" /* ----------------------------------------------------------- diff --git a/src/page.c b/src/page.c index 1347c845..e43ba402 100644 --- a/src/page.c +++ b/src/page.c @@ -12,8 +12,8 @@ terms of the MIT license. A copy of the license can be found in the file ----------------------------------------------------------- */ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" /* ----------------------------------------------------------- Definition of page queues for each block size diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 5a3ca5ab..0ac69f1a 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -21,9 +21,9 @@ terms of the MIT license. A copy of the license can be found in the file #endif #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" -#include "../prim.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" #include // mmap #include // sysconf diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index f995304f..cb3ce1a7 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -8,9 +8,9 @@ terms of the MIT license. A copy of the license can be found in the file // This file is included in `src/prim/prim.c` #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" -#include "../prim.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" //--------------------------------------------- // Initialize diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 1e15273a..bea1d437 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -8,9 +8,9 @@ terms of the MIT license. A copy of the license can be found in the file // This file is included in `src/prim/prim.c` #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" -#include "../prim.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" #include // strerror #include // fputs, stderr @@ -157,6 +157,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) //--------------------------------------------- int _mi_prim_free(void* addr, size_t size ) { + MI_UNUSED(size); DWORD errcode = 0; bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); if (err) { errcode = GetLastError(); } diff --git a/src/random.c b/src/random.c index 3c8372c8..4fc8b2f8 100644 --- a/src/random.c +++ b/src/random.c @@ -5,8 +5,8 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "prim/prim.h" // _mi_prim_random_buf +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" // _mi_prim_random_buf #include // memset /* ---------------------------------------------------------------------------- diff --git a/src/region.c b/src/region.c index 3571abb6..29681f4c 100644 --- a/src/region.c +++ b/src/region.c @@ -32,8 +32,8 @@ Possible issues: do this better without adding too much complexity? -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" #include // memset diff --git a/src/segment.c b/src/segment.c index c3cb7155..6cdf4fe7 100644 --- a/src/segment.c +++ b/src/segment.c @@ -5,8 +5,8 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" #include // memset #include diff --git a/src/static.c b/src/static.c index 0de72fe3..d0652526 100644 --- a/src/static.c +++ b/src/static.c @@ -14,7 +14,7 @@ terms of the MIT license. A copy of the license can be found in the file #endif #include "mimalloc.h" -#include "mimalloc-internal.h" +#include "mimalloc/internal.h" // For a static override we create a single object file // containing the whole library. If it is linked first diff --git a/src/stats.c b/src/stats.c index 8273740f..435a2b38 100644 --- a/src/stats.c +++ b/src/stats.c @@ -5,9 +5,9 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" -#include "prim/prim.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" #include // snprintf #include // memset diff --git a/test/test-api-fill.c b/test/test-api-fill.c index 2ad06808..7ba79880 100644 --- a/test/test-api-fill.c +++ b/test/test-api-fill.c @@ -5,7 +5,7 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-types.h" +#include "mimalloc/types.h" #include "testhelper.h" diff --git a/test/test-api.c b/test/test-api.c index 20050ce8..c78e1972 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -33,8 +33,8 @@ we therefore test the API over various inputs. Please add more tests :-) #endif #include "mimalloc.h" -// #include "mimalloc-internal.h" -#include "mimalloc-types.h" // for MI_DEBUG and MI_ALIGNMENT_MAX +// #include "mimalloc/internal.h" +#include "mimalloc/types.h" // for MI_DEBUG and MI_ALIGNMENT_MAX #include "testhelper.h" From c0c762611c309e86e31a04778059101fa6536bb0 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 10:49:56 -0700 Subject: [PATCH 054/102] add prim/osx directory --- CMakeLists.txt | 20 +++++++++---------- .../osx/alloc-override-zone.c} | 0 src/prim/osx/prim.c | 9 +++++++++ src/prim/prim.c | 6 ++++++ src/static.c | 14 ++++++------- 5 files changed, 32 insertions(+), 17 deletions(-) rename src/{alloc-override-osx.c => prim/osx/alloc-override-zone.c} (100%) create mode 100644 src/prim/osx/prim.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 68b7cab4..5c8e55f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,20 +38,20 @@ include(GNUInstallDirs) include("cmake/mimalloc-config-version.cmake") set(mi_sources - src/stats.c - src/random.c - src/os.c - src/bitmap.c - src/arena.c - src/region.c - src/segment.c - src/page.c src/alloc.c src/alloc-aligned.c src/alloc-posix.c + src/arena.c + src/bitmap.c src/heap.c - src/options.c src/init.c + src/options.c + src/os.c + src/page.c + src/random.c + src/region.c + src/segment.c + src/stats.c src/prim/prim.c) set(mi_cflags "") @@ -92,7 +92,7 @@ if(MI_OVERRIDE) if(MI_OSX_ZONE) # use zone's on macOS message(STATUS " Use malloc zone to override malloc (MI_OSX_ZONE=ON)") - list(APPEND mi_sources src/alloc-override-osx.c) + list(APPEND mi_sources src/prim/osx/alloc-override-zone.c) list(APPEND mi_defines MI_OSX_ZONE=1) if (NOT MI_OSX_INTERPOSE) message(STATUS " WARNING: zone overriding usually also needs interpose (use -DMI_OSX_INTERPOSE=ON)") diff --git a/src/alloc-override-osx.c b/src/prim/osx/alloc-override-zone.c similarity index 100% rename from src/alloc-override-osx.c rename to src/prim/osx/alloc-override-zone.c diff --git a/src/prim/osx/prim.c b/src/prim/osx/prim.c new file mode 100644 index 00000000..8a2f4e8a --- /dev/null +++ b/src/prim/osx/prim.c @@ -0,0 +1,9 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// We use the unix/prim.c with the mmap API on macOSX +#include "../unix/prim.c" diff --git a/src/prim/prim.c b/src/prim/prim.c index 109ab8e8..9a597d8e 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -10,9 +10,15 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(_WIN32) #include "windows/prim.c" // VirtualAlloc (Windows) + +#elif defined(__APPLE__) +#include "osx/prim.c" // macOSX (actually defers to mmap in unix/prim.c) + #elif defined(__wasi__) #define MI_USE_SBRK #include "wasi/prim.c" // memory-grow or sbrk (Wasm) + #else #include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) + #endif diff --git a/src/static.c b/src/static.c index d0652526..090f0c25 100644 --- a/src/static.c +++ b/src/static.c @@ -20,11 +20,8 @@ terms of the MIT license. A copy of the license can be found in the file // containing the whole library. If it is linked first // it will override all the standard library allocation // functions (on Unix's). -#include "alloc.c" +#include "alloc.c" // includes alloc-override.c #include "alloc-aligned.c" -#if MI_OSX_ZONE -#include "alloc-override-osx.c" -#endif #include "alloc-posix.c" #include "arena.c" #include "bitmap.c" @@ -32,9 +29,12 @@ terms of the MIT license. A copy of the license can be found in the file #include "init.c" #include "options.c" #include "os.c" -#include "page.c" -#include "prim/prim.c" -#include "random.c" +#include "page.c" // includes page-queue.c +#include "random.c" #include "region.c" #include "segment.c" #include "stats.c" +#include "prim/prim.c" +#if MI_OSX_ZONE +#include "prim/osx/alloc-override-zone.c" +#endif From ec5f4904b0762639fe938e1d906527f763e4df77 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 11:01:40 -0700 Subject: [PATCH 055/102] more comments --- include/mimalloc/internal.h | 6 ++++++ include/mimalloc/prim.h | 7 +++++++ include/mimalloc/types.h | 11 +++++++++++ 3 files changed, 24 insertions(+) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 51d8aa1e..791eeb99 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -8,6 +8,12 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_INTERNAL_H #define MIMALLOC_INTERNAL_H + +// -------------------------------------------------------------------------- +// This file contains the interal API's of mimalloc and various utility +// functions and macros. +// -------------------------------------------------------------------------- + #include "mimalloc/types.h" #include "mimalloc/track.h" diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 1a4fb5d8..97d8b45d 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -8,10 +8,17 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_PRIM_H #define MIMALLOC_PRIM_H + +// -------------------------------------------------------------------------- +// This file specifies the primitive portability API. +// Each OS/host needs to implement these primitives, see `src/prim` +// for implementations on Window, macOS, WASI, and Linux/Unix. +// // note: on all primitive functions, we always get: // addr != NULL and page aligned // size > 0 and page aligned // return value is an error code an int where 0 is success. +// -------------------------------------------------------------------------- // OS memory configuration typedef struct mi_os_mem_config_s { diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index da4b423f..90b7b3e4 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -8,6 +8,17 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_TYPES_H #define MIMALLOC_TYPES_H +// -------------------------------------------------------------------------- +// This file contains the main type definitions for mimalloc: +// mi_heap_t : all data for a thread-local heap, contains +// lists of all managed heap pages. +// mi_segment_t : a larger chunk of memory (32GiB) from where pages +// are allocated. +// mi_page_t : a mimalloc page (usually 64KiB or 512KiB) from +// where objects are allocated. +// -------------------------------------------------------------------------- + + #include // ptrdiff_t #include // uintptr_t, uint16_t, etc #include "mimalloc/atomic.h" // _Atomic From 0509d11ac55f28ddd4784fcd1a3f5e42238bbd41 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 11:02:16 -0700 Subject: [PATCH 056/102] more comments --- include/mimalloc/internal.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 791eeb99..3ee20151 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -50,6 +50,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_externc #endif +// pthreads #if !defined(_WIN32) && !defined(__wasi__) #define MI_USE_PTHREADS #include From e24e1125eea4102542f0afcabbe58b01acfd5e7a Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 11:03:50 -0700 Subject: [PATCH 057/102] bump version to 1.8.0 --- cmake/mimalloc-config-version.cmake | 4 ++-- include/mimalloc.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index dbe8fdaa..1141128c 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 1) -set(mi_version_minor 7) -set(mi_version_patch 9) +set(mi_version_minor 8) +set(mi_version_patch 0) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/include/mimalloc.h b/include/mimalloc.h index 4fc7a752..66545a8c 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 179 // major + 2 digits minor +#define MI_MALLOC_VERSION 180 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes From 993c0a49b4196c807da226b6796a262a062ff1eb Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 11:06:28 -0700 Subject: [PATCH 058/102] fix includes --- src/segment-cache.c | 6 +++--- src/static.c | 7 ------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/segment-cache.c b/src/segment-cache.c index d93fd644..4a16a18a 100644 --- a/src/segment-cache.c +++ b/src/segment-cache.c @@ -11,10 +11,10 @@ terms of the MIT license. A copy of the license can be found in the file The full memory map of all segments is also implemented here. -----------------------------------------------------------------------------*/ #include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" -#include "bitmap.h" // atomic bitmap +#include "./bitmap.h" // atomic bitmap //#define MI_CACHE_DISABLE 1 // define to completely disable the segment cache diff --git a/src/static.c b/src/static.c index a71cddca..d992f4da 100644 --- a/src/static.c +++ b/src/static.c @@ -29,15 +29,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "init.c" #include "options.c" #include "os.c" -<<<<<<< HEAD -#include "page.c" -#include "prim/prim.c" -#include "random.c" -======= #include "page.c" // includes page-queue.c #include "random.c" -#include "region.c" ->>>>>>> dev-platform #include "segment.c" #include "segment-cache.c" #include "stats.c" From 2f9b2f51b9869d3f11a7fbbfdb9fb69ead33c0f6 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 11:11:01 -0700 Subject: [PATCH 059/102] update 2022 ide --- ide/vs2022/mimalloc.vcxproj | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index bf8f7d50..e6474d0f 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -205,12 +205,6 @@ false false - - true - true - true - true - true true From 287010578d9b004c54c879cf12a735ccb1b4fb64 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 11:20:30 -0700 Subject: [PATCH 060/102] update ide project files --- ide/vs2017/mimalloc-override.vcxproj | 8 ++++-- ide/vs2017/mimalloc-override.vcxproj.filters | 24 ++++++++++------ ide/vs2017/mimalloc.vcxproj | 14 ++++----- ide/vs2017/mimalloc.vcxproj.filters | 27 ++++++++++-------- ide/vs2019/mimalloc-override.vcxproj | 8 ++++-- ide/vs2019/mimalloc-override.vcxproj.filters | 28 +++++++++++------- ide/vs2019/mimalloc.vcxproj | 20 +++++++------ ide/vs2019/mimalloc.vcxproj.filters | 30 ++++++++++++-------- ide/vs2022/mimalloc-override.vcxproj | 6 ++++ ide/vs2022/mimalloc.vcxproj | 6 ++++ src/prim/windows/prim.c | 4 --- 11 files changed, 104 insertions(+), 71 deletions(-) diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index f308225b..0d11068b 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -209,12 +209,14 @@ - - - + + + + + diff --git a/ide/vs2017/mimalloc-override.vcxproj.filters b/ide/vs2017/mimalloc-override.vcxproj.filters index a67dddad..009962dd 100644 --- a/ide/vs2017/mimalloc-override.vcxproj.filters +++ b/ide/vs2017/mimalloc-override.vcxproj.filters @@ -14,15 +14,6 @@ Header Files - - Header Files - - - Header Files - - - Header Files - Header Files @@ -32,6 +23,21 @@ Header Files + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index e7be785c..05024448 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -215,12 +215,6 @@ false false - - true - true - true - true - true true @@ -249,12 +243,14 @@ - - - + + + + + diff --git a/ide/vs2017/mimalloc.vcxproj.filters b/ide/vs2017/mimalloc.vcxproj.filters index 27cd4b5e..249757b6 100644 --- a/ide/vs2017/mimalloc.vcxproj.filters +++ b/ide/vs2017/mimalloc.vcxproj.filters @@ -35,9 +35,6 @@ Source Files - - Source Files - Source Files @@ -70,23 +67,29 @@ Header Files - - Header Files - - - Header Files - Header Files - - Header Files - Header Files Header Files + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + \ No newline at end of file diff --git a/ide/vs2019/mimalloc-override.vcxproj b/ide/vs2019/mimalloc-override.vcxproj index 5fb809c8..d80133e7 100644 --- a/ide/vs2019/mimalloc-override.vcxproj +++ b/ide/vs2019/mimalloc-override.vcxproj @@ -209,12 +209,14 @@ - - - + + + + + diff --git a/ide/vs2019/mimalloc-override.vcxproj.filters b/ide/vs2019/mimalloc-override.vcxproj.filters index 737e0600..357a9a2f 100644 --- a/ide/vs2019/mimalloc-override.vcxproj.filters +++ b/ide/vs2019/mimalloc-override.vcxproj.filters @@ -49,30 +49,38 @@ Source Files - + + Source Files + Header Files - - Header Files - - - Header Files - Header Files Header Files - - Header Files - Source Files + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj index b18674ad..79146c99 100644 --- a/ide/vs2019/mimalloc.vcxproj +++ b/ide/vs2019/mimalloc.vcxproj @@ -205,12 +205,6 @@ false false - - true - true - true - true - true true @@ -226,6 +220,12 @@ + + true + true + true + true + @@ -241,12 +241,14 @@ - - - + + + + + diff --git a/ide/vs2019/mimalloc.vcxproj.filters b/ide/vs2019/mimalloc.vcxproj.filters index 5ab88914..9b215312 100644 --- a/ide/vs2019/mimalloc.vcxproj.filters +++ b/ide/vs2019/mimalloc.vcxproj.filters @@ -10,9 +10,6 @@ Source Files - - Source Files - Source Files @@ -55,29 +52,38 @@ Source Files + + Source Files + Header Files - - Header Files - - - Header Files - Header Files Header Files - - Header Files - Source Files + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + diff --git a/ide/vs2022/mimalloc-override.vcxproj b/ide/vs2022/mimalloc-override.vcxproj index 6eac2fe0..50a3d6b9 100644 --- a/ide/vs2022/mimalloc-override.vcxproj +++ b/ide/vs2022/mimalloc-override.vcxproj @@ -241,6 +241,12 @@ + + true + true + true + true + diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index e6474d0f..9a7bf18c 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -220,6 +220,12 @@ + + true + true + true + true + diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index bea1d437..e3dc33e3 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -11,12 +11,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/internal.h" #include "mimalloc/atomic.h" #include "mimalloc/prim.h" -#include // strerror #include // fputs, stderr -#ifdef _MSC_VER -#pragma warning(disable:4996) // strerror -#endif //--------------------------------------------- // Dynamically bind Windows API points for portability From 65402836aeb4363290cd7c66f6818ef9b5efd5ae Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 11:30:52 -0700 Subject: [PATCH 061/102] comments --- include/mimalloc/atomic.h | 2 +- include/mimalloc/internal.h | 2 +- include/mimalloc/types.h | 2 +- src/prim/readme.md | 5 +++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/mimalloc/atomic.h b/include/mimalloc/atomic.h index c66f8049..971b374a 100644 --- a/include/mimalloc/atomic.h +++ b/include/mimalloc/atomic.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2021 Microsoft Research, Daan Leijen +Copyright (c) 2018-2023 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 3ee20151..32c71ff9 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2022, Microsoft Research, Daan Leijen +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 90b7b3e4..28343d21 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/prim/readme.md b/src/prim/readme.md index eb02f274..380dd3a7 100644 --- a/src/prim/readme.md +++ b/src/prim/readme.md @@ -2,7 +2,8 @@ This is the portability layer where all primitives needed from the OS are defined. -- `prim.h`: API definition -- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform. +- `include/mimalloc/prim.h`: primitive portability API definition. +- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform + (and on macOS, `osx/prim.c` defers to `unix/prim.c`). Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's. \ No newline at end of file From 54ad5e76fd07a7381ca7dd92a646cc98ee1dea8a Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 11:43:31 -0700 Subject: [PATCH 062/102] fix warnings for issues #709 --- src/heap.c | 13 ++++++++++++- src/page.c | 4 ++++ src/stats.c | 18 ++++++++++-------- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/heap.c b/src/heap.c index fea033dc..0c372c5b 100644 --- a/src/heap.c +++ b/src/heap.c @@ -31,15 +31,18 @@ static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void // visit all pages #if MI_DEBUG>1 size_t total = heap->page_count; - #endif size_t count = 0; + #endif + for (size_t i = 0; i <= MI_BIN_FULL; i++) { mi_page_queue_t* pq = &heap->pages[i]; mi_page_t* page = pq->first; while(page != NULL) { mi_page_t* next = page->next; // save next in case the page gets removed from the queue mi_assert_internal(mi_page_heap(page) == heap); + #if MI_DEBUG>1 count++; + #endif if (!fn(heap, pq, page, arg1, arg2)) return false; page = next; // and continue } @@ -516,9 +519,13 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v uintptr_t free_map[MI_MAX_BLOCKS / sizeof(uintptr_t)]; memset(free_map, 0, sizeof(free_map)); + #if MI_DEBUG>1 size_t free_count = 0; + #endif for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { + #if MI_DEBUG>1 free_count++; + #endif mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize)); size_t offset = (uint8_t*)block - pstart; mi_assert_internal(offset % bsize == 0); @@ -531,7 +538,9 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v mi_assert_internal(page->capacity == (free_count + page->used)); // walk through all blocks skipping the free ones + #if MI_DEBUG>1 size_t used_count = 0; + #endif for (size_t i = 0; i < page->capacity; i++) { size_t bitidx = (i / sizeof(uintptr_t)); size_t bit = i - (bitidx * sizeof(uintptr_t)); @@ -540,7 +549,9 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v i += (sizeof(uintptr_t) - 1); // skip a run of free blocks } else if ((m & ((uintptr_t)1 << bit)) == 0) { + #if MI_DEBUG>1 used_count++; + #endif uint8_t* block = pstart + (i * bsize); if (!visitor(mi_page_heap(page), area, block, ubsize, arg)) return false; } diff --git a/src/page.c b/src/page.c index e43ba402..531293f3 100644 --- a/src/page.c +++ b/src/page.c @@ -699,12 +699,16 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* pq, bool first_try) { // search through the pages in "next fit" order + #if MI_STAT size_t count = 0; + #endif mi_page_t* page = pq->first; while (page != NULL) { mi_page_t* next = page->next; // remember next + #if MI_STAT count++; + #endif // 0. collect freed blocks by us and other threads _mi_page_free_collect(page, false); diff --git a/src/stats.c b/src/stats.c index 435a2b38..0ab2acd2 100644 --- a/src/stats.c +++ b/src/stats.c @@ -430,17 +430,19 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) { mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept { - mi_process_info_t pinfo = { 0 }; - pinfo.elapsed = _mi_clock_end(mi_process_start); - pinfo.utime = 0; - pinfo.stime = 0; + mi_process_info_t pinfo; + _mi_memzero(&pinfo,sizeof(pinfo)); + pinfo.elapsed = _mi_clock_end(mi_process_start); pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); - pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); - pinfo.current_rss = pinfo.current_commit; - pinfo.peak_rss = pinfo.peak_commit; - pinfo.page_faults = 0; + pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); + pinfo.current_rss = pinfo.current_commit; + pinfo.peak_rss = pinfo.peak_commit; + pinfo.utime = 0; + pinfo.stime = 0; + pinfo.page_faults = 0; _mi_prim_process_info(&pinfo); + if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX)); if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX)); if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX)); From 90f866c5bcc77496bf19df7bc85ab8a42b6b2490 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 11:45:34 -0700 Subject: [PATCH 063/102] fix warnings for issues #709 --- src/segment.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/segment.c b/src/segment.c index 648116c3..1b73b19a 100644 --- a/src/segment.c +++ b/src/segment.c @@ -959,7 +959,9 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t // Remove the free pages mi_slice_t* slice = &segment->slices[0]; const mi_slice_t* end = mi_segment_slices_end(segment); + #if MI_DEBUG>1 size_t page_count = 0; + #endif while (slice < end) { mi_assert_internal(slice->slice_count > 0); mi_assert_internal(slice->slice_offset == 0); @@ -967,7 +969,9 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t if (slice->xblock_size == 0 && segment->kind != MI_SEGMENT_HUGE) { mi_segment_span_remove_from_queue(slice, tld); } + #if MI_DEBUG>1 page_count++; + #endif slice = slice + slice->slice_count; } mi_assert_internal(page_count == 2); // first page is allocated by the segment itself From 30df80b05ad30d6d66b85c6d100ab9312bfef3ec Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 11:50:05 -0700 Subject: [PATCH 064/102] proper prototype --- src/prim/osx/alloc-override-zone.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prim/osx/alloc-override-zone.c b/src/prim/osx/alloc-override-zone.c index a517ddea..80bcfa93 100644 --- a/src/prim/osx/alloc-override-zone.c +++ b/src/prim/osx/alloc-override-zone.c @@ -420,7 +420,7 @@ __attribute__((constructor(0))) #else __attribute__((constructor)) // seems not supported by g++-11 on the M1 #endif -static void _mi_macos_override_malloc() { +static void _mi_macos_override_malloc(void) { malloc_zone_t* purgeable_zone = NULL; #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) From 4bf63300b3935245559833887277e4aa06814940 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 12:29:36 -0700 Subject: [PATCH 065/102] fix alignment issue #700 --- ide/vs2022/mimalloc.vcxproj | 2 +- src/segment.c | 4 +++- test/main-override-static.c | 9 +++++++++ test/test-api.c | 18 ++++++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index 07a854ab..894c5030 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -116,7 +116,7 @@ true Default ../../include - MI_DEBUG=4;MI_SECURE=0;%(PreprocessorDefinitions); + MI_DEBUG=0;MI_SECURE=0;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 diff --git a/src/segment.c b/src/segment.c index 1b73b19a..1e23bb1a 100644 --- a/src/segment.c +++ b/src/segment.c @@ -316,7 +316,9 @@ static uint8_t* _mi_segment_page_start_from_slice(const mi_segment_t* segment, c ptrdiff_t idx = slice - segment->slices; size_t psize = (size_t)slice->slice_count * MI_SEGMENT_SLICE_SIZE; // make the start not OS page aligned for smaller blocks to avoid page/cache effects - size_t start_offset = (xblock_size >= MI_INTPTR_SIZE && xblock_size <= 1024 ? 3*MI_MAX_ALIGN_GUARANTEE : 0); + // note: the offset must always be an xblock_size multiple since we assume small allocations + // are aligned (see `mi_heap_malloc_aligned`). + size_t start_offset = (xblock_size >= MI_INTPTR_SIZE && xblock_size <= 512 ? xblock_size : 0); if (page_size != NULL) { *page_size = psize - start_offset; } return (uint8_t*)segment + ((idx*MI_SEGMENT_SLICE_SIZE) + start_offset); } diff --git a/test/main-override-static.c b/test/main-override-static.c index 534c8849..5e8b7333 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -20,6 +20,7 @@ static void negative_stat(void); static void alloc_huge(void); static void test_heap_walk(void); static void test_heap_arena(void); +static void test_align(void); int main() { mi_version(); @@ -37,6 +38,7 @@ int main() { // alloc_huge(); // test_heap_walk(); // test_heap_arena(); + test_align(); void* p1 = malloc(78); void* p2 = malloc(24); @@ -68,6 +70,13 @@ int main() { return 0; } +static void test_align() { + void* p = mi_malloc_aligned(256, 256); + if (((uintptr_t)p % 256) != 0) { + fprintf(stderr, "%p is not 256 alignend!\n", p); + } +} + static void invalid_free() { free((void*)0xBADBEEF); realloc((void*)0xBADBEEF,10); diff --git a/test/test-api.c b/test/test-api.c index c78e1972..1967dad7 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -212,6 +212,24 @@ int main(void) { result = mi_heap_contains_block(heap, p); mi_heap_destroy(heap); } + CHECK_BODY("malloc-aligned12") { + bool ok = true; + const size_t align = 256; + for (int j = 1; j < 1000; j++) { + void* ps[1000]; + for (int i = 0; i < 1000 && ok; i++) { + ps[i] = mi_malloc_aligned(j // size + , align); + if (ps[i] == NULL || ((uintptr_t)(ps[i]) % align) != 0) { + ok = false; + } + } + for (int i = 0; i < 1000 && ok; i++) { + mi_free(ps[i]); + } + } + result = ok; + }; CHECK_BODY("malloc-aligned-at1") { void* p = mi_malloc_aligned_at(48,32,0); result = (p != NULL && ((uintptr_t)(p) + 0) % 32 == 0); mi_free(p); }; From c935521bf92acfa539ea5f896e0f6d12611c9d5d Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 12:32:41 -0700 Subject: [PATCH 066/102] fix test and project --- ide/vs2022/mimalloc.vcxproj | 2 +- test/main-override-static.c | 2 +- test/test-api.c | 18 ------------------ 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index 894c5030..07a854ab 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -116,7 +116,7 @@ true Default ../../include - MI_DEBUG=0;MI_SECURE=0;%(PreprocessorDefinitions); + MI_DEBUG=4;MI_SECURE=0;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 diff --git a/test/main-override-static.c b/test/main-override-static.c index 5e8b7333..e71be29e 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -38,7 +38,7 @@ int main() { // alloc_huge(); // test_heap_walk(); // test_heap_arena(); - test_align(); + // test_align(); void* p1 = malloc(78); void* p2 = malloc(24); diff --git a/test/test-api.c b/test/test-api.c index 1967dad7..c78e1972 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -212,24 +212,6 @@ int main(void) { result = mi_heap_contains_block(heap, p); mi_heap_destroy(heap); } - CHECK_BODY("malloc-aligned12") { - bool ok = true; - const size_t align = 256; - for (int j = 1; j < 1000; j++) { - void* ps[1000]; - for (int i = 0; i < 1000 && ok; i++) { - ps[i] = mi_malloc_aligned(j // size - , align); - if (ps[i] == NULL || ((uintptr_t)(ps[i]) % align) != 0) { - ok = false; - } - } - for (int i = 0; i < 1000 && ok; i++) { - mi_free(ps[i]); - } - } - result = ok; - }; CHECK_BODY("malloc-aligned-at1") { void* p = mi_malloc_aligned_at(48,32,0); result = (p != NULL && ((uintptr_t)(p) + 0) % 32 == 0); mi_free(p); }; From a582d760ed8266af9fab445bf3e06e65d073a6f3 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 12:39:15 -0700 Subject: [PATCH 067/102] refine start offset in a page --- src/segment.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/segment.c b/src/segment.c index 1e23bb1a..451ef250 100644 --- a/src/segment.c +++ b/src/segment.c @@ -318,7 +318,11 @@ static uint8_t* _mi_segment_page_start_from_slice(const mi_segment_t* segment, c // make the start not OS page aligned for smaller blocks to avoid page/cache effects // note: the offset must always be an xblock_size multiple since we assume small allocations // are aligned (see `mi_heap_malloc_aligned`). - size_t start_offset = (xblock_size >= MI_INTPTR_SIZE && xblock_size <= 512 ? xblock_size : 0); + size_t start_offset = 0; + if (xblock_size >= MI_INTPTR_SIZE) { + if (xblock_size <= 64) { start_offset = 3*xblock_size; } + else if (xblock_size <= 512) { start_offset = xblock_size; } + } if (page_size != NULL) { *page_size = psize - start_offset; } return (uint8_t*)segment + ((idx*MI_SEGMENT_SLICE_SIZE) + start_offset); } From 01b460fedb5715051397ecacbfe9d330e32528f7 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 20 Mar 2023 13:24:11 -0700 Subject: [PATCH 068/102] add std::string test for macos --- test/main-override.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index db96efb1..902cfdd4 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -36,14 +36,16 @@ static void fail_aslr(); // issue #372 static void tsan_numa_test(); // issue #414 static void strdup_test(); // issue #445 static void heap_thread_free_huge(); +static void test_std_string(); // issue #697 static void test_stl_allocators(); int main() { - mi_stats_reset(); // ignore earlier allocations - - heap_thread_free_huge(); + // mi_stats_reset(); // ignore earlier allocations + + test_std_string(); + // heap_thread_free_huge(); /* heap_thread_free_large(); heap_no_delete(); @@ -56,7 +58,7 @@ int main() { test_mt_shutdown(); */ //fail_aslr(); - mi_stats_print(NULL); + // mi_stats_print(NULL); return 0; } @@ -196,6 +198,13 @@ static void heap_no_delete() { } +// Issue #697 +static void test_std_string() { + std::string path = "/Users/xxxx/Library/Developer/Xcode/DerivedData/xxxxxxxxxx/Build/Intermediates.noindex/xxxxxxxxxxx/arm64/XX_lto.o/0.arm64.lto.o"; + std::string path1 = "/Users/xxxx/Library/Developer/Xcode/DerivedData/xxxxxxxxxx/Build/Intermediates.noindex/xxxxxxxxxxx/arm64/XX_lto.o/1.arm64.lto.o"; + std::cout << path + "\n>>> " + path1 + "\n>>> " << std::endl; +} + // Issue #204 static volatile void* global_p; From 0b4c3da2e90c90dc3f379618195e2e9388c9201c Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 13:55:39 -0700 Subject: [PATCH 069/102] make process init race free (issue #701) --- include/mimalloc/atomic.h | 9 +++++++++ src/init.c | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/mimalloc/atomic.h b/include/mimalloc/atomic.h index 971b374a..fe79fbca 100644 --- a/include/mimalloc/atomic.h +++ b/include/mimalloc/atomic.h @@ -275,6 +275,15 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) { return (intptr_t)mi_atomic_addi(p, -sub); } +typedef _Atomic(uintptr_t) mi_atomic_once_t; + +// Returns true only on the first invocation +static inline bool mi_atomic_once( mi_atomic_once_t* once ) { + if (mi_atomic_load_relaxed(once) != 0) return false; // quick test + uintptr_t expected = 0; + return mi_atomic_cas_strong_acq_rel(once, &expected, 1); // try to set to 1 +} + // Yield #if defined(__cplusplus) #include diff --git a/src/init.c b/src/init.c index 8c5c8049..9a11f6e0 100644 --- a/src/init.c +++ b/src/init.c @@ -511,7 +511,8 @@ static void mi_detect_cpu_features(void) { // Initialize the process; called by thread_init or the process loader void mi_process_init(void) mi_attr_noexcept { // ensure we are called once - if (_mi_process_is_initialized) return; + static mi_atomic_once_t process_init; + if (!mi_atomic_once(&process_init)) return; _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); _mi_process_is_initialized = true; mi_process_setup_auto_thread_done(); From c92e9e7bf76add4b7de35de8ac243dc144b92e56 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 14:01:09 -0700 Subject: [PATCH 070/102] add comment that thread id's should not be zero, issue #698 --- include/mimalloc/prim.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 97d8b45d..68f0871e 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -104,12 +104,13 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); //------------------------------------------------------------------- -// Thread id +// Thread id: `_mi_prim_thread_id()` // // Getting the thread id should be performant as it is called in the // fast path of `_mi_free` and we specialize for various platforms as // inlined definitions. Regular code should call `init.c:_mi_thread_id()`. -// We only require _mi_prim_thread_id() to return a unique id for each thread. +// We only require _mi_prim_thread_id() to return a unique id +// for each thread (unequal to zero). //------------------------------------------------------------------- static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; From 06f0ba232e515d9d30ab5a89ef3e929641cde61b Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 20 Mar 2023 14:23:52 -0700 Subject: [PATCH 071/102] prevent reentrancy on thread_done (issue #699) --- src/init.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/init.c b/src/init.c index 9a11f6e0..cd95526c 100644 --- a/src/init.c +++ b/src/init.c @@ -380,15 +380,24 @@ void mi_thread_done(void) mi_attr_noexcept { _mi_thread_done(NULL); } +#include + void _mi_thread_done(mi_heap_t* heap) { - mi_atomic_decrement_relaxed(&thread_count); - _mi_stat_decrease(&_mi_stats_main.threads, 1); - + // calling with NULL implies using the default heap if (heap == NULL) { heap = mi_prim_get_default_heap(); if (heap == NULL) return; } + + // prevent re-entrancy through heap_done/heap_set_default_direct (issue #699) + if (!mi_heap_is_initialized(heap)) { + return; + } + + // adjust stats + mi_atomic_decrement_relaxed(&thread_count); + _mi_stat_decrease(&_mi_stats_main.threads, 1); // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... if (heap->thread_id != _mi_thread_id()) return; From 1ded6e2dec4722f5dada6e34dc16cc53d7ec9510 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 20 Mar 2023 14:30:38 -0700 Subject: [PATCH 072/102] increase env limit to 10000 entries (issue #685) --- src/prim/unix/prim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 0ac69f1a..0ca9bc64 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -629,8 +629,8 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { if (len == 0) return false; char** env = mi_get_environ(); if (env == NULL) return false; - // compare up to 256 entries - for (int i = 0; i < 256 && env[i] != NULL; i++) { + // compare up to 10000 entries + for (int i = 0; i < 10000 && env[i] != NULL; i++) { const char* s = env[i]; if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive // found it From 70fefec8379d85596b01a5deaedd3ef900eb0405 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 21 Mar 2023 19:42:25 -0700 Subject: [PATCH 073/102] fix huge OS page count when a timeout happens (issue #711) --- src/os.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/os.c b/src/os.c index d5a3398f..3e8a9172 100644 --- a/src/os.c +++ b/src/os.c @@ -526,14 +526,15 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse // We allocate one page at the time to be able to abort if it takes too long // or to at least allocate as many as available on the system. mi_msecs_t start_t = _mi_clock_start(); - size_t page; - for (page = 0; page < pages; page++) { + size_t page = 0; + while (page < pages) { // allocate a page void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE); void* p = NULL; int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &p); if (err != 0) { - _mi_warning_message("unable to allocate huge OS page (error: %d (0x%d), address: %p, size: %zx bytes)", err, err, addr, MI_HUGE_OS_PAGE_SIZE); + _mi_warning_message("unable to allocate huge OS page (error: %d (0x%d), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE); + break; } // Did we succeed at a contiguous address? @@ -547,6 +548,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse } // success, record it + page++; // increase before timeout check (see issue #711) _mi_stat_increase(&_mi_stats_main.committed, MI_HUGE_OS_PAGE_SIZE); _mi_stat_increase(&_mi_stats_main.reserved, MI_HUGE_OS_PAGE_SIZE); @@ -560,7 +562,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse } } if (elapsed > max_msecs) { - _mi_warning_message("huge page allocation timed out\n"); + _mi_warning_message("huge OS page allocation timed out (after allocating %zu page(s))\n", page); break; } } From 96b55bd0bb6f2f2939c1452391f536aa35706923 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 22 Mar 2023 09:48:40 -0700 Subject: [PATCH 074/102] potential fix for macOS issue #697 --- src/init.c | 8 ++++---- src/options.c | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/init.c b/src/init.c index cd95526c..5dc57300 100644 --- a/src/init.c +++ b/src/init.c @@ -433,7 +433,7 @@ static bool os_preloading = true; // true until this module is initialized static bool mi_redirected = false; // true if malloc redirects to mi_malloc // Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. -bool _mi_preloading(void) { +bool mi_decl_noinline _mi_preloading(void) { return os_preloading; } @@ -476,9 +476,9 @@ static void mi_allocator_done(void) { // Called once by the process loader static void mi_process_load(void) { mi_heap_main_init(); - #if defined(MI_TLS_RECURSE_GUARD) + #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true; - MI_UNUSED(dummy); + if (dummy == NULL) return; // use dummy or otherwise the access may get optimized away (issue #697) #endif os_preloading = false; mi_assert_internal(_mi_is_main_thread()); @@ -522,8 +522,8 @@ void mi_process_init(void) mi_attr_noexcept { // ensure we are called once static mi_atomic_once_t process_init; if (!mi_atomic_once(&process_init)) return; - _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); _mi_process_is_initialized = true; + _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); mi_process_setup_auto_thread_done(); mi_detect_cpu_features(); diff --git a/src/options.c b/src/options.c index 816a2919..6f6655f2 100644 --- a/src/options.c +++ b/src/options.c @@ -275,7 +275,7 @@ static mi_decl_noinline void mi_recurse_exit_prim(void) { static bool mi_recurse_enter(void) { #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) - if (_mi_preloading()) return true; + if (_mi_preloading()) return false; #endif return mi_recurse_enter_prim(); } From d976fbe08bb19112dc0c037d7f47da228fd6dc11 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 22 Mar 2023 09:56:40 -0700 Subject: [PATCH 075/102] remove spurious include --- src/init.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/init.c b/src/init.c index 5dc57300..0f4d4f40 100644 --- a/src/init.c +++ b/src/init.c @@ -380,8 +380,6 @@ void mi_thread_done(void) mi_attr_noexcept { _mi_thread_done(NULL); } -#include - void _mi_thread_done(mi_heap_t* heap) { // calling with NULL implies using the default heap From c9dcca6a648aa4203e5381308dc0a89090e7f8bd Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 21 Mar 2023 19:51:10 -0700 Subject: [PATCH 076/102] update comments --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 3e8a9172..242a4eef 100644 --- a/src/os.c +++ b/src/os.c @@ -21,7 +21,7 @@ static mi_os_mem_config_t mi_os_mem_config = { 0, // large page size (usually 2MiB) 4096, // allocation granularity true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems) - false // must free whole? + false // must free whole? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) }; bool _mi_os_has_overcommit(void) { From a21ddd03fe9dbc6b45f3fd502d695c96e23f3e8e Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 23 Mar 2023 11:21:45 -0700 Subject: [PATCH 077/102] add verbose message if thread sanitizer is enabled --- src/init.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/init.c b/src/init.c index 0f4d4f40..565cffd9 100644 --- a/src/init.c +++ b/src/init.c @@ -527,11 +527,14 @@ void mi_process_init(void) mi_attr_noexcept { mi_detect_cpu_features(); _mi_os_init(); mi_heap_main_init(); - #if (MI_DEBUG) + #if MI_DEBUG _mi_verbose_message("debug level : %d\n", MI_DEBUG); #endif _mi_verbose_message("secure level: %d\n", MI_SECURE); _mi_verbose_message("mem tracking: %s\n", MI_TRACK_TOOL); + #if MI_TSAN + _mi_verbose_message("thread santizer enabled\n"); + #endif mi_thread_init(); #if defined(_WIN32) From 1cbc55f2b8baccf8225024923cc840a5cc0773e7 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 23 Mar 2023 13:05:10 -0700 Subject: [PATCH 078/102] fix initialization of decommit mask for huge pages --- src/segment.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/segment.c b/src/segment.c index 451ef250..c9525490 100644 --- a/src/segment.c +++ b/src/segment.c @@ -509,6 +509,7 @@ static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_ mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->decommit_mask)); // note: assumes commit_mask is always full for huge segments as otherwise the commit mask bits can overflow if (mi_commit_mask_is_full(&segment->commit_mask) && mi_commit_mask_is_empty(&segment->decommit_mask)) return true; // fully committed + mi_assert_internal(segment->kind != MI_SEGMENT_HUGE); return mi_segment_commitx(segment,true,p,size,stats); } @@ -904,6 +905,10 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi mi_assert_internal(!mi_commit_mask_any_set(&segment->decommit_mask, &commit_needed_mask)); #endif } + else { + segment->decommit_expire = 0; + mi_commit_mask_create_empty( &segment->decommit_mask ); + } // initialize segment info const size_t slice_entries = (segment_slices > MI_SLICES_PER_SEGMENT ? MI_SLICES_PER_SEGMENT : segment_slices); From 165b84705132bac86dc680620bd1c35f639809bc Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 23 Mar 2023 16:11:38 -0700 Subject: [PATCH 079/102] improve segment_cache assertions --- include/mimalloc/internal.h | 2 +- src/os.c | 2 +- src/page.c | 2 + src/prim/unix/prim.c | 2 +- src/segment-cache.c | 77 ++++++++++++++++++++----------------- src/segment.c | 2 +- 6 files changed, 48 insertions(+), 39 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 8c9e98a1..710b9e6f 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -115,7 +115,7 @@ mi_arena_id_t _mi_arena_id_none(void); bool _mi_arena_memid_is_suitable(size_t memid, mi_arena_id_t req_arena_id); // "segment-cache.c" -void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); +void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool large_allowed, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld); void _mi_segment_cache_collect(bool force, mi_os_tld_t* tld); void _mi_segment_cache_free_all(mi_os_tld_t* tld); diff --git a/src/os.c b/src/os.c index d6c94b11..5ac37c2e 100644 --- a/src/os.c +++ b/src/os.c @@ -364,7 +364,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ int err = _mi_prim_commit(start, csize, commit); if (err != 0) { - _mi_warning_message("cannot %s OS memory (error: %d (0x%d), address: %p, size: 0x%zx bytes)\n", commit ? "commit" : "decommit", err, err, start, csize); + _mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", commit ? "commit" : "decommit", err, err, start, csize); } mi_assert_internal(err == 0); return (err == 0); diff --git a/src/page.c b/src/page.c index f650af31..fd1af187 100644 --- a/src/page.c +++ b/src/page.c @@ -92,8 +92,10 @@ static bool mi_page_is_valid_init(mi_page_t* page) { } #endif + #if !MI_TSAN mi_block_t* tfree = mi_page_thread_free(page); mi_assert_internal(mi_page_list_is_valid(page, tfree)); + #endif //size_t tfree_count = mi_page_list_count(page, tfree); //mi_assert_internal(tfree_count <= page->thread_freed + 1); diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 0ca9bc64..e51fb6bd 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -395,7 +395,7 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, vo long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); if (err != 0) { err = errno; - _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%d))\n", numa_node, err, err); + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err); } } return (*addr != NULL ? 0 : errno); diff --git a/src/segment-cache.c b/src/segment-cache.c index 4a16a18a..6f9d5fcb 100644 --- a/src/segment-cache.c +++ b/src/segment-cache.c @@ -35,8 +35,8 @@ typedef struct mi_cache_slot_s { static mi_decl_cache_align mi_cache_slot_t cache[MI_CACHE_MAX]; // = 0 -static mi_decl_cache_align mi_bitmap_field_t cache_available[MI_CACHE_FIELDS] = { MI_CACHE_BITS_SET }; // zero bit = available! -static mi_decl_cache_align mi_bitmap_field_t cache_available_large[MI_CACHE_FIELDS] = { MI_CACHE_BITS_SET }; +static mi_decl_cache_align mi_bitmap_field_t cache_unavailable[MI_CACHE_FIELDS] = { MI_CACHE_BITS_SET }; // zero bit = available! +static mi_decl_cache_align mi_bitmap_field_t cache_unavailable_large[MI_CACHE_FIELDS] = { MI_CACHE_BITS_SET }; static mi_decl_cache_align mi_bitmap_field_t cache_inuse[MI_CACHE_FIELDS]; // zero bit = free static bool mi_cdecl mi_segment_cache_is_suitable(mi_bitmap_index_t bitidx, void* arg) { @@ -48,7 +48,8 @@ static bool mi_cdecl mi_segment_cache_is_suitable(mi_bitmap_index_t bitidx, void mi_decl_noinline static void* mi_segment_cache_pop_ex( bool all_suitable, size_t size, mi_commit_mask_t* commit_mask, - mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, + mi_commit_mask_t* decommit_mask, bool large_allowed, + bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t _req_arena_id, size_t* memid, mi_os_tld_t* tld) { #ifdef MI_CACHE_DISABLE @@ -66,23 +67,28 @@ mi_decl_noinline static void* mi_segment_cache_pop_ex( if (start_field >= MI_CACHE_FIELDS) start_field = 0; } - // find an available slot + // find an available slot and make it unavailable mi_bitmap_index_t bitidx = 0; bool claimed = false; mi_arena_id_t req_arena_id = _req_arena_id; mi_bitmap_pred_fun_t pred_fun = (all_suitable ? NULL : &mi_segment_cache_is_suitable); // cannot pass NULL as the arena may be exclusive itself; todo: do not put exclusive arenas in the cache? - if (*large) { // large allowed? - claimed = _mi_bitmap_try_find_from_claim_pred(cache_available_large, MI_CACHE_FIELDS, start_field, 1, pred_fun, &req_arena_id, &bitidx); + if (large_allowed) { // large allowed? + claimed = _mi_bitmap_try_find_from_claim_pred(cache_unavailable_large, MI_CACHE_FIELDS, start_field, 1, pred_fun, &req_arena_id, &bitidx); if (claimed) *large = true; } if (!claimed) { - claimed = _mi_bitmap_try_find_from_claim_pred (cache_available, MI_CACHE_FIELDS, start_field, 1, pred_fun, &req_arena_id, &bitidx); + claimed = _mi_bitmap_try_find_from_claim_pred (cache_unavailable, MI_CACHE_FIELDS, start_field, 1, pred_fun, &req_arena_id, &bitidx); if (claimed) *large = false; } if (!claimed) return NULL; + // no longer available but still in-use + mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable, MI_CACHE_FIELDS, 1, bitidx)); + mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable_large, MI_CACHE_FIELDS, 1, bitidx)); + mi_assert_internal(_mi_bitmap_is_claimed(cache_inuse, MI_CACHE_FIELDS, 1, bitidx)); + // found a slot mi_cache_slot_t* slot = &cache[mi_bitmap_index_bit(bitidx)]; void* p = slot->p; @@ -95,16 +101,15 @@ mi_decl_noinline static void* mi_segment_cache_pop_ex( mi_atomic_storei64_release(&slot->expire,(mi_msecs_t)0); // mark the slot as free again - mi_assert_internal(_mi_bitmap_is_claimed(cache_inuse, MI_CACHE_FIELDS, 1, bitidx)); _mi_bitmap_unclaim(cache_inuse, MI_CACHE_FIELDS, 1, bitidx); return p; #endif } -mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t _req_arena_id, size_t* memid, mi_os_tld_t* tld) +mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool large_allowed, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t _req_arena_id, size_t* memid, mi_os_tld_t* tld) { - return mi_segment_cache_pop_ex(false, size, commit_mask, decommit_mask, large, is_pinned, is_zero, _req_arena_id, memid, tld); + return mi_segment_cache_pop_ex(false, size, commit_mask, decommit_mask, large_allowed, large, is_pinned, is_zero, _req_arena_id, memid, tld); } static mi_decl_noinline void mi_commit_mask_decommit(mi_commit_mask_t* cmask, void* p, size_t total, mi_stats_t* stats) @@ -113,10 +118,11 @@ static mi_decl_noinline void mi_commit_mask_decommit(mi_commit_mask_t* cmask, vo // nothing } else if (mi_commit_mask_is_full(cmask)) { + // decommit the whole in one call _mi_os_decommit(p, total, stats); } else { - // todo: one call to decommit the whole at once? + // decommit parts mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0); size_t part = total/MI_COMMIT_MASK_BITS; size_t idx; @@ -148,21 +154,25 @@ static mi_decl_noinline void mi_segment_cache_purge(bool visit_all, bool force, if (expire != 0 && (force || now >= expire)) { // racy read // seems expired, first claim it from available purged++; - mi_bitmap_index_t bitidx = mi_bitmap_index_create_from_bit(idx); - if (_mi_bitmap_claim(cache_available, MI_CACHE_FIELDS, 1, bitidx, NULL)) { - // was available, we claimed it + mi_bitmap_index_t bitidx = mi_bitmap_index_create_from_bit(idx); + if (_mi_bitmap_claim(cache_unavailable, MI_CACHE_FIELDS, 1, bitidx, NULL)) { // no need to check large as those cannot be decommitted anyways + // it was available, we claimed it (and made it unavailable) + mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable, MI_CACHE_FIELDS, 1, bitidx)); + mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable_large, MI_CACHE_FIELDS, 1, bitidx)); + // we can now access it safely expire = mi_atomic_loadi64_acquire(&slot->expire); if (expire != 0 && (force || now >= expire)) { // safe read + mi_assert_internal(_mi_bitmap_is_claimed(cache_inuse, MI_CACHE_FIELDS, 1, bitidx)); // still expired, decommit it mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0); - mi_assert_internal(!mi_commit_mask_is_empty(&slot->commit_mask) && _mi_bitmap_is_claimed(cache_available_large, MI_CACHE_FIELDS, 1, bitidx)); + mi_assert_internal(!mi_commit_mask_is_empty(&slot->commit_mask)); _mi_abandoned_await_readers(); // wait until safe to decommit // decommit committed parts // TODO: instead of decommit, we could also free to the OS? mi_commit_mask_decommit(&slot->commit_mask, slot->p, MI_SEGMENT_SIZE, tld->stats); mi_commit_mask_create_empty(&slot->decommit_mask); } - _mi_bitmap_unclaim(cache_available, MI_CACHE_FIELDS, 1, bitidx); // make it available again for a pop + _mi_bitmap_unclaim(cache_unavailable, MI_CACHE_FIELDS, 1, bitidx); // make it available again for a pop } if (!visit_all && purged > MI_MAX_PURGE_PER_PUSH) break; // bound to no more than N purge tries per push } @@ -184,23 +194,20 @@ void _mi_segment_cache_free_all(mi_os_tld_t* tld) { mi_commit_mask_t decommit_mask; bool is_pinned; bool is_zero; + bool is_large; size_t memid; const size_t size = MI_SEGMENT_SIZE; - // iterate twice: first large pages, then regular memory - for (int i = 0; i < 2; i++) { - void* p; - do { - // keep popping and freeing the memory - bool large = (i == 0); - p = mi_segment_cache_pop_ex(true /* all */, size, &commit_mask, &decommit_mask, - &large, &is_pinned, &is_zero, _mi_arena_id_none(), &memid, tld); - if (p != NULL) { - size_t csize = _mi_commit_mask_committed_size(&commit_mask, size); - if (csize > 0 && !is_pinned) _mi_stat_decrease(&_mi_stats_main.committed, csize); - _mi_arena_free(p, size, MI_SEGMENT_ALIGN, 0, memid, is_pinned /* pretend not committed to not double count decommits */, tld->stats); - } - } while (p != NULL); - } + void* p; + do { + // keep popping and freeing the memory + p = mi_segment_cache_pop_ex(true /* all */, size, &commit_mask, &decommit_mask, + true /* allow large */, &is_large, &is_pinned, &is_zero, _mi_arena_id_none(), &memid, tld); + if (p != NULL) { + size_t csize = _mi_commit_mask_committed_size(&commit_mask, size); + if (csize > 0 && !is_pinned) { _mi_stat_decrease(&_mi_stats_main.committed, csize); } + _mi_arena_free(p, size, MI_SEGMENT_ALIGN, 0, memid, is_pinned /* pretend not committed to not double count decommits */, tld->stats); + } + } while (p != NULL); } mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld) @@ -228,8 +235,8 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me bool claimed = _mi_bitmap_try_find_from_claim(cache_inuse, MI_CACHE_FIELDS, start_field, 1, &bitidx); if (!claimed) return false; - mi_assert_internal(_mi_bitmap_is_claimed(cache_available, MI_CACHE_FIELDS, 1, bitidx)); - mi_assert_internal(_mi_bitmap_is_claimed(cache_available_large, MI_CACHE_FIELDS, 1, bitidx)); + mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable, MI_CACHE_FIELDS, 1, bitidx)); + mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable_large, MI_CACHE_FIELDS, 1, bitidx)); #if MI_DEBUG>1 if (is_pinned || is_large) { mi_assert_internal(mi_commit_mask_is_full(commit_mask)); @@ -257,7 +264,7 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me } // make it available - _mi_bitmap_unclaim((is_large ? cache_available_large : cache_available), MI_CACHE_FIELDS, 1, bitidx); + _mi_bitmap_unclaim((is_large ? cache_unavailable_large : cache_unavailable), MI_CACHE_FIELDS, 1, bitidx); return true; #endif } @@ -273,7 +280,7 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me #if (MI_INTPTR_SIZE==8) -#define MI_MAX_ADDRESS ((size_t)20 << 40) // 20TB +#define MI_MAX_ADDRESS ((size_t)40 << 40) // 20TB #else #define MI_MAX_ADDRESS ((size_t)2 << 30) // 2Gb #endif diff --git a/src/segment.c b/src/segment.c index c9525490..f8a5d6a0 100644 --- a/src/segment.c +++ b/src/segment.c @@ -809,7 +809,7 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment // get from cache? if (page_alignment == 0) { - segment = (mi_segment_t*)_mi_segment_cache_pop(segment_size, pcommit_mask, pdecommit_mask, &mem_large, &is_pinned, is_zero, req_arena_id, &memid, os_tld); + segment = (mi_segment_t*)_mi_segment_cache_pop(segment_size, pcommit_mask, pdecommit_mask, mem_large, &mem_large, &is_pinned, is_zero, req_arena_id, &memid, os_tld); } // get from OS From 560e32b2e10d50458e1b7c26a92f8fb2d1af777d Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 28 Mar 2023 09:14:17 -0700 Subject: [PATCH 080/102] update comments --- src/arena.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/arena.c b/src/arena.c index d8178cc1..f7957036 100644 --- a/src/arena.c +++ b/src/arena.c @@ -11,12 +11,10 @@ large blocks (>= MI_ARENA_BLOCK_SIZE, 32MiB). In contrast to the rest of mimalloc, the arenas are shared between threads and need to be accessed using atomic operations. -Currently arenas are only used to for huge OS page (1GiB) reservations, -otherwise it delegates to direct allocation from the OS. -In the future, we can expose an API to manually add more kinds of arenas -which is sometimes needed for embedded devices or shared memory for example. -(We can also employ this with WASI or `sbrk` systems to reserve large arenas - on demand and be able to reuse them efficiently). +Arenas are used to for huge OS page (1GiB) reservations or for reserving +OS memory upfront which can be improve performance or is sometimes needed +on embedded devices. We can also employ this with WASI or `sbrk` systems +to reserve large arenas upfront and be able to reuse the memory more effectively. The arena allocation needs to be thread safe and we use an atomic bitmap to allocate. -----------------------------------------------------------------------------*/ From 9792b6364d61e6e6d03fff0f3a53bb7798003957 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 28 Mar 2023 09:25:32 -0700 Subject: [PATCH 081/102] move more prototypes in internal.h for safety --- include/mimalloc/internal.h | 24 +++++++++++++++++++----- src/region.c | 19 ++----------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 32c71ff9..5d06281d 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -86,20 +86,37 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea void _mi_thread_done(mi_heap_t* heap); // os.c -size_t _mi_os_page_size(void); void _mi_os_init(void); // called from process init void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocate thread local data void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data +size_t _mi_os_page_size(void); size_t _mi_os_good_alloc_size(size_t size); bool _mi_os_has_overcommit(void); -bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats); +bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats); +bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); +bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); +bool _mi_os_protect(void* addr, size_t size); +bool _mi_os_unprotect(void* addr, size_t size); + +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_stats_t* stats); void* _mi_os_alloc_aligned_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool* large, mi_stats_t* tld_stats); void _mi_os_free_aligned(void* p, size_t size, size_t alignment, size_t align_offset, bool was_committed, mi_stats_t* tld_stats); void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); bool _mi_os_use_large_page(size_t size, size_t alignment); size_t _mi_os_large_page_size(void); +void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* stats); +void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize); +void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats); + +// arena.c +mi_arena_id_t _mi_arena_id_none(void); +void _mi_arena_free(void* p, size_t size, size_t alignment, size_t align_offset, size_t memid, bool all_committed, mi_stats_t* stats); +void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); +void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); + + // memory.c void* _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t offset, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* id, mi_os_tld_t* tld); void _mi_mem_free(void* p, size_t size, size_t alignment, size_t align_offset, size_t id, bool fully_committed, bool any_reset, mi_os_tld_t* tld); @@ -129,8 +146,6 @@ void _mi_segment_thread_collect(mi_segments_tld_t* tld); void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); void _mi_abandoned_await_readers(void); - - // "page.c" void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; @@ -161,7 +176,6 @@ void _mi_heap_destroy_all(void); // "stats.c" void _mi_stats_done(mi_stats_t* stats); - mi_msecs_t _mi_clock_now(void); mi_msecs_t _mi_clock_end(mi_msecs_t start); mi_msecs_t _mi_clock_start(void); diff --git a/src/region.c b/src/region.c index 29681f4c..77b07eb5 100644 --- a/src/region.c +++ b/src/region.c @@ -39,23 +39,8 @@ Possible issues: #include "bitmap.h" -// Internal raw OS interface -size_t _mi_os_large_page_size(void); -bool _mi_os_protect(void* addr, size_t size); -bool _mi_os_unprotect(void* addr, size_t size); -bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); -bool _mi_os_decommit(void* p, size_t size, mi_stats_t* stats); -bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats); -bool _mi_os_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats); -bool _mi_os_commit_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats); - -// arena.c -mi_arena_id_t _mi_arena_id_none(void); -void _mi_arena_free(void* p, size_t size, size_t alignment, size_t align_offset, size_t memid, bool all_committed, mi_stats_t* stats); -void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); -void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); - - +// os.c +bool _mi_os_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats); // Constants #if (MI_INTPTR_SIZE==8) From 90600188a8624bc30e807b62dc643c0dc7e3d6e7 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 28 Mar 2023 09:58:31 -0700 Subject: [PATCH 082/102] remove superfluous prototypes --- src/arena.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/arena.c b/src/arena.c index 674df73f..18e3f2ac 100644 --- a/src/arena.c +++ b/src/arena.c @@ -28,17 +28,6 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo #include "bitmap.h" // atomic bitmap -// os.c -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_stats_t* stats); -void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* stats); - -void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize); -void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats); - -bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); -bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); - - /* ----------------------------------------------------------- Arena allocation ----------------------------------------------------------- */ From 176b6e6aa0b9eb888f361e6d35b4511825d1d865 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 28 Mar 2023 09:59:41 -0700 Subject: [PATCH 083/102] add mi_arena_is_os_allocated --- include/mimalloc/internal.h | 2 ++ src/arena.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 5d06281d..2d8269e0 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -115,6 +115,8 @@ mi_arena_id_t _mi_arena_id_none(void); void _mi_arena_free(void* p, size_t size, size_t alignment, size_t align_offset, size_t memid, bool all_committed, mi_stats_t* stats); void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); +bool _mi_arena_memid_is_suitable(size_t arena_memid, mi_arena_id_t request_arena_id); +bool _mi_arena_is_os_allocated(size_t arena_memid); // memory.c diff --git a/src/arena.c b/src/arena.c index f7957036..152f7bea 100644 --- a/src/arena.c +++ b/src/arena.c @@ -127,6 +127,10 @@ bool _mi_arena_memid_is_suitable(size_t arena_memid, mi_arena_id_t request_arena return mi_arena_id_is_suitable(id, exclusive, request_arena_id); } +bool _mi_arena_is_os_allocated(size_t arena_memid) { + return (arena_memid == MI_MEMID_OS); +} + static size_t mi_block_count_of_size(size_t size) { return _mi_divide_up(size, MI_ARENA_BLOCK_SIZE); } From 6dd3073a752e479a53b1b7760193103d0e83e5d6 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 28 Mar 2023 10:16:19 -0700 Subject: [PATCH 084/102] avoid caching segments in pinned arenas; happes with huge OS page reservations --- src/segment-cache.c | 17 ++++++++++++----- src/segment.c | 6 ++++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/segment-cache.c b/src/segment-cache.c index 6f9d5fcb..eeae1b50 100644 --- a/src/segment-cache.c +++ b/src/segment-cache.c @@ -216,20 +216,27 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me return false; #else - // only for normal segment blocks + // purge expired entries + mi_segment_cache_purge(false /* limit purges to a constant N */, false /* don't force unexpired */, tld); + + // only cache normal segment blocks if (size != MI_SEGMENT_SIZE || ((uintptr_t)start % MI_SEGMENT_ALIGN) != 0) return false; + // Also do not cache arena allocated segments that cannot be decommitted. (as arena allocation is fast) + // This is a common case with reserved huge OS pages. + // + // (note: we could also allow segments that are already fully decommitted but that never happens + // as the first slice is always committed (for the segment metadata)) + if (!_mi_arena_is_os_allocated(memid) && is_pinned) return false; + // numa node determines start field int numa_node = _mi_os_numa_node(NULL); size_t start_field = 0; if (numa_node > 0) { - start_field = (MI_CACHE_FIELDS / _mi_os_numa_node_count())*numa_node; + start_field = (MI_CACHE_FIELDS / _mi_os_numa_node_count()) * numa_node; if (start_field >= MI_CACHE_FIELDS) start_field = 0; } - // purge expired entries - mi_segment_cache_purge(false /* limit purges to a constant N */, false /* don't force unexpired */, tld); - // find an available slot mi_bitmap_index_t bitidx; bool claimed = _mi_bitmap_try_find_from_claim(cache_inuse, MI_CACHE_FIELDS, start_field, 1, &bitidx); diff --git a/src/segment.c b/src/segment.c index f8a5d6a0..dc25dbda 100644 --- a/src/segment.c +++ b/src/segment.c @@ -397,8 +397,10 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) { if (size != MI_SEGMENT_SIZE || segment->mem_align_offset != 0 || segment->kind == MI_SEGMENT_HUGE || // only push regular segments on the cache !_mi_segment_cache_push(segment, size, segment->memid, &segment->commit_mask, &segment->decommit_mask, segment->mem_is_large, segment->mem_is_pinned, tld->os)) { - const size_t csize = _mi_commit_mask_committed_size(&segment->commit_mask, size); - if (csize > 0 && !segment->mem_is_pinned) _mi_stat_decrease(&_mi_stats_main.committed, csize); + if (!segment->mem_is_pinned) { + const size_t csize = _mi_commit_mask_committed_size(&segment->commit_mask, size); + if (csize > 0) { _mi_stat_decrease(&_mi_stats_main.committed, csize); } + } _mi_abandoned_await_readers(); // wait until safe to free _mi_arena_free(segment, mi_segment_size(segment), segment->mem_alignment, segment->mem_align_offset, segment->memid, segment->mem_is_pinned /* pretend not committed to not double count decommits */, tld->stats); } From 79f31b0e8f7665ceb87b65af57d819dfde7d13b0 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 28 Mar 2023 16:44:35 -0700 Subject: [PATCH 085/102] use syscalls for open/close etc when initializing to avoid recursion when these are intercepted (issue #713) --- src/prim/unix/prim.c | 80 +++++++++++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 0ca9bc64..328dfab3 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -50,6 +50,51 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif +#if !defined(__HAIKU__) + #define MI_HAS_SYSCALL_H + #include +#endif + +//------------------------------------------------------------------------------------ +// Use syscalls for some primitives to allow for libraries that override open/read/close etc. +// and do allocation themselves; using syscalls prevents recursion when mimalloc is +// still initializing (issue #713) +//------------------------------------------------------------------------------------ + +#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) + +static int mi_prim_open(const char* fpath, int open_flags) { + return syscall(SYS_open,fpath,open_flags,0); +} +static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { + return syscall(SYS_read,fd,buf,bufsize); +} +static int mi_prim_close(int fd) { + return syscall(SYS_close,fd); +} +static int mi_prim_access(const char *fpath, int mode) { + return syscall(SYS_access,fpath,mode); +} + +#else + +static int mi_prim_open(const char* fpath, int open_flags) { + return open(fpath,open_flags,mode); +} +static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { + return read(fd,buf,bufsize); +} +static int mi_prim_close(int fd) { + return close(fd); +} +static int mi_prim_access(const char *fpath, int mode) { + return access(fpath,mode); +} + +#endif + + + //--------------------------------------------- // init //--------------------------------------------- @@ -57,11 +102,11 @@ terms of the MIT license. A copy of the license can be found in the file static bool unix_detect_overcommit(void) { bool os_overcommit = true; #if defined(__linux__) - int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); + int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY); if (fd >= 0) { char buf[32]; - ssize_t nread = read(fd, &buf, sizeof(buf)); - close(fd); + ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); + mi_prim_close(fd); // // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) if (nread >= 1) { @@ -367,13 +412,11 @@ int _mi_prim_protect(void* start, size_t size, bool protect) { #if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) -#include - #ifndef MPOL_PREFERRED #define MPOL_PREFERRED 1 #endif -#if defined(SYS_mbind) +#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mbind) static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags); } @@ -417,11 +460,10 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, vo #if defined(__linux__) -#include // getcpu -#include // access +#include // snprintf size_t _mi_prim_numa_node(void) { - #ifdef SYS_getcpu + #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu) unsigned long node = 0; unsigned long ncpu = 0; long err = syscall(SYS_getcpu, &ncpu, &node, NULL); @@ -438,14 +480,14 @@ size_t _mi_prim_numa_node_count(void) { for(node = 0; node < 256; node++) { // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation) snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1); - if (access(buf,R_OK) != 0) break; + if (mi_prim_access(buf,R_OK) != 0) break; } return (node+1); } #elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000 -size_t mi_prim_numa_node(void) { +size_t _mi_prim_numa_node(void) { domainset_t dom; size_t node; int policy; @@ -568,14 +610,14 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) } pinfo->page_faults = 0; #elif defined(__APPLE__) - pinfo->peak_rss = rusage.ru_maxrss; // BSD reports in bytes + pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes struct mach_task_basic_info info; mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { pinfo->current_rss = (size_t)info.resident_size; } #else - pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux reports in KiB + pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB #endif // use defaults for commit } @@ -698,19 +740,17 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { #elif defined(__linux__) || defined(__HAIKU__) -#if defined(__linux__) -#include -#endif #include #include #include #include + bool _mi_prim_random_buf(void* buf, size_t buf_len) { // Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h` // and for the latter the actual `getrandom` call is not always defined. // (see ) // We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed. - #ifdef SYS_getrandom + #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getrandom) #ifndef GRND_NONBLOCK #define GRND_NONBLOCK (1) #endif @@ -726,11 +766,11 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { #if defined(O_CLOEXEC) flags |= O_CLOEXEC; #endif - int fd = open("/dev/urandom", flags, 0); + int fd = mi_prim_open("/dev/urandom", flags); if (fd < 0) return false; size_t count = 0; while(count < buf_len) { - ssize_t ret = read(fd, (char*)buf + count, buf_len - count); + ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count); if (ret<=0) { if (errno!=EAGAIN && errno!=EINTR) break; } @@ -738,7 +778,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { count += ret; } } - close(fd); + mi_prim_close(fd); return (count==buf_len); } From 8ecbc29a020f1c36c3d6c0dcff6646e04d0d25e3 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 11:46:56 -0700 Subject: [PATCH 086/102] prepare readme for release --- readme.md | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/readme.md b/readme.md index b102a50f..b435a8be 100644 --- a/readme.md +++ b/readme.md @@ -12,15 +12,15 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the run-time systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release tag: `v2.0.9` (2022-12-23). -Latest stable tag: `v1.7.9` (2022-12-23). +Latest release tag: `v2.1.0` (2023-03-29). +Latest stable tag: `v1.8.0` (2023-03-29). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: ``` > LD_PRELOAD=/usr/lib/libmimalloc.so myprogram ``` -It also has an easy way to override the default allocator in [Windows](#override_on_windows). Notable aspects of the design include: +It also includes a robust way to override the default allocator in [Windows](#override_on_windows). Notable aspects of the design include: - __small and consistent__: the library is about 8k LOC using simple and consistent data structures. This makes it very suitable @@ -78,6 +78,10 @@ Note: the `v2.x` version has a new algorithm for managing internal mimalloc page and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance (see [below](#performance)); please report if you observe any significant performance regression. +* 2023-03-29, `v1.8.0`, `v2.1.0`: Improved support dynamic overriding on Windows 11. Improved tracing precision + with [#asan] and [#Valgrind], and added Windows event tracing [#ETW] (contributed by Xinglong He). Created an OS + abstraction layer to make it easier to port and separate platform dependent code (in `src/prim`). Fixed C++ STL compilation on older Microsoft C++ compilers, and various small bug fixes. + * 2022-12-23, `v1.7.9`, `v2.0.9`: Supports building with [#asan] and improved [#Valgrind] support. Support abitrary large alignments (in particular for `std::pmr` pools). Added C++ STL allocators attached to a specific heap (thanks @vmarkovtsev). @@ -351,6 +355,7 @@ When _mimalloc_ is built using debug mode, various checks are done at runtime to Generally, we recommend using the standard allocator with memory tracking tools, but mimalloc can also be build to support the [address sanitizer][asan] or the excellent [Valgrind] tool. +Moreover, it can be build to support Windows event tracing ([ETW]). This has a small performance overhead but does allow detecting memory leaks and byte-precise buffer overflows directly on final executables. See also the `test/test-wrong.c` file to test with various tools. @@ -417,6 +422,24 @@ Adress sanitizer support is in its initial development -- please report any issu [asan]: https://github.com/google/sanitizers/wiki/AddressSanitizer +### ETW + +Event tracing for Windows ([ETW]) provides a high performance way to capture all allocations though +mimalloc and analyze them later. To build with ETW support, use the `-DMI_TRACE_ETW=ON` cmake option. + +You can then capture an allocation trace using the Windows performance recorder (WPR), using the +`src/prim/windows/etw-mimalloc.wprp` profile. In an admin prompt, you can use: +``` +> wpr -start src\prim\windows\etw-mimalloc.wprp -filemode +> +> wpr -stop .etl +``` +and then open `.etl` in the Windows Performance Analyzer (WPA), or +use a tool like [TraceControl] that is specialized for analyzing mimalloc traces. + +[ETW]: https://learn.microsoft.com/en-us/windows-hardware/test/wpt/event-tracing-for-windows +[TraceControl]: https://github.com/xinglonghe/TraceControl + # Overriding Standard Malloc @@ -426,7 +449,7 @@ Overriding the standard `malloc` (and `new`) can be done either _dynamically_ or This is the recommended way to override the standard malloc interface. -### Override on Linux, BSD +### Dynamic Override on Linux, BSD On these ELF-based systems we preload the mimalloc shared library so all calls to the standard `malloc` interface are @@ -445,7 +468,7 @@ or run with the debug version to get detailed statistics: > env MIMALLOC_SHOW_STATS=1 LD_PRELOAD=/usr/lib/libmimalloc-debug.so myprogram ``` -### Override on MacOS +### Dynamic Override on MacOS On macOS we can also preload the mimalloc shared library so all calls to the standard `malloc` interface are @@ -458,7 +481,7 @@ Note that certain security restrictions may apply when doing this from the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-ignored-when-calling-application-through-bash). -### Override on Windows +### Dynamic Override on Windows Overriding on Windows is robust and has the particular advantage to be able to redirect all malloc/free calls that go through @@ -491,13 +514,13 @@ Such patching can be done for example with [CFF Explorer](https://ntcore.com/?pa On Unix-like systems, you can also statically link with _mimalloc_ to override the standard malloc interface. The recommended way is to link the final program with the -_mimalloc_ single object file (`mimalloc-override.o`). We use +_mimalloc_ single object file (`mimalloc.o`). We use an object file instead of a library file as linkers give preference to that over archives to resolve symbols. To ensure that the standard malloc interface resolves to the _mimalloc_ library, link it as the first object file. For example: ``` -> gcc -o myprogram mimalloc-override.o myfile1.c ... +> gcc -o myprogram mimalloc.o myfile1.c ... ``` Another way to override statically that works on all platforms, is to From 2440e60d95a9127bb995232a38661559bf59a037 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 11:48:01 -0700 Subject: [PATCH 087/102] copy static.o to the cmake directory (issue #706) --- CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c8e55f1..b1a06360 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -457,12 +457,18 @@ if (MI_BUILD_OBJECT) $ ) + # Copy the generated object file (`static.o`) to the output directory (as `mimalloc.o`) + set(mimalloc-obj-static "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/mimalloc-obj.dir/src/static.c${CMAKE_C_OUTPUT_EXTENSION}") + set(mimalloc-obj-out "${CMAKE_CURRENT_BINARY_DIR}/${mi_basename}${CMAKE_C_OUTPUT_EXTENSION}") + add_custom_command(OUTPUT ${mimalloc-obj-out} DEPENDS mimalloc-obj COMMAND "${CMAKE_COMMAND}" -E copy "${mimalloc-obj-static}" "${mimalloc-obj-out}") + add_custom_target(mimalloc-obj-target ALL DEPENDS ${mimalloc-obj-out}) + # the following seems to lead to cmake warnings/errors on some systems, disable for now :-( # install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_objdir}) # the FILES expression can also be: $ # but that fails cmake versions less than 3.10 so we leave it as is for now - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/mimalloc-obj.dir/src/static.c${CMAKE_C_OUTPUT_EXTENSION} + install(FILES ${mimalloc-obj-static} DESTINATION ${mi_install_objdir} RENAME ${mi_basename}${CMAKE_C_OUTPUT_EXTENSION} ) endif() From e1e1e25d2136b2a94da2331f7f73ad36e7312d26 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 11:52:46 -0700 Subject: [PATCH 088/102] add ASAN to pipeline --- azure-pipelines.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5900b225..01133630 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -98,6 +98,12 @@ jobs: CXX: clang++ BuildType: debug-clang-cxx cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Debug ASAN Clang: + CC: clang + CXX: clang++ + BuildType: debug-asan-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_TRACK_ASAN=ON + steps: - task: CMake@1 inputs: From 651ff2c68b9cd00a31a495f7f0ce64a2014eeadb Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 11:55:00 -0700 Subject: [PATCH 089/102] fix cmake for windows --- CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1a06360..b7ea6efa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -458,10 +458,12 @@ if (MI_BUILD_OBJECT) ) # Copy the generated object file (`static.o`) to the output directory (as `mimalloc.o`) - set(mimalloc-obj-static "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/mimalloc-obj.dir/src/static.c${CMAKE_C_OUTPUT_EXTENSION}") - set(mimalloc-obj-out "${CMAKE_CURRENT_BINARY_DIR}/${mi_basename}${CMAKE_C_OUTPUT_EXTENSION}") - add_custom_command(OUTPUT ${mimalloc-obj-out} DEPENDS mimalloc-obj COMMAND "${CMAKE_COMMAND}" -E copy "${mimalloc-obj-static}" "${mimalloc-obj-out}") - add_custom_target(mimalloc-obj-target ALL DEPENDS ${mimalloc-obj-out}) + if(NOT WIN32) + set(mimalloc-obj-static "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/mimalloc-obj.dir/src/static.c${CMAKE_C_OUTPUT_EXTENSION}") + set(mimalloc-obj-out "${CMAKE_CURRENT_BINARY_DIR}/${mi_basename}${CMAKE_C_OUTPUT_EXTENSION}") + add_custom_command(OUTPUT ${mimalloc-obj-out} DEPENDS mimalloc-obj COMMAND "${CMAKE_COMMAND}" -E copy "${mimalloc-obj-static}" "${mimalloc-obj-out}") + add_custom_target(mimalloc-obj-target ALL DEPENDS ${mimalloc-obj-out}) + endif() # the following seems to lead to cmake warnings/errors on some systems, disable for now :-( # install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_objdir}) From 8e6a475386d16e91799c1e975e9ff83380ca9381 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 12:13:59 -0700 Subject: [PATCH 090/102] add ubsan and tsan to pipeline --- azure-pipelines.yml | 10 ++++++++++ src/alloc.c | 8 ++++---- src/os.c | 2 +- src/page.c | 2 ++ src/region.c | 4 ++-- src/segment.c | 4 ++-- 6 files changed, 21 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 01133630..89710dce 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -103,6 +103,16 @@ jobs: CXX: clang++ BuildType: debug-asan-clang cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_TRACK_ASAN=ON + Debug UBSAN Clang: + CC: clang + CXX: clang++ + BuildType: debug-ubsan-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_UBSAN=ON + Debug TSAN Clang++: + CC: clang + CXX: clang++ + BuildType: debug-tsan-clang-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_USE_CXX=ON -DMI_DEBUG_TSAN=ON steps: - task: CMake@1 diff --git a/src/alloc.c b/src/alloc.c index 301166eb..75d8999c 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -50,7 +50,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz _mi_memzero_aligned(block, zsize - MI_PADDING_SIZE); } -#if (MI_DEBUG>0) && !MI_TRACK_ENABLED +#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN if (!page->is_zero && !zero && !mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); } @@ -406,7 +406,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc } - #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED // note: when tracking, cannot use mi_usable_size with multi-threading + #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading memset(block, MI_DEBUG_FREED, mi_usable_size(block)); #endif @@ -458,7 +458,7 @@ static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block // owning thread can free a block directly if mi_unlikely(mi_check_is_double_free(page, block)) return; mi_check_padding(page, block); - #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED + #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED && !MI_TSAN memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif mi_block_set_next(page, block, page->local_free); @@ -546,7 +546,7 @@ void mi_free(void* p) mi_attr_noexcept if mi_unlikely(mi_check_is_double_free(page, block)) return; mi_check_padding(page, block); mi_stat_free(page, block); - #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED + #if (MI_DEBUG!=0) && !MI_TRACK_ENABLED && !MI_TSAN memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif mi_track_free_size(p, mi_page_usable_size_of(page,block)); // faster then mi_usable_size as we already know the page and that p is unaligned diff --git a/src/os.c b/src/os.c index 242a4eef..f6cc1e68 100644 --- a/src/os.c +++ b/src/os.c @@ -411,7 +411,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) else _mi_stat_decrease(&stats->reset, csize); if (!reset) return true; // nothing to do on unreset! - #if (MI_DEBUG>1) && !MI_TRACK_ENABLED + #if (MI_DEBUG>1) && !MI_TRACK_ENABLED // && !MI_TSAN if (MI_SECURE==0) { memset(start, 0, csize); // pretend it is eagerly reset } diff --git a/src/page.c b/src/page.c index 531293f3..aa9f1bde 100644 --- a/src/page.c +++ b/src/page.c @@ -92,10 +92,12 @@ static bool mi_page_is_valid_init(mi_page_t* page) { } #endif + #if !MI_TRACK_ENABLED && !MI_TSAN mi_block_t* tfree = mi_page_thread_free(page); mi_assert_internal(mi_page_list_is_valid(page, tfree)); //size_t tfree_count = mi_page_list_count(page, tfree); //mi_assert_internal(tfree_count <= page->thread_freed + 1); + #endif size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free); mi_assert_internal(page->used + free_count == page->capacity); diff --git a/src/region.c b/src/region.c index 77b07eb5..6c8ffb79 100644 --- a/src/region.c +++ b/src/region.c @@ -315,7 +315,7 @@ static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* large, bool* } mi_assert_internal(!_mi_bitmap_is_any_claimed(®ion->reset, 1, blocks, bit_idx)); - #if (MI_DEBUG>=2) && !MI_TRACK_ENABLED + #if (MI_DEBUG>=2) && !MI_TRACK_ENABLED // && !MI_TSAN if (*commit) { ((uint8_t*)p)[0] = 0; } #endif @@ -361,7 +361,7 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t align_offset, if (p != NULL) { mi_assert_internal(((uintptr_t)p + align_offset) % alignment == 0); - #if (MI_DEBUG>=2) && !MI_TRACK_ENABLED + #if (MI_DEBUG>=2) && !MI_TRACK_ENABLED // && !MI_TSAN if (*commit) { ((uint8_t*)p)[0] = 0; } // ensure the memory is committed #endif } diff --git a/src/segment.c b/src/segment.c index 6cdf4fe7..8a9c8fe1 100644 --- a/src/segment.c +++ b/src/segment.c @@ -1182,7 +1182,7 @@ static mi_page_t* mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, mi_p mi_assert_internal(free_queue->first != NULL); mi_page_t* const page = mi_segment_page_alloc_in(free_queue->first, tld); mi_assert_internal(page != NULL); -#if MI_DEBUG>=2 && !MI_TRACK_ENABLED +#if MI_DEBUG>=2 && !MI_TRACK_ENABLED // && !MI_TSAN // verify it is committed _mi_segment_page_start(_mi_page_segment(page), page, sizeof(void*), NULL, NULL)[0] = 0; #endif @@ -1206,7 +1206,7 @@ static mi_page_t* mi_segment_large_page_alloc(mi_heap_t* heap, size_t block_size if (segment == NULL) return NULL; mi_page_t* page = mi_segment_find_free(segment, tld); mi_assert_internal(page != NULL); -#if MI_DEBUG>=2 && !MI_TRACK_ENABLED +#if MI_DEBUG>=2 && !MI_TRACK_ENABLED // && !MI_TSAN _mi_segment_page_start(segment, page, sizeof(void*), NULL, NULL)[0] = 0; #endif return page; From 0ca31d2db1f15f1524045050e7e623aa22c4470b Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 12:20:29 -0700 Subject: [PATCH 091/102] fix pipeline for tsan --- azure-pipelines.yml | 2 +- test/test-api-fill.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 89710dce..9f4f7bac 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,7 +43,7 @@ jobs: solution: $(BuildType)/libmimalloc.sln configuration: '$(MSBuildConfiguration)' msbuildArguments: -m - - script: ctest --verbose --timeout 120 -C $(MSBuildConfiguration) + - script: ctest --verbose --timeout 180 -C $(MSBuildConfiguration) workingDirectory: $(BuildType) displayName: CTest #- script: $(BuildType)\$(BuildType)\mimalloc-test-stress diff --git a/test/test-api-fill.c b/test/test-api-fill.c index 7ba79880..3fca3b9d 100644 --- a/test/test-api-fill.c +++ b/test/test-api-fill.c @@ -164,7 +164,7 @@ int main(void) { mi_free(p); }; -#if MI_DEBUG >= 2 +#if (MI_DEBUG >= 2) && !MI_TSAN // --------------------------------------------------- // Debug filling // --------------------------------------------------- From 068d1d8842ac95f4b684a7e929b944dc5cbeefdb Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 12:27:22 -0700 Subject: [PATCH 092/102] increase timeout in pipeline for linux for tsan run --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9f4f7bac..5054a532 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,7 +43,7 @@ jobs: solution: $(BuildType)/libmimalloc.sln configuration: '$(MSBuildConfiguration)' msbuildArguments: -m - - script: ctest --verbose --timeout 180 -C $(MSBuildConfiguration) + - script: ctest --verbose --timeout 120 -C $(MSBuildConfiguration) workingDirectory: $(BuildType) displayName: CTest #- script: $(BuildType)\$(BuildType)\mimalloc-test-stress @@ -121,7 +121,7 @@ jobs: cmakeArgs: .. $(cmakeExtraArgs) - script: make -j$(nproc) -C $(BuildType) displayName: Make - - script: ctest --verbose --timeout 120 + - script: ctest --verbose --timeout 180 workingDirectory: $(BuildType) displayName: CTest # - upload: $(Build.SourcesDirectory)/$(BuildType) From 5bc724d18fcfa375c8eac4f648b093d340208914 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 29 Mar 2023 12:39:34 -0700 Subject: [PATCH 093/102] fix warnings on macOS --- src/prim/unix/prim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 328dfab3..df39737a 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -50,7 +50,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif -#if !defined(__HAIKU__) +#if !defined(__HAIKU__) && !defined(__APPLE__) #define MI_HAS_SYSCALL_H #include #endif @@ -76,7 +76,7 @@ static int mi_prim_access(const char *fpath, int mode) { return syscall(SYS_access,fpath,mode); } -#else +#elif !defined(__APPLE__) // avoid unused warnings static int mi_prim_open(const char* fpath, int open_flags) { return open(fpath,open_flags,mode); From 1cb0193ff13d7c3382808b2b7d1fb2bc51c41d18 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 13:00:35 -0700 Subject: [PATCH 094/102] fix hex formatting of errors --- src/os.c | 4 ++-- src/prim/unix/prim.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/os.c b/src/os.c index f6cc1e68..a6732081 100644 --- a/src/os.c +++ b/src/os.c @@ -375,7 +375,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ int err = _mi_prim_commit(start, csize, commit); if (err != 0) { - _mi_warning_message("cannot %s OS memory (error: %d (0x%d), address: %p, size: 0x%zx bytes)\n", commit ? "commit" : "decommit", err, err, start, csize); + _mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", commit ? "commit" : "decommit", err, err, start, csize); } mi_assert_internal(err == 0); return (err == 0); @@ -533,7 +533,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse void* p = NULL; int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &p); if (err != 0) { - _mi_warning_message("unable to allocate huge OS page (error: %d (0x%d), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE); + _mi_warning_message("unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE); break; } diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index df39737a..189631f0 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -170,7 +170,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { int err = errno; - _mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%d), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + _mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); } if (p!=MAP_FAILED) return p; // fall back to regular mmap @@ -191,7 +191,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p p = mmap(hint, size, protect_flags, flags, fd, 0); if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { int err = errno; - _mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%d), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + _mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); } if (p!=MAP_FAILED) return p; // fall back to regular mmap @@ -438,7 +438,7 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, vo long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); if (err != 0) { err = errno; - _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%d))\n", numa_node, err, err); + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err); } } return (*addr != NULL ? 0 : errno); From f20d22a3e42e14258dc33f7653bfd20bc9c990e1 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 15:53:05 -0700 Subject: [PATCH 095/102] fix arm compilation (PR 694) --- src/prim/unix/prim.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 189631f0..a9fc9134 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -759,7 +759,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK); if (ret >= 0) return (buf_len == (size_t)ret); if (errno != ENOSYS) return false; - mi_atomic_store_release(&no_getrandom, 1UL); // don't call again, and fall back to /dev/urandom + mi_atomic_store_release(&no_getrandom, (uintptr_t)1); // don't call again, and fall back to /dev/urandom } #endif int flags = O_RDONLY; From 61a32e62b827182f5c3b2bb694e82fb0328a7c88 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 15:56:31 -0700 Subject: [PATCH 096/102] support cygwin compilation (pr #712) --- src/prim/unix/prim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index a9fc9134..608e96ce 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -50,7 +50,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif -#if !defined(__HAIKU__) && !defined(__APPLE__) +#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) #define MI_HAS_SYSCALL_H #include #endif @@ -410,7 +410,7 @@ int _mi_prim_protect(void* start, size_t size, bool protect) { // Huge page allocation //--------------------------------------------- -#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) +#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__) #ifndef MPOL_PREFERRED #define MPOL_PREFERRED 1 From c2a2533fba6fa08449d2831cef818accd7c15256 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 29 Mar 2023 16:02:46 -0700 Subject: [PATCH 097/102] fix haiku compilation (pr #679) --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b7ea6efa..4957ee7b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -262,6 +262,11 @@ if(MI_USE_CXX) endif() endif() +if(CMAKE_SYSTEM_NAME MATCHES "Haiku") + SET(CMAKE_INSTALL_LIBDIR ~/config/non-packaged/lib) + SET(CMAKE_INSTALL_INCLUDEDIR ~/config/non-packaged/headers) + endif() + # Compiler flags if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden) From dd7b99d477c4a92c8e27f86cbfe660849995f242 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 16:10:01 -0700 Subject: [PATCH 098/102] apply some spelling fixes from PR #710 --- doc/mimalloc-doc.h | 2 +- docs/build.html | 2 +- src/init.c | 2 +- src/prim/unix/prim.c | 2 +- test/test-stress.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index 9525c9ea..4c23a5fa 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -927,7 +927,7 @@ template struct mi_stl_allocator { } /*! \page build Building -Checkout the sources from Github: +Checkout the sources from GitHub: ``` git clone https://github.com/microsoft/mimalloc ``` diff --git a/docs/build.html b/docs/build.html index eac5392b..41e0199f 100644 --- a/docs/build.html +++ b/docs/build.html @@ -98,7 +98,7 @@ $(document).ready(function(){initNavTree('build.html',''); initResizable(); });
Building
-

Checkout the sources from Github:

git clone https://github.com/microsoft/mimalloc
+

Checkout the sources from GitHub:

git clone https://github.com/microsoft/mimalloc

Windows

Open ide/vs2019/mimalloc.sln in Visual Studio 2019 and build (or ide/vs2017/mimalloc.sln). The mimalloc project builds a static library (in out/msvc-x64), while the mimalloc-override project builds a DLL for overriding malloc in the entire program.

macOS, Linux, BSD, etc.

diff --git a/src/init.c b/src/init.c index 565cffd9..38f62f65 100644 --- a/src/init.c +++ b/src/init.c @@ -507,7 +507,7 @@ static void mi_detect_cpu_features(void) { // FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) int32_t cpu_info[4]; __cpuid(cpu_info, 7); - _mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see + _mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see } #else static void mi_detect_cpu_features(void) { diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 608e96ce..c12e4c0d 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -652,7 +652,7 @@ void _mi_prim_out_stderr( const char* msg ) { //---------------------------------------------------------------- #if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0) -// On Posix systemsr use `environ` to acces environment variables +// On Posix systemsr use `environ` to access environment variables // even before the C runtime is initialized. #if defined(__APPLE__) && defined(__has_include) && __has_include() #include diff --git a/test/test-stress.c b/test/test-stress.c index 87e41736..f253000f 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -7,7 +7,7 @@ terms of the MIT license. /* This is a stress test for the allocator, using multiple threads and transferring objects between threads. It tries to reflect real-world workloads: - allocation size is distributed linearly in powers of two - - with some fraction extra large (and some extra extra large) + - with some fraction extra large (and some very large) - the allocations are initialized and read again at free - pointers transfer between threads - threads are terminated and recreated with some objects surviving in between From f806eb84984b4a64de5ffaba4b931b0dcb2b72a6 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 16:15:20 -0700 Subject: [PATCH 099/102] remove redundant OR in mmap flags (pr #675) --- src/prim/unix/prim.c | 1 - src/segment.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index c12e4c0d..077bf820 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -166,7 +166,6 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { size_t n = mi_bsr(try_alignment); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB - flags |= MAP_ALIGNED(n); p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { int err = errno; diff --git a/src/segment.c b/src/segment.c index 8a9c8fe1..ae15847c 100644 --- a/src/segment.c +++ b/src/segment.c @@ -771,7 +771,7 @@ We maintain a global list of abandoned segments that are reclaimed on demand. Since this is shared among threads the implementation needs to avoid the A-B-A problem on popping abandoned segments: -We use tagged pointers to avoid accidentially identifying +We use tagged pointers to avoid accidentally identifying reused segments, much like stamped references in Java. Secondly, we maintain a reader counter to avoid resetting or decommitting segments that have a pending read operation. From accdc4de708b724a8fad414f4b9a6e3a1c7ed19d Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 16:17:10 -0700 Subject: [PATCH 100/102] fix MI_SKIP_COLLECT_ON_EXIT syntax (pr #671 by @Unsttopabull) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4957ee7b..9550f77f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,7 @@ option(MI_BUILD_OBJECT "Build object library" ON) option(MI_BUILD_TESTS "Build test executables" ON) option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) -option(MI_SKIP_COLLECT_ON_EXIT, "Skip collecting memory on program exit" OFF) +option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) # deprecated options From 06b94fc5e6cb8a01facc4cf0d917b1a8b0b5db89 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 16:19:07 -0700 Subject: [PATCH 101/102] remove redundant code (pr #674 by @rganesan) --- src/heap.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/heap.c b/src/heap.c index 0c372c5b..84b0ec4f 100644 --- a/src/heap.c +++ b/src/heap.c @@ -226,9 +226,6 @@ static void mi_heap_reset_pages(mi_heap_t* heap) { mi_assert_internal(mi_heap_is_initialized(heap)); // TODO: copy full empty heap instead? memset(&heap->pages_free_direct, 0, sizeof(heap->pages_free_direct)); -#ifdef MI_MEDIUM_DIRECT - memset(&heap->pages_free_medium, 0, sizeof(heap->pages_free_medium)); -#endif _mi_memcpy_aligned(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages)); heap->thread_delayed_free = NULL; heap->page_count = 0; From 62949f55b3d83d770868a0f4b151ae1ff33e1a78 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 29 Mar 2023 16:29:59 -0700 Subject: [PATCH 102/102] increment doxy file version number --- doc/doxyfile | 2 +- readme.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/doxyfile b/doc/doxyfile index 73266f4e..55cae8bf 100644 --- a/doc/doxyfile +++ b/doc/doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = mi-malloc # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.7/2.0 +PROJECT_NUMBER = 1.8/2.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/readme.md b/readme.md index b435a8be..74f95098 100644 --- a/readme.md +++ b/readme.md @@ -9,7 +9,7 @@ mimalloc (pronounced "me-malloc") is a general purpose allocator with excellent [performance](#performance) characteristics. -Initially developed by Daan Leijen for the run-time systems of the +Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. Latest release tag: `v2.1.0` (2023-03-29).