From 4f9d991449380ed7d881a25ba02cc5668c1ff394 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 15 Jan 2019 14:17:55 +0000 Subject: [PATCH] Initial commit of snmalloc History squashed from internal development. Internal history has commit hash: e27a0e485c44a5003a802de2661ce3b21e120316 --- .clang-format | 110 ++ .gitignore | 333 +---- .vsts-ci.yml | 92 ++ CMakeLists.txt | 107 ++ README.md | 78 ++ check-format.sh | 18 + src/ds/aba.h | 113 ++ src/ds/bits.h | 466 +++++++ src/ds/csv.h | 55 + src/ds/dllist.h | 120 ++ src/ds/flaglock.h | 24 + src/ds/helpers.h | 33 + src/ds/mpmcstack.h | 75 ++ src/ds/mpscq.h | 99 ++ src/mem/alloc.h | 1091 +++++++++++++++++ src/mem/allocconfig.h | 121 ++ src/mem/allocslab.h | 19 + src/mem/allocstats.h | 391 ++++++ src/mem/baseslab.h | 27 + src/mem/globalalloc.h | 171 +++ src/mem/largealloc.h | 207 ++++ src/mem/mediumslab.h | 130 ++ src/mem/metaslab.h | 166 +++ src/mem/pagemap.h | 227 ++++ src/mem/remoteallocator.h | 64 + src/mem/sizeclass.h | 148 +++ src/mem/sizeclasstable.h | 71 ++ src/mem/slab.h | 165 +++ src/mem/superslab.h | 233 ++++ src/mem/threadalloc.h | 265 ++++ src/mem/typealloc.h | 86 ++ src/mem/typeallocated.h | 19 + src/override/malloc.cc | 188 +++ src/override/new.cc | 67 + src/pal/pal.h | 46 + src/pal/pal_free_bsd_kernel.h | 90 ++ src/pal/pal_freebsd.h | 88 ++ src/pal/pal_linux.h | 93 ++ src/pal/pal_open_enclave.h | 58 + src/pal/pal_plain.h | 34 + src/pal/pal_windows.h | 111 ++ src/snmalloc.h | 3 + src/test/func/fixed_region/fixed_region.cc | 52 + src/test/func/memory/memory.cc | 275 +++++ src/test/func/rounding/rounding.cc | 36 + src/test/histogram.h | 249 ++++ src/test/measuretime.h | 14 + src/test/opt.h | 91 ++ src/test/perf/contention/contention.cc | 176 +++ .../perf/external_pointer/externalpointer.cc | 78 ++ src/test/perf/singlethread/singlethread.cc | 83 ++ src/test/usage.h | 42 + src/test/xoroshiro.h | 71 ++ 53 files changed, 6940 insertions(+), 329 deletions(-) create mode 100644 .clang-format create mode 100644 .vsts-ci.yml create mode 100644 CMakeLists.txt create mode 100644 check-format.sh create mode 100644 src/ds/aba.h create mode 100644 src/ds/bits.h create mode 100644 src/ds/csv.h create mode 100644 src/ds/dllist.h create mode 100644 src/ds/flaglock.h create mode 100644 src/ds/helpers.h create mode 100644 src/ds/mpmcstack.h create mode 100644 src/ds/mpscq.h create mode 100644 src/mem/alloc.h create mode 100644 src/mem/allocconfig.h create mode 100644 src/mem/allocslab.h create mode 100644 src/mem/allocstats.h create mode 100644 src/mem/baseslab.h create mode 100644 src/mem/globalalloc.h create mode 100644 src/mem/largealloc.h create mode 100644 src/mem/mediumslab.h create mode 100644 src/mem/metaslab.h create mode 100644 src/mem/pagemap.h create mode 100644 src/mem/remoteallocator.h create mode 100644 src/mem/sizeclass.h create mode 100644 src/mem/sizeclasstable.h create mode 100644 src/mem/slab.h create mode 100644 src/mem/superslab.h create mode 100644 src/mem/threadalloc.h create mode 100644 src/mem/typealloc.h create mode 100644 src/mem/typeallocated.h create mode 100644 src/override/malloc.cc create mode 100644 src/override/new.cc create mode 100644 src/pal/pal.h create mode 100644 src/pal/pal_free_bsd_kernel.h create mode 100644 src/pal/pal_freebsd.h create mode 100644 src/pal/pal_linux.h create mode 100644 src/pal/pal_open_enclave.h create mode 100644 src/pal/pal_plain.h create mode 100644 src/pal/pal_windows.h create mode 100644 src/snmalloc.h create mode 100644 src/test/func/fixed_region/fixed_region.cc create mode 100644 src/test/func/memory/memory.cc create mode 100644 src/test/func/rounding/rounding.cc create mode 100644 src/test/histogram.h create mode 100644 src/test/measuretime.h create mode 100644 src/test/opt.h create mode 100644 src/test/perf/contention/contention.cc create mode 100644 src/test/perf/external_pointer/externalpointer.cc create mode 100644 src/test/perf/singlethread/singlethread.cc create mode 100644 src/test/usage.h create mode 100644 src/test/xoroshiro.h diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..ca08a86 --- /dev/null +++ b/.clang-format @@ -0,0 +1,110 @@ +--- +Language: Cpp +# BasedOnStyle: LLVM +AccessModifierOffset: -2 +AlignAfterOpenBracket: AlwaysBreak +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: DontAlign +AlignOperands: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: true + AfterControlStatement: true + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: true + AfterStruct: true + AfterUnion: true + AfterExternBlock: true + BeforeCatch: true + BeforeElse: true + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: false +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 0 +ContinuationIndentWidth: 2 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: false +ForEachMacros: + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + - Regex: '.*' + Priority: 1 +IncludeIsMainRegex: '(Test)?$' +IndentCaseLabels: true +IndentPPDirectives: AfterHash +IndentWidth: 2 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: All +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Left +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: false +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 2 +UseTab: Never +... + diff --git a/.gitignore b/.gitignore index 3e759b7..dc61f34 100644 --- a/.gitignore +++ b/.gitignore @@ -1,330 +1,5 @@ -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. -## -## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore - -# User-specific files -*.suo -*.user -*.userosscache -*.sln.docstates - -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ - -# Visual Studio 2015/2017 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ - -# Visual Studio 2017 auto generated files -Generated\ Files/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -# NUNIT -*.VisualState.xml -TestResult.xml - -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# Benchmark Results -BenchmarkDotNet.Artifacts/ - -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ -**/Properties/launchSettings.json - -# StyleCop -StyleCopReport.xml - -# Files built by Visual Studio -*_i.c -*_p.c -*_i.h -*.ilk -*.meta -*.obj -*.iobj -*.pch -*.pdb -*.ipdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*.log -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# JustCode is a .NET coding add-in -.JustCode - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - -# Visual Studio code coverage results -*.coverage -*.coveragexml - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# The packages folder can be ignored because of Package Restore -**/[Pp]ackages/* -# except build/, which is used as an MSBuild target. -!**/[Pp]ackages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/[Pp]ackages/repositories.config -# NuGet v3's project.json files produces more ignorable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt -*.appx - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!*.[Cc]ache/ - -# Others -ClientBin/ -~$* +build*/ +CMakeFiles/ +.vscode/ *~ -*.dbmdl -*.dbproj.schemaview -*.jfm -*.pfx -*.publishsettings -orleans.codegen.cs - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm -ServiceFabricBackup/ -*.rptproj.bak - -# SQL Server files -*.mdf -*.ldf -*.ndf - -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings -*.rptproj.rsuser - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat -node_modules/ - -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# JetBrains Rider -.idea/ -*.sln.iml - -# CodeRush -.cr/ - -# Python Tools for Visual Studio (PTVS) -__pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig - -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs - -# OpenCover UI analysis results -OpenCover/ - -# Azure Stream Analytics local run output -ASALocalRun/ - -# MSBuild Binary and Structured Log -*.binlog - -# NVidia Nsight GPU debugger configuration file -*.nvuser - -# MFractors (Xamarin productivity tool) working folder -.mfractor/ +*.sw? diff --git a/.vsts-ci.yml b/.vsts-ci.yml new file mode 100644 index 0000000..5242703 --- /dev/null +++ b/.vsts-ci.yml @@ -0,0 +1,92 @@ +resources: +- repo: self + +phases: +- phase: Linux + queue: + name: 'Hosted Ubuntu 1604' + parallel: 2 + matrix: + Debug: + BuildType: Debug + Release: + BuildType: Release + + steps: + - script: | + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + sudo apt-get update + sudo apt-get install -y ninja-build libc++-dev libc++abi-dev libc++abi1 libstdc++-7-dev + # sudo apt-get install clang-6.0 clang++-6.0 + + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100 + sudo update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 + sudo update-alternatives --set cc /usr/bin/clang + + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100 + sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 + sudo update-alternatives --set c++ /usr/bin/clang++ + + displayName: 'Install Build Dependencies' + + - task: CMake@1 + displayName: 'CMake .. -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="-stdlib=libstdc++ -std=c++17"' + inputs: + cmakeArgs: '.. -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="-stdlib=libstdc++ -std=c++17"' + + - script: | + ninja + ctest -j 4 --output-on-failure + + workingDirectory: build + failOnStderr: true + displayName: 'Compile & Test' + + - script: | + sudo cp libsnmallocshim.so /usr/local/lib/ + ninja clean + LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja + workingDirectory: build + failOnStderr: true + displayName: 'LD_PRELOAD Compile' + +- phase: Windows + queue: + name: 'Hosted VS2017' + parallel: 2 + matrix: + Debug: + BuildType: Debug + Release: + BuildType: Release + + steps: + - task: CMake@1 + displayName: 'CMake .. -G"Visual Studio 15 2017 Win64"' + inputs: + cmakeArgs: '.. -G"Visual Studio 15 2017 Win64"' + + - task: MSBuild@1 + displayName: 'Build solution build/snmalloc.sln' + inputs: + solution: build/snmalloc.sln + msbuildArguments: '/m /p:Configuration=$(BuildType)' + + - script: 'ctest -j 4 --interactive-debug-mode 0 --output-on-failure' + workingDirectory: build + displayName: 'Run Ctest' + +- phase: Format + queue: + name: 'Hosted Ubuntu 1604' + steps: + - script: | + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" + sudo apt-get update + sudo apt-get install -y clang-format-6.0 + sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-6.0 100 + + bash check-format.sh + + displayName: 'Check Format' diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..d39ff29 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,107 @@ +cmake_minimum_required(VERSION 3.8) +project(snmalloc C CXX) + +option(USE_SNMALLOC_STATS "Track allocation stats" OFF) +option(USE_MEASURE "Measure performance with histograms" OFF) +option(USE_SBRK "Use sbrk instead of mmap" OFF) + +macro(subdirlist result curdir) + file(GLOB children LIST_DIRECTORIES true RELATIVE ${curdir} ${curdir}/*) + set(dirlist "") + foreach(child ${children}) + if(IS_DIRECTORY ${curdir}/${child}) + list(APPEND dirlist ${child}) + endif() + endforeach() + set(${result} ${dirlist}) +endmacro() + +macro(linklibs project) + if(NOT MSVC) + target_link_libraries(${project} ${CMAKE_THREAD_LIBS_INIT}) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + target_link_libraries(${project} atomic) + endif() + endif() +endmacro() + +if(MSVC) + add_compile_options(/WX /W4 /wd4127 /wd4324 /wd4201 /std:c++latest) + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi") + set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG") +else() + find_package(Threads REQUIRED) + add_compile_options(-mcx16 -march=native -Wall -Wextra -Werror -g) +endif() + +set(CMAKE_CXX_STANDARD 17) + +if(USE_SNMALLOC_STATS) + add_definitions(-DUSE_SNMALLOC_STATS) +endif() + +if(USE_MEASURE) + add_definitions(-DUSE_MEASURE) +endif() + +if(USE_SBRK) + add_definitions(-DUSE_SBRK) +endif() + +if(NOT MSVC) + add_library(snmallocshim SHARED src/override/malloc.cc) + target_link_libraries(snmallocshim -pthread) + target_include_directories(snmallocshim PRIVATE src) +endif() + +enable_testing() + +set(TESTDIR ${CMAKE_CURRENT_SOURCE_DIR}/src/test) +subdirlist(TEST_CATEGORIES ${TESTDIR}) +foreach(TEST_CATEGORY ${TEST_CATEGORIES}) + subdirlist(TESTS ${TESTDIR}/${TEST_CATEGORY}) + foreach(TEST ${TESTS}) + unset(SRC) + aux_source_directory(${TESTDIR}/${TEST_CATEGORY}/${TEST} SRC) + set(TESTNAME "${TEST_CATEGORY}-${TEST}") + add_executable(${TESTNAME} ${SRC} src/override/new.cc) + target_include_directories(${TESTNAME} PRIVATE src) + linklibs(${TESTNAME}) + add_test(${TESTNAME} ${TESTNAME}) + endforeach() +endforeach() + + +# The clang-format tool is installed under a variety of different names. Try +# to find a sensible one. Only look for 6.0 and 7.0 versions explicitly - we +# don't know whether our clang-format file will work with newer versions of the +# tool +set(CLANG_FORMAT_NAMES + clang-format-7.0 + clang-format-6.0 + clang-format70 + clang-format60 + clang-format) + +# Loop over each of the possible names of clang-format and try to find one. +set(CLANG_FORMAT CLANG_FORMAT-NOTFOUND) +foreach (NAME IN ITEMS ${CLANG_FORMAT_NAMES}) + if (${CLANG_FORMAT} STREQUAL "CLANG_FORMAT-NOTFOUND") + find_program(CLANG_FORMAT ${NAME}) + endif () +endforeach() + +# If we've found a clang-format tool, generate a target for it, otherwise emit +# a warning. +if (${CLANG_FORMAT} STREQUAL "CLANG_FORMAT-NOTFOUND") + message(WARNING "Not generating clangformat target, no clang-format tool found") +else () + message(STATUS "Generating clangformat target using ${CLANG_FORMAT}") + file(GLOB_RECURSE ALL_SOURCE_FILES *.cc *.h *.hh) + add_custom_target( + clangformat + COMMAND ${CLANG_FORMAT} + -i + ${ALL_SOURCE_FILES} + ) +endif() diff --git a/README.md b/README.md index 72f1506..da7bf5b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,81 @@ +# snmalloc + +snmalloc is a research allocator. Its key design features are: + +* Memory that is freed by the same thread that allocated it does not require any + synchronising operations. +* Freeing memory in a different thread to initially allocated it, does not take + any locks and instead uses a novel message passing scheme to return the + memory to the original allocator, where it is recycled. +* The allocator uses large ranges of pages to reduce the amount of meta-data + required. + +# Building on Windows + +The Windows build currently depends on Visual Studio 2017. +To build with Visual Studio: + +``` +mkdir build +cd build +cmake -G "Visual Studio 15 2017 Win64" .. +cmake --build . --config Debug +cmake --build . --config Release +cmake --build . --config RelWithDebInfo +``` + +You can also omit the last three steps and build from the IDE. +Visual Studio builds use a separate directory to keep the binaries for each +build configuration. + +Alternatively, you can follow the steps in the next section to build with Ninja +using the Visual Studio compiler. + +# Building on Linux or FreeBSD +Snmalloc has very few dependencies, CMake, Ninja, Clang 6.0 or later and a C++17 +standard library. +Building with GCC is currently not supported because GCC lacks support for the +`selectany` attribute to specify variables in a COMDAT. + +To build a debug configuration: +``` +mkdir build +cd build +cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Debug +ninja +``` +To build a release configuration: +``` +mkdir build +cd build +cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release +ninja +``` +To build with optimizations on, but with debug information: +``` +mkdir build +cd build +cmake -G Ninja .. -DCMAKE_BUILD_TYPE=RelWithDebInfo +ninja +``` + +The build produces a binary `libsnmallocshim.so`. This file can be +`LD_PRELOAD`ed to use the allocator in place of the system allocator, for +example, you can run the build script using the snmalloc as the allocator for +your toolchain: + +``` +LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja +``` + +# CMake Feature Flags + +These can be added to your cmake command line. + +``` +-DUSE_SNMALLOC_STATS=ON // Track allocation stats +-DUSE_MEASURE=ON // Measure performance with histograms +``` # Contributing diff --git a/check-format.sh b/check-format.sh new file mode 100644 index 0000000..bcd1deb --- /dev/null +++ b/check-format.sh @@ -0,0 +1,18 @@ + +set -u + +unformatted_files="" +for f in `find . -name *.h -o -name *.hh -o -name *.cc`; do + d=`clang-format -style=file $f | diff $f -` + if [ "$d" != "" ]; then + if [ "$unformatted_files" != "" ]; then + unformatted_files+=$'\n' + fi + unformatted_files+="$f" + fi +done + +if [ "$unformatted_files" != "" ]; then + echo "$unformatted_files" + exit 1 +fi diff --git a/src/ds/aba.h b/src/ds/aba.h new file mode 100644 index 0000000..6c89c7c --- /dev/null +++ b/src/ds/aba.h @@ -0,0 +1,113 @@ +#pragma once + +#include "bits.h" + +namespace snmalloc +{ + template + class ABA + { + public: +#ifdef PLATFORM_IS_X86 + struct alignas(2 * sizeof(std::size_t)) Linked + { + T* ptr; + uintptr_t aba; + }; + + struct Independent + { + std::atomic ptr; + std::atomic aba; + }; + + static_assert( + sizeof(Linked) == sizeof(Independent), + "Expecting identical struct sizes in union"); + static_assert( + sizeof(Linked) == (2 * sizeof(std::size_t)), + "Expecting ABA to be the size of two pointers"); + + using Cmp = Linked; +#else + using Cmp = T*; +#endif + + private: +#ifdef PLATFORM_IS_X86 + union + { + alignas(2 * sizeof(std::size_t)) std::atomic linked; + Independent independent; + }; +#else + std::atomic ptr; +#endif + + public: + ABA() + { + if constexpr (c == RequiresInit) + init(nullptr); + } + + void init(T* x) + { +#ifdef PLATFORM_IS_X86 + independent.ptr.store(x, std::memory_order_relaxed); + independent.aba.store(0, std::memory_order_relaxed); +#else + ptr.store(x, std::memory_order_relaxed); +#endif + } + + T* peek() + { + return independent.ptr.load(std::memory_order_relaxed); + } + + Cmp read() + { + return +#ifdef PLATFORM_IS_X86 + Cmp{independent.ptr.load(std::memory_order_relaxed), + independent.aba.load(std::memory_order_relaxed)}; +#else + ptr.load(std::memory_order_relaxed); +#endif + } + + static T* load(Cmp& from) + { +#ifdef PLATFORM_IS_X86 + return from.ptr; +#else + return from; +#endif + } + + bool compare_exchange(Cmp& expect, T* value) + { +#ifdef PLATFORM_IS_X86 +# if defined(_MSC_VER) && defined(PLATFORM_BITS_64) + return _InterlockedCompareExchange128( + (volatile __int64*)&linked, + expect.aba + 1, + (__int64)value, + (__int64*)&expect); +# else +# if defined(__GNUC__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) +#error You must compile with -mcx16 to enable 16-bit atomic compare and swap. +# endif + Cmp xchg{value, expect.aba + 1}; + + return linked.compare_exchange_weak( + expect, xchg, std::memory_order_relaxed, std::memory_order_relaxed); +# endif +#else + return ptr.compare_exchange_weak( + expect, value, std::memory_order_relaxed, std::memory_order_relaxed); +#endif + } + }; +} diff --git a/src/ds/bits.h b/src/ds/bits.h new file mode 100644 index 0000000..1ad466d --- /dev/null +++ b/src/ds/bits.h @@ -0,0 +1,466 @@ +#pragma once + +#include + +#ifdef _MSC_VER +# include +# include +# define ALWAYSINLINE __forceinline +# define NOINLINE __declspec(noinline) +# define HEADER_GLOBAL __declspec(selectany) +#else +# include +# include +# define ALWAYSINLINE __attribute__((always_inline)) +# define NOINLINE __attribute__((noinline)) +# define HEADER_GLOBAL __attribute__((selectany)) +#endif + +#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \ + defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) +# define PLATFORM_IS_X86 +# if defined(__linux__) && !defined(OPEN_ENCLAVE) +# include +# endif +# if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) +# define PLATFORM_BITS_64 +# else +# define PLATFORM_BITS_32 +# endif +#endif + +#if defined(_MSC_VER) && defined(PLATFORM_BITS_32) +# include +#endif + +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +#define UNUSED(x) ((void)x) + +// #define USE_LZCNT + +#include +#include +#include +#include + +#ifdef pause +# undef pause +#endif + +namespace snmalloc +{ + // Used to enable trivial constructors for + // class that zero init is sufficient. + // Supplying PreZeroed means the memory is pre-zeroed i.e. a global section + // RequiresInit is if the class needs to zero its fields. + enum Construction + { + PreZeroed, + RequiresInit + }; + + namespace bits + { + static constexpr size_t BITS = sizeof(size_t) * 8; + + static constexpr bool is64() + { + return BITS == 64; + } + + static constexpr size_t ADDRESS_BITS = is64() ? 48 : 32; + + inline void pause() + { +#if defined(PLATFORM_IS_X86) + _mm_pause(); +#else +# warning "Missing pause intrinsic" +#endif + } + + inline uint64_t tick() + { +#if defined(PLATFORM_IS_X86) +# if defined(_MSC_VER) + return __rdtsc(); +# elif defined(__clang__) + return __builtin_readcyclecounter(); +# else + return __builtin_ia32_rdtsc(); +# endif +#else +# error Define CPU tick for this platform +#endif + } + + inline uint64_t tickp() + { +#if defined(PLATFORM_IS_X86) +# if defined(_MSC_VER) + unsigned int aux; + return __rdtscp(&aux); +# else + unsigned aux; + return __builtin_ia32_rdtscp(&aux); +# endif +#else +# error Define CPU tick for this platform +#endif + } + + inline void halt_out_of_order() + { +#if defined(PLATFORM_IS_X86) +# if defined(_MSC_VER) + int cpu_info[4]; + __cpuid(cpu_info, 0); +# else + unsigned int eax, ebx, ecx, edx; + __get_cpuid(0, &eax, &ebx, &ecx, &edx); +# endif +#else +# error Define CPU benchmark start time for this platform +#endif + } + + inline uint64_t benchmark_time_start() + { + halt_out_of_order(); + return tick(); + } + + inline uint64_t benchmark_time_end() + { + uint64_t t = tickp(); + halt_out_of_order(); + return t; + } + + inline size_t clz(size_t x) + { +#if defined(_MSC_VER) +# ifdef USE_LZCNT +# ifdef PLATFORM_BITS_64 + return __lzcnt64(x); +# else + return __lzcnt((uint32_t)x); +# endif +# else + unsigned long index; + +# ifdef PLATFORM_BITS_64 + _BitScanReverse64(&index, x); +# else + _BitScanReverse(&index, (unsigned long)x); +# endif + + return BITS - index - 1; +# endif +#else + return (size_t)__builtin_clzl(x); +#endif + } + + inline constexpr size_t rotr_const(size_t x, size_t n) + { + size_t nn = n & (BITS - 1); + return (x >> nn) | (x << (((size_t) - (int)nn) & (BITS - 1))); + } + + inline constexpr size_t rotl_const(size_t x, size_t n) + { + size_t nn = n & (BITS - 1); + return (x << nn) | (x >> (((size_t) - (int)nn) & (BITS - 1))); + } + + inline size_t rotr(size_t x, size_t n) + { +#if defined(_MSC_VER) +# ifdef PLATFORM_BITS_64 + return _rotr64(x, (int)n); +# else + return _rotr((uint32_t)x, (int)n); +# endif +#else + return rotr_const(x, n); +#endif + } + + inline size_t rotl(size_t x, size_t n) + { +#if defined(_MSC_VER) +# ifdef PLATFORM_BITS_64 + return _rotl64(x, (int)n); +# else + return _rotl((uint32_t)x, (int)n); +# endif +#else + return rotl_const(x, n); +#endif + } + + constexpr size_t clz_const(size_t x) + { + size_t n = 0; + + for (int i = BITS - 1; i >= 0; i--) + { + size_t mask = (size_t)1 << i; + + if ((x & mask) == mask) + return n; + + n++; + } + + return n; + } + + inline size_t ctz(size_t x) + { +#if defined(_MSC_VER) +# ifdef PLATFORM_BITS_64 + return _tzcnt_u64(x); +# else + return _tzcnt_u32((uint32_t)x); +# endif +#else + return (size_t)__builtin_ctzl(x); +#endif + } + + constexpr size_t ctz_const(size_t x) + { + size_t n = 0; + + for (size_t i = 0; i < BITS; i++) + { + size_t mask = (size_t)1 << i; + + if ((x & mask) == mask) + return n; + + n++; + } + + return n; + } + + inline size_t umul(size_t x, size_t y, bool& overflow) + { +#if __has_builtin(__builtin_mul_overflow) + size_t prod; + overflow = __builtin_mul_overflow(x, y, &prod); + return prod; +#elif defined(_MSC_VER) +# if defined(PLATFORM_BITS_64) + size_t high_prod; + size_t prod = _umul128(x, y, &high_prod); + overflow = high_prod != 0; + return prod; +# else + size_t prod; + overflow = S_OK == UIntMult(x, y, &prod); + return prod; +# endif +#else + size_t prod = x * y; + return y && (x > ((size_t)-1 / y)); +#endif + } + + inline size_t next_pow2(size_t x) + { + // Correct for numbers [0..MAX_SIZE >> 1). + // Returns 1 for x > (MAX_SIZE >> 1). + if (x <= 2) + return x; + + return (size_t)1 << (BITS - clz(x - 1)); + } + + inline size_t next_pow2_bits(size_t x) + { + // Correct for numbers [1..MAX_SIZE]. + // Returns 64 for 0. Approximately 2 cycles. + return BITS - clz(x - 1); + } + + constexpr size_t next_pow2_const(size_t x) + { + if (x <= 2) + return x; + + return (size_t)1 << (BITS - clz_const(x - 1)); + } + + constexpr size_t next_pow2_bits_const(size_t x) + { + return BITS - clz_const(x - 1); + } + + inline static size_t hash(void* p) + { + size_t x = (size_t)p; + + if (is64()) + { + x = ~x + (x << 21); + x = x ^ (x >> 24); + x = (x + (x << 3)) + (x << 8); + x = x ^ (x >> 14); + x = (x + (x << 2)) + (x << 4); + x = x ^ (x >> 28); + x = x + (x << 31); + } + else + { + x = ~x + (x << 15); + x = x ^ (x >> 12); + x = x + (x << 2); + x = x ^ (x >> 4); + x = (x + (x << 3)) + (x << 11); + x = x ^ (x >> 16); + } + + return x; + } + + static inline size_t align_down(size_t value, size_t alignment) + { + assert(next_pow2(alignment) == alignment); + + size_t align_1 = alignment - 1; + value &= ~align_1; + return value; + } + + static inline size_t align_up(size_t value, size_t alignment) + { + assert(next_pow2(alignment) == alignment); + + size_t align_1 = alignment - 1; + value += align_1; + value &= ~align_1; + return value; + } + + template + static inline bool is_aligned_block(void* p, size_t size) + { + assert(next_pow2(alignment) == alignment); + + return (((size_t)p | size) & (alignment - 1)) == 0; + } + + template + constexpr T inc_mod(T v, T mod) + { + static_assert( + std::is_integral::value, "inc_mod can only be used on integers"); + + using S = std::make_signed_t; + constexpr S shift = (sizeof(S) * 8) - 1; + + S a = (S)(v + 1); + S b = (S)(mod - a - 1); + return a & ~(b >> shift); + } + + /************************************************ + * + * Map large range of strictly positive integers + * into an exponent and mantissa pair. + * + * The reverse mapping is given as: + * + * e | m | value + * --------------------------------- + * 0 | x1 ... xm | 0..00 x1 .. xm + * 1 | x1 ... xm | 0..01 x1 .. xm + * 2 | x1 ... xm | 0..1 x1 .. xm 0 + * 3 | x1 ... xm | 0.1 x1 .. xm 00 + * + * The forward mapping maps a value to the + * smallest exponent and mantissa with a + * reverse mapping not less than the value. + * + * Does not work for value=0. + ***********************************************/ + template + static size_t to_exp_mant(size_t value) + { + value += ((size_t)1 << (LOW_BITS)) - 1; + value >>= LOW_BITS; + + if (MANTISSA_BITS > 0) + { + size_t LEADING_BIT = ((size_t)1 << MANTISSA_BITS) >> 1; + size_t MANTISSA_MASK = ((size_t)1 << MANTISSA_BITS) - 1; + + value = value - 1; + + size_t e = (bits::BITS - clz(value | LEADING_BIT)) - MANTISSA_BITS; + size_t shift_e = (e == 0) ? 0 : e - 1; + size_t m = (value >> shift_e) & MANTISSA_MASK; + + return (e << MANTISSA_BITS) + m; + } + else + { + return bits::next_pow2_bits(value); + } + } + + template + constexpr static size_t to_exp_mant_const(size_t value) + { + value += ((size_t)1 << LOW_BITS) - 1; + value >>= LOW_BITS; + + if (MANTISSA_BITS > 0) + { + size_t LEADING_BIT = (size_t)1 << (MANTISSA_BITS - 1); + size_t MANTISSA_MASK = ((size_t)1 << MANTISSA_BITS) - 1; + + value = value - 1; + + size_t e = + (bits::BITS - clz_const(value | LEADING_BIT)) - MANTISSA_BITS; + size_t shift_e = (e == 0) ? 0 : e - 1; + size_t m = (value >> shift_e) & MANTISSA_MASK; + + return (e << MANTISSA_BITS) + m; + } + else + { + return bits::next_pow2_bits_const(value); + } + } + + template + constexpr static size_t from_exp_mant(size_t m_e) + { + if (MANTISSA_BITS > 0) + { + size_t MANTISSA_MASK = ((size_t)1 << MANTISSA_BITS) - 1; + size_t m = m_e & MANTISSA_MASK; + size_t e = m_e >> MANTISSA_BITS; + size_t b = e == 0 ? 0 : 1; + size_t shifted_e = e - b; + size_t extended_m = (m + ((size_t)b << MANTISSA_BITS)) + 1; + return extended_m << (shifted_e + LOW_BITS); + } + else + { + return (size_t)1 << (m_e + LOW_BITS); + } + } + } +} diff --git a/src/ds/csv.h b/src/ds/csv.h new file mode 100644 index 0000000..9c7f8f7 --- /dev/null +++ b/src/ds/csv.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include + +namespace snmalloc +{ + class CSVStream + { + private: + std::ostream* out; + bool first = true; + + public: + class Endl + {}; + + Endl endl; + + CSVStream(std::ostream* o) : out(o) {} + + void preprint() + { + if (!first) + { + *out << ", "; + } + else + { + first = false; + } + } + + CSVStream& operator<<(const std::string& str) + { + preprint(); + *out << str; + return *this; + } + + CSVStream& operator<<(uint64_t u) + { + preprint(); + *out << u; + return *this; + } + + CSVStream& operator<<(Endl) + { + *out << std::endl; + first = true; + return *this; + } + }; +} \ No newline at end of file diff --git a/src/ds/dllist.h b/src/ds/dllist.h new file mode 100644 index 0000000..22a1945 --- /dev/null +++ b/src/ds/dllist.h @@ -0,0 +1,120 @@ +#pragma once + +#include +#include +#include + +namespace snmalloc +{ + template + class DLList + { + private: + static_assert( + std::is_sameprev), T*>::value, "T->prev must be a T*"); + static_assert( + std::is_samenext), T*>::value, "T->next must be a T*"); + + T* head = (T*)terminator; + + public: + T* get_head() + { + return head; + } + + T* pop() + { + T* item = head; + + if (item != (T*)terminator) + remove(item); + + return item; + } + + void insert(T* item) + { +#ifndef NDEBUG + debug_check_not_contains(item); +#endif + + item->next = head; + item->prev = (T*)terminator; + + if (head != (T*)terminator) + head->prev = item; + + head = item; +#ifndef NDEBUG + debug_check(); +#endif + } + + void remove(T* item) + { +#ifndef NDEBUG + debug_check_contains(item); +#endif + + if (item->next != (T*)terminator) + item->next->prev = item->prev; + + if (item->prev != (T*)terminator) + item->prev->next = item->next; + else + head = item->next; + +#ifndef NDEBUG + debug_check(); +#endif + } + + void debug_check_contains(T* item) + { +#ifndef NDEBUG + debug_check(); + T* curr = head; + + while (curr != item) + { + assert(curr != (T*)terminator); + curr = curr->next; + } +#else + UNUSED(item); +#endif + } + + void debug_check_not_contains(T* item) + { +#ifndef NDEBUG + debug_check(); + T* curr = head; + + while (curr != (T*)terminator) + { + assert(curr != item); + curr = curr->next; + } +#else + UNUSED(item); +#endif + } + + void debug_check() + { +#ifndef NDEBUG + T* item = head; + T* prev = (T*)terminator; + + while (item != (T*)terminator) + { + assert(item->prev == prev); + prev = item; + item = item->next; + } +#endif + } + }; +} diff --git a/src/ds/flaglock.h b/src/ds/flaglock.h new file mode 100644 index 0000000..8ae69a9 --- /dev/null +++ b/src/ds/flaglock.h @@ -0,0 +1,24 @@ +#pragma once + +#include "bits.h" + +namespace snmalloc +{ + class FlagLock + { + private: + std::atomic_flag& lock; + + public: + FlagLock(std::atomic_flag& lock) : lock(lock) + { + while (lock.test_and_set(std::memory_order_acquire)) + bits::pause(); + } + + ~FlagLock() + { + lock.clear(std::memory_order_release); + } + }; +} diff --git a/src/ds/helpers.h b/src/ds/helpers.h new file mode 100644 index 0000000..fc4461c --- /dev/null +++ b/src/ds/helpers.h @@ -0,0 +1,33 @@ +#pragma once + +#include "flaglock.h" + +namespace snmalloc +{ + /* + * In some use cases we need to run before any of the C++ runtime has been + * initialised. This singleton class is design to not depend on the runtime. + */ + template + class Singleton + { + public: + inline static Object& get() + { + static std::atomic_flag flag; + static std::atomic initialised; + static Object obj; + + if (!initialised.load(std::memory_order_acquire)) + { + FlagLock lock(flag); + if (!initialised) + { + obj = init(); + initialised.store(true, std::memory_order_release); + } + } + return obj; + } + }; +} diff --git a/src/ds/mpmcstack.h b/src/ds/mpmcstack.h new file mode 100644 index 0000000..787e721 --- /dev/null +++ b/src/ds/mpmcstack.h @@ -0,0 +1,75 @@ +#pragma once + +#include "aba.h" + +namespace snmalloc +{ + template + class MPMCStack + { + using ABA = ABA; + + private: + static_assert( + std::is_samenext), std::atomic>::value, + "T->next must be a std::atomic"); + + ABA stack; + + public: + void push(T* item) + { + return push(item, item); + } + + void push(T* first, T* last) + { + // Pushes an item on the stack. + auto cmp = stack.read(); + + do + { + T* top = ABA::load(cmp); + last->next.store(top, std::memory_order_release); + } while (!stack.compare_exchange(cmp, first)); + } + + T* pop() + { + // Returns the next item. If the returned value is decommitted, it is + // possible for the read of top->next to segfault. + auto cmp = stack.read(); + T* top; + T* next; + + do + { + top = ABA::load(cmp); + + if (top == nullptr) + break; + + next = top->next.load(std::memory_order_acquire); + } while (!stack.compare_exchange(cmp, next)); + + return top; + } + + T* pop_all() + { + // Returns all items as a linked list, leaving an empty stack. + auto cmp = stack.read(); + T* top; + + do + { + top = ABA::load(cmp); + + if (top == nullptr) + break; + } while (!stack.compare_exchange(cmp, nullptr)); + + return top; + } + }; +} diff --git a/src/ds/mpscq.h b/src/ds/mpscq.h new file mode 100644 index 0000000..671cdf3 --- /dev/null +++ b/src/ds/mpscq.h @@ -0,0 +1,99 @@ +#pragma once + +#include "bits.h" + +#include +#include + +namespace snmalloc +{ + template + class MPSCQ + { + private: + static_assert( + std::is_samenext), std::atomic>::value, + "T->next must be a std::atomic"); + + std::atomic head; + T* tail; + + public: + void invariant() + { +#ifndef NDEBUG + assert(head != nullptr); + assert(tail != nullptr); +#endif + } + + void init(T* stub) + { + stub->next.store(nullptr, std::memory_order_relaxed); + tail = stub; + head.store(stub, std::memory_order_relaxed); + invariant(); + } + + T* destroy() + { + T* tl = tail; + head.store(nullptr, std::memory_order_relaxed); + tail = nullptr; + return tl; + } + + T* get_head() + { + return head.load(std::memory_order_relaxed); + } + + inline void push(T* item) + { + push(item, item); + } + + inline bool is_empty() + { + T* hd = head.load(std::memory_order_relaxed); + + return hd == tail; + } + + void push(T* first, T* last) + { + // Pushes a list of messages to the queue. Each message from first to + // last should be linked together through their next pointers. + invariant(); + last->next.store(nullptr, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_release); + T* prev = head.exchange(last, std::memory_order_relaxed); + prev->next.store(first, std::memory_order_relaxed); + } + + std::pair pop() + { + // Returns the next message and the tail message. If the next message + // is not null, the tail message should be freed by the caller. + invariant(); + T* tl = tail; + T* next = tl->next.load(std::memory_order_relaxed); + + if (next != nullptr) + { + tail = next; + + assert(tail); + std::atomic_thread_fence(std::memory_order_acquire); + } + + invariant(); + return std::make_pair(next, tl); + } + + T* peek() + { + return tail->next.load(std::memory_order_relaxed); + } + }; +} diff --git a/src/mem/alloc.h b/src/mem/alloc.h new file mode 100644 index 0000000..598dc7b --- /dev/null +++ b/src/mem/alloc.h @@ -0,0 +1,1091 @@ +#pragma once + +#if !defined(NDEBUG) && !defined(OPEN_ENCLAVE) && !defined(FreeBSD_KERNEL) && \ + !defined(USE_SNMALLOC_STATS) +# define USE_SNMALLOC_STATS +#endif + +#ifdef _MSC_VER +# define ALLOCATOR __declspec(allocator) +#else +# define ALLOCATOR +#endif + +#include "../test/histogram.h" +#include "allocstats.h" +#include "largealloc.h" +#include "mediumslab.h" +#include "pagemap.h" +#include "remoteallocator.h" +#include "sizeclasstable.h" +#include "slab.h" +#include "typeallocated.h" + +#include + +namespace snmalloc +{ + enum Boundary + { + Start, + End + }; + + enum PageMapSuperslabKind + { + PMNotOurs = 0, + PMSuperslab = 1, + PMMediumslab = 2 + }; + + using SuperslabPagemap = Pagemap; + + HEADER_GLOBAL SuperslabPagemap global_pagemap; + /** + * Class that defines an interface to the pagemap. This is provided to + * `Allocator` as a template argument and so can be replaced by a compatible + * implementation (for example, to move pagemap updates to a different + * protection domain). + */ + struct SuperslabMap + { + /** + * Get the pagemap entry corresponding to a specific address. + */ + uint8_t get(void* p) + { + return global_pagemap.get(p); + } + /** + * Set a pagemap entry indicating that there is a superslab at the + * specified index. + */ + void set_slab(Superslab* slab) + { + set(slab, (size_t)PMSuperslab); + } + /** + * Add a pagemap entry indicating that a medium slab has been allocated. + */ + void set_slab(Mediumslab* slab) + { + set(slab, (size_t)PMMediumslab); + } + /** + * Remove an entry from the pagemap corresponding to a superslab. + */ + void clear_slab(Superslab* slab) + { + assert(get(slab) == PMSuperslab); + set(slab, (size_t)PMNotOurs); + } + /** + * Remove an entry corresponding to a medium slab. + */ + void clear_slab(Mediumslab* slab) + { + assert(get(slab) == PMMediumslab); + set(slab, (size_t)PMNotOurs); + } + /** + * Update the pagemap to reflect a large allocation, of `size` bytes from + * address `p`. + */ + void set_large_size(void* p, size_t size) + { + size_t size_bits = bits::next_pow2_bits(size); + set(p, (uint8_t)size_bits); + // Set redirect slide + uintptr_t ss = (uintptr_t)((size_t)p + SUPERSLAB_SIZE); + for (size_t i = 0; i < size_bits - SUPERSLAB_BITS; i++) + { + size_t run = 1ULL << i; + global_pagemap.set_range( + (void*)ss, (uint8_t)(64 + i + SUPERSLAB_BITS), run); + ss = (uintptr_t)ss + SUPERSLAB_SIZE * run; + } + global_pagemap.set(p, (uint8_t)size_bits); + } + /** + * Update the pagemap to remove a large allocation, of `size` bytes from + * address `p`. + */ + void clear_large_size(void* p, size_t size) + { + size_t rounded_size = bits::next_pow2(size); + assert(get(p) == bits::next_pow2_bits(size)); + auto count = rounded_size >> SUPERSLAB_BITS; + global_pagemap.set_range((void*)p, PMNotOurs, count); + } + + private: + /** + * Helper function to set a pagemap entry. This is not part of the public + * interface and exists to make it easy to reuse the code in the public + * methods in other pagemap adaptors. + */ + void set(void* p, uint8_t x) + { + global_pagemap.set(p, x); + } + }; + + static_assert( + SUPERSLAB_SIZE == Pagemap::GRANULARITY, + "The superslab size should be the same as the pagemap granularity"); + +#ifndef SNMALLOC_DEFAULT_PAGEMAP +# define SNMALLOC_DEFAULT_PAGEMAP snmalloc::SuperslabMap +#endif + + /** + * Allocator. This class is parameterised on three template parameters. The + * `MemoryProvider` defines the source of memory for this allocator. + * Allocators try to reuse address space by allocating from existing slabs or + * reusing freed large allocations. When they need to allocate a new chunk + * of memory they request space from the `MemoryProvider`. + * + * The `PageMap` parameter provides the adaptor to the pagemap. This is used + * to associate metadata with large (16MiB, by default) regions, allowing an + * allocator to find the allocator responsible for that region. + * + * The final template parameter, `IsQueueInline`, defines whether the + * message queue for this allocator should be stored as a field of the + * allocator (`true`) or provided externally, allowing it to be anywhere else + * in the address space (`false`). + */ + template< + class MemoryProvider = GlobalVirtual, + class PageMap = SNMALLOC_DEFAULT_PAGEMAP, + bool IsQueueInline = true> + class Allocator + : public TypeAllocated> + { + LargeAlloc large_allocator; + PageMap page_map; + + public: + Stats& stats() + { + return large_allocator.stats; + } + + template + friend class AllocPool; + + template< + size_t size, + ZeroMem zero_mem = NoZero, + AllowReserve allow_reserve = YesReserve> + ALLOCATOR void* alloc() + { + static_assert(size != 0, "Size must not be zero."); +#ifdef USE_MALLOC + static_assert( + allow_reserve == YesReserve, + "When passing to malloc, cannot require NoResereve"); + if constexpr (zero_mem == NoZero) + return malloc(size); + else + return calloc(1, size); +#else + constexpr uint8_t sizeclass = size_to_sizeclass_const(size); + + stats().alloc_request(size); + + handle_message_queue(); + + // Allocate memory of a statically known size. + if constexpr (sizeclass < NUM_SMALL_CLASSES) + { + constexpr size_t rsize = sizeclass_to_size(sizeclass); + return small_alloc(sizeclass, rsize); + } + else if constexpr (sizeclass < NUM_SIZECLASSES) + { + constexpr size_t rsize = sizeclass_to_size(sizeclass); + return medium_alloc(sizeclass, rsize, size); + } + else + { + return large_alloc(size); + } +#endif + } + + template + ALLOCATOR void* alloc(size_t size) + { +#ifdef USE_MALLOC + static_assert( + allow_reserve == YesReserve, + "When passing to malloc, cannot require NoResereve"); + if constexpr (zero_mem == NoZero) + return malloc(size); + else + return calloc(1, size); +#else + stats().alloc_request(size); + + handle_message_queue(); + + uint8_t sizeclass = size_to_sizeclass(size); + + // Allocate memory of a dynamically known size. + if (sizeclass < NUM_SMALL_CLASSES) + { + // Allocations smaller than the slab size are more likely. Improve + // branch prediction by placing this case first. + size_t rsize = sizeclass_to_size(sizeclass); + return small_alloc(sizeclass, rsize); + } + else if (sizeclass < NUM_SIZECLASSES) + { + size_t rsize = sizeclass_to_size(sizeclass); + return medium_alloc(sizeclass, rsize, size); + } + else + { + return large_alloc(size); + } +#endif + } + + template + void dealloc(void* p) + { +#ifdef USE_MALLOC + UNUSED(size); + return free(p); +#else + + constexpr uint8_t sizeclass = size_to_sizeclass_const(size); + + handle_message_queue(); + + // Free memory of a statically known size. Must be called with an + // external pointer. + if (sizeclass < NUM_SMALL_CLASSES) + { + Superslab* super = Superslab::get(p); + RemoteAllocator* target = super->get_allocator(); + + if (target == public_state()) + small_dealloc(super, p, sizeclass); + else + remote_dealloc(target, p, sizeclass); + } + else if (sizeclass < NUM_SIZECLASSES) + { + Mediumslab* slab = Mediumslab::get(p); + RemoteAllocator* target = slab->get_allocator(); + + if (target == public_state()) + medium_dealloc(slab, p, sizeclass); + else + remote_dealloc(target, p, sizeclass); + } + else + { + large_dealloc(p, size); + } +#endif + } + + void dealloc(void* p, size_t size) + { +#ifdef USE_MALLOC + UNUSED(size); + return free(p); +#else + handle_message_queue(); + + // Free memory of a dynamically known size. Must be called with an + // external pointer. + uint8_t sizeclass = size_to_sizeclass(size); + + if (sizeclass < NUM_SMALL_CLASSES) + { + Superslab* super = Superslab::get(p); + RemoteAllocator* target = super->get_allocator(); + + if (target == public_state()) + small_dealloc(super, p, sizeclass); + else + remote_dealloc(target, p, sizeclass); + } + else if (sizeclass < NUM_SIZECLASSES) + { + Mediumslab* slab = Mediumslab::get(p); + RemoteAllocator* target = slab->get_allocator(); + + if (target == public_state()) + medium_dealloc(slab, p, sizeclass); + else + remote_dealloc(target, p, sizeclass); + } + else + { + large_dealloc(p, size); + } +#endif + } + + void dealloc(void* p) + { +#ifdef USE_MALLOC + return free(p); +#else + handle_message_queue(); + + // Free memory of an unknown size. Must be called with an external + // pointer. + uint8_t size = pagemap().get(p); + + if (size == 0) + { + error("Not allocated by this allocator"); + } + + Superslab* super = Superslab::get(p); + + if (size == PMSuperslab) + { + RemoteAllocator* target = super->get_allocator(); + Slab* slab = Slab::get(p); + Metaslab* meta = super->get_meta(slab); + + // Reading a remote sizeclass won't fail, since the other allocator + // can't reuse the slab, as we have not yet deallocated this + // pointer. + uint8_t sizeclass = meta->sizeclass; + + if (super->get_allocator() == public_state()) + small_dealloc(super, p, sizeclass); + else + remote_dealloc(target, p, sizeclass); + return; + } + else if (size == PMMediumslab) + { + Mediumslab* slab = (Mediumslab*)super; + RemoteAllocator* target = slab->get_allocator(); + + // Reading a remote sizeclass won't fail, since the other allocator + // can't reuse the slab, as we have no yet deallocated this pointer. + uint8_t sizeclass = slab->get_sizeclass(); + + if (target == public_state()) + medium_dealloc(slab, p, sizeclass); + else + remote_dealloc(target, p, sizeclass); + return; + } + +# ifndef SNMALLOC_SAFE_CLIENT + if (size > 64 || (void*)super != p) + { + error("Not deallocating start of an object"); + } +# endif + large_dealloc(p, 1ULL << size); +#endif + } + + template + static void* external_pointer(void* p) + { +#ifdef USE_MALLOC + error("Unsupported"); + UNUSED(p); +#else + uint8_t size = global_pagemap.get(p); + + Superslab* super = Superslab::get(p); + if (size == PMSuperslab) + { + Slab* slab = Slab::get(p); + Metaslab* meta = super->get_meta(slab); + + uint8_t sc = meta->sizeclass; + size_t slab_end = (size_t)slab + SLAB_SIZE - 1; + + return external_pointer(p, sc, slab_end); + } + else if (size == PMMediumslab) + { + Mediumslab* slab = (Mediumslab*)super; + + uint8_t sc = slab->get_sizeclass(); + size_t slab_end = (size_t)slab + SUPERSLAB_SIZE - 1; + + return external_pointer(p, sc, slab_end); + } + + uintptr_t ss = (uintptr_t)super; + + while (size > 64) + { + // This is a large alloc redirect. + ss = ss - (1ULL << (size - 64)); + size = global_pagemap.get((void*)ss); + } + + if (size == 0) + { + if (location == End) + // We don't know the End, so return MAX_PTR + return (void*)-1; + else + // We don't know the Start, so return MIN_PTR + return 0; + } + + // This is a large alloc, mask off to the slab size. + if (location == Start) + return (void*)ss; + else + return (void*)((size_t)ss + (1ULL << size) - 1ULL); +#endif + } + + static size_t alloc_size(void* p) + { + // This must be called on an external pointer. + size_t size = global_pagemap.get(p); + + if (size == 0) + { + error("Not allocated by this allocator"); + } + else if (size == PMSuperslab) + { + Superslab* super = Superslab::get(p); + + // Reading a remote sizeclass won't fail, since the other allocator + // can't reuse the slab, as we have no yet deallocated this pointer. + Slab* slab = Slab::get(p); + Metaslab* meta = super->get_meta(slab); + + return sizeclass_to_size(meta->sizeclass); + } + else if (size == PMMediumslab) + { + Superslab* super = Superslab::get(p); + // Reading a remote sizeclass won't fail, since the other allocator + // can't reuse the slab, as we have no yet deallocated this pointer. + Mediumslab* slab = (Mediumslab*)super; + + return sizeclass_to_size(slab->get_sizeclass()); + } + + return 1ULL << size; + } + + size_t get_id() + { + return id(); + } + + private: + using alloc_id_t = typename Remote::alloc_id_t; + + struct RemoteList + { + Remote head; + Remote* last; + + RemoteList() + { + clear(); + } + + void clear() + { + last = &head; + } + + bool empty() + { + return last == &head; + } + }; + + struct RemoteCache + { + size_t size = 0; + RemoteList list[REMOTE_SLOTS]; + + void dealloc(alloc_id_t target_id, void* p, uint8_t sizeclass) + { + this->size += sizeclass_to_size(sizeclass); + + Remote* r = (Remote*)p; + r->set_sizeclass(sizeclass); + assert(r->sizeclass() == sizeclass); + r->set_target_id(target_id); + assert(r->sizeclass() == sizeclass); + assert(r->target_id() == target_id); + + RemoteList* l = &list[target_id & REMOTE_MASK]; + l->last->non_atomic_next = r; + l->last = r; + } + + void post(alloc_id_t id) + { + // When the cache gets big, post lists to their target allocators. + size = 0; + + size_t shift = 0; + + while (true) + { + auto my_slot = (id >> shift) & REMOTE_MASK; + + for (size_t i = 0; i < REMOTE_SLOTS; i++) + { + if (i == my_slot) + continue; + + RemoteList* l = &list[i]; + Remote* first = l->head.non_atomic_next; + + if (!l->empty()) + { + // Send all slots to the target at the head of the list. + Superslab* super = Superslab::get(first); + super->get_allocator()->message_queue.push(first, l->last); + l->clear(); + } + } + + RemoteList* resend = &list[my_slot]; + if (resend->empty()) + break; + + // Entries could map back onto the "resend" list, + // so take copy of the head, mark the last element, + // and clear the original list. + Remote* r = resend->head.non_atomic_next; + resend->last->non_atomic_next = nullptr; + resend->clear(); + + shift += REMOTE_SLOT_BITS; + + while (r != nullptr) + { + // Use the next N bits to spread out remote deallocs in our own + // slot. + size_t slot = (r->target_id() >> shift) & REMOTE_MASK; + RemoteList* l = &list[slot]; + l->last->non_atomic_next = r; + l->last = r; + + r = r->non_atomic_next; + } + } + } + }; + + static_assert( + sizeof(Remote) <= MIN_ALLOC_SIZE, + "Need to be able to cast any small alloc to Remote"); + + SlabList small_classes[NUM_SMALL_CLASSES]; + DLList medium_classes[NUM_MEDIUM_CLASSES]; + + DLList super_available; + DLList super_only_short_available; + + RemoteCache remote; + Remote stub; + + std::conditional_t + remote_alloc; + + auto* public_state() + { + if constexpr (IsQueueInline) + { + return &remote_alloc; + } + else + { + return remote_alloc; + } + } + + alloc_id_t id() + { + return public_state()->id(); + } + + auto& message_queue() + { + return public_state()->message_queue; + } + + template + friend class TypeAlloc; + + public: + Allocator( + MemoryProvider& m, PageMap&& p = PageMap(), RemoteAllocator* r = nullptr) + : large_allocator(m), page_map(p) + { + if constexpr (IsQueueInline) + { + assert(r == nullptr); + (void)r; + } + else + { + remote_alloc = r; + } + + if (id() >= (alloc_id_t)-1) + error("Id should not be -1"); + + init_message_queue(); + message_queue().invariant(); + +#ifndef NDEBUG + for (uint8_t i = 0; i < NUM_SIZECLASSES; i++) + { + size_t size = sizeclass_to_size(i); + uint8_t sc1 = size_to_sizeclass(size); + uint8_t sc2 = size_to_sizeclass_const(size); + size_t size1 = sizeclass_to_size(sc1); + size_t size2 = sizeclass_to_size(sc2); + + // All medium size classes are page aligned. + if (i > NUM_SMALL_CLASSES) + { + assert(bits::is_aligned_block(nullptr, size1)); + } + + assert(sc1 == i); + assert(sc1 == sc2); + assert(size1 == size); + assert(size1 == size2); + } +#endif + } + + template + static void* external_pointer(void* p, uint8_t sizeclass, size_t end_point) + { + size_t rsize = sizeclass_to_size(sizeclass); + size_t end_point_correction = + location == End ? end_point : end_point - rsize + 1; + size_t offset_from_end = end_point - (size_t)p; + size_t end_to_end = round_by_sizeclass(rsize, offset_from_end); + return (void*)(end_point_correction - end_to_end); + } + + void init_message_queue() + { + message_queue().init(&stub); + } + + void handle_dealloc_remote(Remote* p) + { + if (p != &stub) + { + uint8_t sizeclass = p->sizeclass(); + + if (p->target_id() == id()) + { + stats().remote_receive(sizeclass); + + if (sizeclass < NUM_SMALL_CLASSES) + small_dealloc(Superslab::get(p), p, sizeclass); + else + medium_dealloc(Mediumslab::get(p), p, sizeclass); + } + else + { + // Queue for remote dealloc elsewhere. + remote.dealloc(p->target_id(), p, sizeclass); + } + } + } + + NOINLINE void handle_message_queue_inner() + { + for (size_t i = 0; i < REMOTE_BATCH; i++) + { + std::pair r = message_queue().pop(); + + if (std::get<0>(r) == nullptr) + break; + + handle_dealloc_remote(std::get<1>(r)); + } + + // Our remote queues may be larger due to forwarding remote frees. + if (remote.size < REMOTE_CACHE) + return; + + stats().remote_post(); + remote.post(id()); + } + + inline void handle_message_queue() + { + // Inline the empty check, but not necessarily the full queue handling. + if (message_queue().is_empty()) + return; + + handle_message_queue_inner(); + } + + template + Superslab* get_superslab() + { + Superslab* super = super_available.get_head(); + + if (super != nullptr) + return super; + + super = (Superslab*)large_allocator.template alloc( + 0, SUPERSLAB_SIZE); + + if ((allow_reserve == NoReserve) && (super == nullptr)) + return super; + + super->init(public_state(), large_allocator.memory_provider); + pagemap().set_slab(super); + super_available.insert(super); + return super; + } + + void reposition_superslab(Superslab* super) + { + switch (super->get_status()) + { + case Superslab::Full: + { + // Remove from the list of superslabs that have available slabs. + super_available.remove(super); + break; + } + + case Superslab::Available: + { + // Do nothing. + break; + } + + case Superslab::OnlyShortSlabAvailable: + { + // Move from the general list to the short slab only list. + super_available.remove(super); + super_only_short_available.insert(super); + break; + } + + case Superslab::Empty: + { + // Can't be empty since we just allocated. + error("Unreachable"); + break; + } + } + } + + template + Slab* alloc_slab(uint8_t sizeclass) + { + stats().sizeclass_alloc_slab(sizeclass); + if (Superslab::is_short_sizeclass(sizeclass)) + { + // Pull a short slab from the list of superslabs that have only the + // short slab available. + Superslab* super = super_only_short_available.pop(); + + if (super != nullptr) + { + Slab* slab = + super->alloc_short_slab(sizeclass, large_allocator.memory_provider); + assert(super->is_full()); + return slab; + } + + super = get_superslab(); + + if ((allow_reserve == NoReserve) && (super == nullptr)) + return nullptr; + + Slab* slab = + super->alloc_short_slab(sizeclass, large_allocator.memory_provider); + reposition_superslab(super); + return slab; + } + + Superslab* super = get_superslab(); + + if ((allow_reserve == NoReserve) && (super == nullptr)) + return nullptr; + + Slab* slab = + super->alloc_slab(sizeclass, large_allocator.memory_provider); + reposition_superslab(super); + return slab; + } + + template + void* small_alloc(uint8_t sizeclass, size_t rsize) + { + MEASURE_TIME_MARKERS( + small_alloc, + 4, + 16, + MARKERS( + zero_mem == YesZero ? "zeromem" : "nozeromem", + allow_reserve == NoReserve ? "noreserve" : "reserve")); + + stats().sizeclass_alloc(sizeclass); + + SlabList* sc = &small_classes[sizeclass]; + SlabLink* link = sc->get_head(); + Slab* slab; + + if (link != (SlabLink*)~0) + { + slab = link->get_slab(); + } + else + { + slab = alloc_slab(sizeclass); + + if ((allow_reserve == NoReserve) && (slab == nullptr)) + return nullptr; + + sc->insert(slab->get_link()); + } + + return slab->alloc(sc, rsize, large_allocator.memory_provider); + } + + void small_dealloc(Superslab* super, void* p, uint8_t sizeclass) + { + MEASURE_TIME(small_dealloc, 4, 16); + stats().sizeclass_dealloc(sizeclass); + + bool was_full = super->is_full(); + SlabList* sc = &small_classes[sizeclass]; + Slab* slab = Slab::get(p); + Superslab::Action a = + slab->dealloc(sc, super, p, large_allocator.memory_provider); + if (a == Superslab::NoSlabReturn) + return; + + stats().sizeclass_dealloc_slab(sizeclass); + + if (a == Superslab::NoStatusChange) + return; + + switch (super->get_status()) + { + case Superslab::Full: + { + error("Unreachable"); + break; + } + + case Superslab::Available: + { + if (was_full) + { + super_available.insert(super); + } + else + { + super_only_short_available.remove(super); + super_available.insert(super); + } + break; + } + + case Superslab::OnlyShortSlabAvailable: + { + super_only_short_available.insert(super); + break; + } + + case Superslab::Empty: + { + super_available.remove(super); + + if (decommit_strategy == DecommitSuper) + { + large_allocator.memory_provider.notify_not_using( + (void*)((size_t)super + OS_PAGE_SIZE), + SUPERSLAB_SIZE - OS_PAGE_SIZE); + } + + pagemap().clear_slab(super); + large_allocator.dealloc(super, 0); + stats().superslab_push(); + break; + } + } + } + + template + void* medium_alloc(uint8_t sizeclass, size_t rsize, size_t size) + { + MEASURE_TIME_MARKERS( + medium_alloc, + 4, + 16, + MARKERS( + zero_mem == YesZero ? "zeromem" : "nozeromem", + allow_reserve == NoReserve ? "noreserve" : "reserve")); + + uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES; + + DLList* sc = &medium_classes[medium_class]; + Mediumslab* slab = sc->get_head(); + void* p; + + if (slab != nullptr) + { + p = slab->alloc(size, large_allocator.memory_provider); + + if (slab->full()) + sc->pop(); + } + else + { + slab = + (Mediumslab*)large_allocator.template alloc( + 0, SUPERSLAB_SIZE); + + if ((allow_reserve == NoReserve) && (slab == nullptr)) + return nullptr; + + slab->init(public_state(), sizeclass, rsize); + pagemap().set_slab(slab); + p = slab->alloc(size, large_allocator.memory_provider); + + if (!slab->full()) + sc->insert(slab); + } + + stats().sizeclass_alloc(sizeclass); + return p; + } + + void medium_dealloc(Mediumslab* slab, void* p, uint8_t sizeclass) + { + MEASURE_TIME(medium_dealloc, 4, 16); + stats().sizeclass_dealloc(sizeclass); + bool was_full = slab->dealloc(p, large_allocator.memory_provider); + +#ifndef SNMALLOC_SAFE_CLIENT + if (!is_multiple_of_sizeclass( + sizeclass_to_size(sizeclass), + (uintptr_t)slab + SUPERSLAB_SIZE - (uintptr_t)p)) + { + error("Not deallocating start of an object"); + } +#endif + + if (slab->empty()) + { + if (!was_full) + { + uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES; + DLList* sc = &medium_classes[medium_class]; + sc->remove(slab); + } + + if (decommit_strategy == DecommitSuper) + { + large_allocator.memory_provider.notify_not_using( + (void*)((size_t)slab + OS_PAGE_SIZE), + SUPERSLAB_SIZE - OS_PAGE_SIZE); + } + + pagemap().clear_slab(slab); + large_allocator.dealloc(slab, 0); + stats().superslab_push(); + } + else if (was_full) + { + uint8_t medium_class = sizeclass - NUM_SMALL_CLASSES; + DLList* sc = &medium_classes[medium_class]; + sc->insert(slab); + } + } + + template + void* large_alloc(size_t size) + { + MEASURE_TIME_MARKERS( + large_alloc, + 4, + 16, + MARKERS( + zero_mem == YesZero ? "zeromem" : "nozeromem", + allow_reserve == NoReserve ? "noreserve" : "reserve")); + + size_t size_bits = bits::next_pow2_bits(size); + size_t large_class = size_bits - SUPERSLAB_BITS; + assert(large_class < NUM_LARGE_CLASSES); + + void* p = large_allocator.template alloc( + large_class, size); + + pagemap().set_large_size(p, size); + + stats().large_alloc(large_class); + return p; + } + + void large_dealloc(void* p, size_t size) + { + MEASURE_TIME(large_dealloc, 4, 16); + + size_t size_bits = bits::next_pow2_bits(size); + size_t rsize = (size_t)1 << size_bits; + assert(rsize >= SUPERSLAB_SIZE); + size_t large_class = size_bits - SUPERSLAB_BITS; + + pagemap().clear_large_size(p, size); + + stats().large_dealloc(large_class); + + if ((decommit_strategy != DecommitNone) || (large_class > 0)) + large_allocator.memory_provider.notify_not_using( + (void*)((size_t)p + OS_PAGE_SIZE), rsize - OS_PAGE_SIZE); + + // Initialise in order to set the correct SlabKind. + Largeslab* slab = (Largeslab*)p; + slab->init(); + large_allocator.dealloc(slab, large_class); + } + + void remote_dealloc(RemoteAllocator* target, void* p, uint8_t sizeclass) + { + MEASURE_TIME(remote_dealloc, 4, 16); + + stats().remote_free(sizeclass); + remote.dealloc(target->id(), p, sizeclass); + + if (remote.size < REMOTE_CACHE) + return; + + stats().remote_post(); + remote.post(id()); + } + + PageMap& pagemap() + { + return page_map; + } + }; +} diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h new file mode 100644 index 0000000..edbdf69 --- /dev/null +++ b/src/mem/allocconfig.h @@ -0,0 +1,121 @@ +#pragma once + +#include "../ds/bits.h" + +namespace snmalloc +{ + enum ZeroMem + { + NoZero, + YesZero + }; + + // 0 intermediate bits results in power of 2 small allocs. 1 intermediate + // bit gives additional sizeclasses at the midpoint between each power of 2. + // 2 intermediate bits gives 3 intermediate sizeclasses, etc. + static constexpr size_t INTERMEDIATE_BITS = +#ifdef USE_INTERMEDIATE_BITS + USE_INTERMEDIATE_BITS +#else + 2 +#endif + ; + + // Return remote small allocs when the local cache reaches this size. + static constexpr size_t REMOTE_CACHE = +#ifdef USE_REMOTE_CACHE + USE_REMOTE_CACHE +#else + 1 << 20 +#endif + ; + + // Handle at most this many object from the remote dealloc queue at a time. + static constexpr size_t REMOTE_BATCH = +#ifdef USE_REMOTE_BATCH + REMOTE_BATCH +#else + 64 +#endif + ; + + static constexpr size_t RESERVE_MULTIPLE = +#ifdef USE_RESERVE_MULTIPLE + USE_RESERVE_MULTIPLE +#else + bits::is64() ? 16 : 2 +#endif + ; + + enum DecommitStrategy + { + DecommitNone, + DecommitSuper, + DecommitAll + }; + + static constexpr DecommitStrategy decommit_strategy = +#ifdef USE_DECOMMIT_STRATEGY + USE_DECOMMIT_STRATEGY +#else + DecommitSuper +#endif + ; + + // The remaining values are derived, not configurable. + + // Used to isolate values on cache lines to prevent false sharing. + static constexpr size_t CACHELINE_SIZE = 64; + + // Used to keep Superslab metadata committed. + static constexpr size_t OS_PAGE_SIZE = 0x1000; + static constexpr size_t PAGE_ALIGNED_SIZE = OS_PAGE_SIZE << INTERMEDIATE_BITS; + // Some system headers (e.g. Linux' sys/user.h, FreeBSD's machine/param.h) + // define `PAGE_SIZE` as a macro. We don't use `PAGE_SIZE` as our variable + // name, to avoid conflicts, but if we do see a macro definition then check + // that our value matches the platform's expected value. +#ifdef PAGE_SIZE + static_assert( + PAGE_SIZE == OS_PAGE_SIZE, + "Page size from system header does not match snmalloc config page size."); +#endif + + // Minimum allocation size is space for two pointers. + static constexpr size_t MIN_ALLOC_BITS = bits::is64() ? 4 : 3; + static constexpr size_t MIN_ALLOC_SIZE = 1 << MIN_ALLOC_BITS; + + // Slabs are 64 kb. + static constexpr size_t SLAB_BITS = 16; + static constexpr size_t SLAB_SIZE = 1 << SLAB_BITS; + static constexpr size_t SLAB_MASK = ~(SLAB_SIZE - 1); + + // Superslabs are composed of this many slabs. Slab offsets are encoded as + // a byte, so the maximum count is 256. This must be a power of two to + // allow fast masking to find a superslab start address. + static constexpr size_t SLAB_COUNT_BITS = 8; + static constexpr size_t SLAB_COUNT = 1 << SLAB_COUNT_BITS; + static constexpr size_t SUPERSLAB_SIZE = SLAB_SIZE * SLAB_COUNT; + static constexpr size_t SUPERSLAB_MASK = ~(SUPERSLAB_SIZE - 1); + static constexpr size_t SUPERSLAB_BITS = SLAB_BITS + SLAB_COUNT_BITS; + static constexpr size_t RESERVE_SIZE = SUPERSLAB_SIZE * RESERVE_MULTIPLE; + + // Number of slots for remote deallocation. + static constexpr size_t REMOTE_SLOT_BITS = 6; + static constexpr size_t REMOTE_SLOTS = 1 << REMOTE_SLOT_BITS; + static constexpr size_t REMOTE_MASK = REMOTE_SLOTS - 1; + + static_assert( + INTERMEDIATE_BITS < MIN_ALLOC_BITS, + "INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS"); + static_assert( + MIN_ALLOC_SIZE >= (sizeof(void*) * 2), + "MIN_ALLOC_SIZE must be sufficient for two pointers"); + static_assert( + SLAB_BITS == (sizeof(uint16_t) * 8), + "SLAB_BITS must be the bits in a uint16_t"); + static_assert( + SLAB_COUNT == bits::next_pow2_const(SLAB_COUNT), + "SLAB_COUNT must be a power of 2"); + static_assert( + SLAB_COUNT <= (UINT8_MAX + 1), "SLAB_COUNT must fit in a uint8_t"); +}; diff --git a/src/mem/allocslab.h b/src/mem/allocslab.h new file mode 100644 index 0000000..f1ac1dd --- /dev/null +++ b/src/mem/allocslab.h @@ -0,0 +1,19 @@ +#pragma once + +#include "../mem/baseslab.h" +#include "remoteallocator.h" + +namespace snmalloc +{ + class Allocslab : public Baseslab + { + protected: + RemoteAllocator* allocator; + + public: + RemoteAllocator* get_allocator() + { + return allocator; + } + }; +} diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h new file mode 100644 index 0000000..a2ec29f --- /dev/null +++ b/src/mem/allocstats.h @@ -0,0 +1,391 @@ +#pragma once + +#include "../ds/bits.h" + +#include + +#ifdef USE_SNMALLOC_STATS +# include "../ds/csv.h" +# include "sizeclass.h" + +# include +# include +#endif + +namespace snmalloc +{ + template + struct AllocStats + { + struct CurrentMaxPair + { + size_t current = 0; + size_t max = 0; + + void inc() + { + current++; + if (current > max) + max++; + } + + void dec() + { + current--; + } + + bool is_empty() + { + return current == 0; + } + + bool is_unused() + { + return max == 0; + } + + void add(CurrentMaxPair& that) + { + current += that.current; + max += that.max; + } +#ifdef USE_SNMALLOC_STATS + void print(CSVStream& csv, size_t multiplier = 1) + { + csv << current * multiplier << max * multiplier; + } +#endif + }; + + struct Stats + { + CurrentMaxPair count; + CurrentMaxPair slab_count; + uint64_t time = bits::tick(); + uint64_t ticks = 0; + double online_average = 0; + + bool is_empty() + { + return count.is_empty(); + } + + void add(Stats& that) + { + count.add(that.count); + slab_count.add(that.slab_count); + } + + void addToRunningAverage() + { + uint64_t now = bits::tick(); + + if (slab_count.current != 0) + { + double occupancy = (double)count.current / (double)slab_count.current; + uint64_t duration = now - time; + + if (ticks == 0) + online_average = occupancy; + else + online_average += ((occupancy - online_average) * duration) / ticks; + + ticks += duration; + } + + time = now; + } + +#ifdef USE_SNMALLOC_STATS + void + print(CSVStream& csv, size_t multiplier = 1, size_t slab_multiplier = 1) + { + // Keep in sync with header lower down + count.print(csv, multiplier); + slab_count.print(csv, slab_multiplier); + size_t average = (size_t)(online_average * multiplier); + + csv << average << (slab_multiplier - average) * slab_count.max + << csv.endl; + } +#endif + }; + +#ifdef USE_SNMALLOC_STATS + static constexpr size_t BUCKETS_BITS = 4; + static constexpr size_t BUCKETS = 1 << BUCKETS_BITS; + static constexpr size_t TOTAL_BUCKETS = + bits::to_exp_mant_const( + ((size_t)1 << (bits::ADDRESS_BITS - 1))); + + Stats sizeclass[N]; + Stats large[LARGE_N]; + + size_t remote_freed = 0; + size_t remote_posted = 0; + size_t remote_received = 0; + size_t superslab_push_count = 0; + size_t superslab_pop_count = 0; + size_t superslab_fresh_count = 0; + size_t segment_count = 0; + size_t bucketed_requests[TOTAL_BUCKETS] = {}; +#endif + + void alloc_request(size_t size) + { + UNUSED(size); + +#ifdef USE_SNMALLOC_STATS + bucketed_requests[bits::to_exp_mant(size)]++; +#endif + } + + bool is_empty() + { +#ifdef USE_SNMALLOC_STATS + for (size_t i = 0; i < N; i++) + { + if (!sizeclass[i].is_empty()) + return false; + } + + for (size_t i = 0; i < LARGE_N; i++) + { + if (!large[i].is_empty()) + return false; + } + + return (remote_freed == remote_posted); +#else + return true; +#endif + } + + void sizeclass_alloc(uint8_t sc) + { + UNUSED(sc); + +#ifdef USE_SNMALLOC_STATS + sizeclass[sc].addToRunningAverage(); + sizeclass[sc].count.inc(); +#endif + } + + void sizeclass_dealloc(uint8_t sc) + { + UNUSED(sc); + +#ifdef USE_SNMALLOC_STATS + sizeclass[sc].addToRunningAverage(); + sizeclass[sc].count.dec(); +#endif + } + + void large_alloc(size_t sc) + { + UNUSED(sc); + +#ifdef USE_SNMALLOC_STATS + large[sc].count.inc(); +#endif + } + + void sizeclass_alloc_slab(uint8_t sc) + { + UNUSED(sc); + +#ifdef USE_SNMALLOC_STATS + sizeclass[sc].addToRunningAverage(); + sizeclass[sc].slab_count.inc(); +#endif + } + + void sizeclass_dealloc_slab(uint8_t sc) + { + UNUSED(sc); + +#ifdef USE_SNMALLOC_STATS + sizeclass[sc].addToRunningAverage(); + sizeclass[sc].slab_count.dec(); +#endif + } + + void large_dealloc(size_t sc) + { + UNUSED(sc); + +#ifdef USE_SNMALLOC_STATS + large[sc].count.dec(); +#endif + } + + void segment_create() + { +#ifdef USE_SNMALLOC_STATS + segment_count++; +#endif + } + + void superslab_pop() + { +#ifdef USE_SNMALLOC_STATS + superslab_pop_count++; +#endif + } + + void superslab_push() + { +#ifdef USE_SNMALLOC_STATS + superslab_push_count++; +#endif + } + + void superslab_fresh() + { +#ifdef USE_SNMALLOC_STATS + superslab_fresh_count++; +#endif + } + + void remote_free(uint8_t sc) + { + UNUSED(sc); + +#ifdef USE_SNMALLOC_STATS + remote_freed += sizeclass_to_size(sc); +#endif + } + + void remote_post() + { +#ifdef USE_SNMALLOC_STATS + remote_posted = remote_freed; +#endif + } + + void remote_receive(uint8_t sc) + { + UNUSED(sc); + +#ifdef USE_SNMALLOC_STATS + remote_received += sizeclass_to_size(sc); +#endif + } + + void add(AllocStats& that) + { + UNUSED(that); + +#ifdef USE_SNMALLOC_STATS + for (size_t i = 0; i < N; i++) + sizeclass[i].add(that.sizeclass[i]); + + for (size_t i = 0; i < LARGE_N; i++) + large[i].add(that.large[i]); + + for (size_t i = 0; i < TOTAL_BUCKETS; i++) + bucketed_requests[i] += that.bucketed_requests[i]; + + remote_freed += that.remote_freed; + remote_posted += that.remote_posted; + remote_received += that.remote_received; + superslab_pop_count += that.superslab_pop_count; + superslab_push_count += that.superslab_push_count; + superslab_fresh_count += that.superslab_fresh_count; + segment_count += that.segment_count; +#endif + } + +#ifdef USE_SNMALLOC_STATS + template + void print(std::ostream& o, uint64_t dumpid = 0, uint64_t allocatorid = 0) + { + UNUSED(o); + UNUSED(dumpid); + UNUSED(allocatorid); + + CSVStream csv(&o); + + if (dumpid == 0) + { + // Output headers for initial dump + // Keep in sync with data dump + csv << "GlobalStats" + << "DumpID" + << "AllocatorID" + << "Remote freed" + << "Remote posted" + << "Remote received" + << "Superslab pop" + << "Superslab push" + << "Superslab fresh" + << "Segments" << csv.endl; + + csv << "BucketedStats" + << "DumpID" + << "AllocatorID" + << "Size group" + << "Size" + << "Current bytes" + << "Max bytes" + << "Current Slab bytes" + << "Max Slab bytes" + << "Average Slab Usage" + << "Average wasted space" << csv.endl; + + csv << "AllocSizes" + << "DumpID" + << "AllocatorID" + << "ClassID" + << "Low size" + << "High size" + << "Count" << csv.endl; + } + + for (uint8_t i = 0; i < N; i++) + { + if (sizeclass[i].count.is_unused()) + continue; + + sizeclass[i].addToRunningAverage(); + + csv << "BucketedStats" << dumpid << allocatorid << i + << sizeclass_to_size(i); + + sizeclass[i].print(csv, sizeclass_to_size(i), SLAB_SIZE); + } + + for (uint8_t i = 0; i < LARGE_N; i++) + { + if (large[i].count.is_unused()) + continue; + + csv << "BucketedStats" << dumpid << allocatorid << (i + N) + << large_sizeclass_to_size(i); + + large[i].print(csv, large_sizeclass_to_size(i)); + } + + size_t low = 0; + size_t high = 0; + + for (size_t i = 0; i < TOTAL_BUCKETS; i++) + { + low = high + 1; + high = bits::from_exp_mant(i); + + if (bucketed_requests[i] == 0) + continue; + + csv << "AllocSizes" << dumpid << allocatorid << i << low << high + << bucketed_requests[i] << csv.endl; + } + + csv << "GlobalStats" << dumpid << allocatorid << remote_freed + << remote_posted << remote_received << superslab_pop_count + << superslab_push_count << superslab_fresh_count << segment_count + << csv.endl; + } +#endif + }; +} diff --git a/src/mem/baseslab.h b/src/mem/baseslab.h new file mode 100644 index 0000000..ddbed4d --- /dev/null +++ b/src/mem/baseslab.h @@ -0,0 +1,27 @@ +#pragma once + +#include "../ds/mpmcstack.h" +#include "allocconfig.h" + +namespace snmalloc +{ + enum SlabKind + { + Fresh = 0, + Large, + Medium, + Super + }; + + class Baseslab + { + protected: + SlabKind kind; + + public: + SlabKind get_kind() + { + return kind; + } + }; +} diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h new file mode 100644 index 0000000..37e1522 --- /dev/null +++ b/src/mem/globalalloc.h @@ -0,0 +1,171 @@ +#pragma once + +#include "../ds/helpers.h" +#include "alloc.h" +#include "typealloc.h" + +namespace snmalloc +{ + template + class AllocPool : TypeAlloc, MemoryProvider> + { + using Alloc = Allocator; + using Parent = TypeAlloc, MemoryProvider>; + + public: + static AllocPool* make(MemoryProvider& mp) + { + static_assert( + sizeof(AllocPool) == sizeof(Parent), + "You cannot add fields to this class."); + // This cast is safe due to the static assert. + return (AllocPool*)Parent::make(mp); + } + + static AllocPool* make() noexcept + { + return make(default_memory_provider); + } + + Alloc* acquire() + { + return Parent::alloc(Parent::memory_provider); + } + + void release(Alloc* a) + { + Parent::dealloc(a); + } + + public: + void aggregate_stats(Stats& stats) + { + auto* alloc = Parent::iterate(); + + while (alloc != nullptr) + { + stats.add(alloc->stats()); + alloc = Parent::iterate(alloc); + } + } + + void print_all_stats(std::ostream& o, uint64_t dumpid = 0) + { + auto alloc = Parent::iterate(); + + while (alloc != nullptr) + { + alloc->stats().template print(o, dumpid, alloc->id()); + alloc = Parent::iterate(alloc); + } + } + + void cleanup_unused() + { +#ifndef USE_MALLOC + // Call this periodically to free and coalesce memory allocated by + // allocators that are not currently in use by any thread. + // One atomic operation to extract the stack, another to restore it. + // Handling the message queue for each stack is non-atomic. + auto* first = Parent::extract(); + auto* alloc = first; + decltype(alloc) last; + + if (alloc != nullptr) + { + while (alloc != nullptr) + { + alloc->handle_message_queue(); + last = alloc; + alloc = Parent::extract(alloc); + } + + restore(first, last); + } +#endif + } + + void debug_check_empty() + { +#ifndef USE_MALLOC + // This is a debugging function. It checks that all memory from all + // allocators has been freed. + size_t alloc_count = 0; + + auto* alloc = Parent::iterate(); + + // Count the linked allocators. + while (alloc != nullptr) + { + alloc = Parent::iterate(alloc); + alloc_count++; + } + + bool done = false; + + while (!done) + { + done = true; + alloc = Parent::iterate(); + + while (alloc != nullptr) + { + // Destroy the message queue so that it has no stub message. + Remote* p = alloc->message_queue().destroy(); + + while (p != nullptr) + { + Remote* next = p->non_atomic_next; + alloc->handle_dealloc_remote(p); + p = next; + } + + // Place the static stub message on the queue. + alloc->init_message_queue(); + + // Post all remotes, including forwarded ones. If any allocator posts, + // repeat the loop. + if (alloc->remote.size > 0) + { + alloc->stats().remote_post(); + alloc->remote.post(alloc->id()); + done = false; + } + + alloc = Parent::iterate(alloc); + } + } + + alloc = Parent::iterate(); + size_t empty_count = 0; + + while (alloc != nullptr) + { + // Check that the allocator has freed all memory. + if (alloc->stats().is_empty()) + empty_count++; + + alloc = Parent::iterate(alloc); + } + + if (alloc_count != empty_count) + error("Incorrect number of allocators"); +#endif + } + }; + + inline AllocPool*& current_alloc_pool() + { + return Singleton< + AllocPool*, + AllocPool::make>::get(); + } + + template + inline AllocPool* make_alloc_pool(MemoryProvider& mp) + { + return AllocPool::make(mp); + } + + using Alloc = Allocator; +} diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h new file mode 100644 index 0000000..9619861 --- /dev/null +++ b/src/mem/largealloc.h @@ -0,0 +1,207 @@ +#pragma once + +#include "../ds/flaglock.h" +#include "../ds/mpmcstack.h" +#include "../pal/pal.h" +#include "allocstats.h" +#include "baseslab.h" +#include "sizeclass.h" + +#include + +namespace snmalloc +{ + class Largeslab : public Baseslab + { + // This is the view of a contiguous memory area when it is being kept + // in the global size-classed caches of available contiguous memory areas. + private: + template + friend class MPMCStack; + std::atomic next; + + public: + void init() + { + kind = Large; + } + }; + + // This represents the state that the large allcoator needs to add to the + // global state of the allocator. This is currently stored in the memory + // provider, so we add this in. + template + class MemoryProviderStateMixin : public MemoryProviderState + { + std::atomic_flag lock = ATOMIC_FLAG_INIT; + size_t bump; + size_t remaining; + + std::pair reserve_block() noexcept + { + size_t size = SUPERSLAB_SIZE; + void* r = ((MemoryProviderState*)this) + ->template reserve(&size, SUPERSLAB_SIZE); + + if (size < SUPERSLAB_SIZE) + error("out of memory"); + + ((MemoryProviderState*)this) + ->template notify_using(r, OS_PAGE_SIZE); + return std::make_pair(r, size); + } + + public: + /** + * Stack of large allocations that have been returned for reuse. + */ + MPMCStack large_stack[NUM_LARGE_CLASSES]; + + /** + * Primitive allocator for structure that are required before + * the allocator can be running. + ***/ + void* alloc_chunk(size_t size) + { + // Cache line align + size = bits::align_up(size, 64); + + void* p; + { + FlagLock f(lock); + + if (remaining < size) + { + auto r_size = reserve_block(); + + bump = (size_t)r_size.first; + remaining = r_size.second; + } + + p = (void*)bump; + bump += size; + remaining -= size; + } + + auto page_start = bits::align_down((size_t)p, OS_PAGE_SIZE); + auto page_end = bits::align_up((size_t)p + size, OS_PAGE_SIZE); + + ((MemoryProviderState*)this) + ->template notify_using( + (void*)page_start, page_end - page_start); + + return p; + } + }; + + using Stats = AllocStats; + + enum AllowReserve + { + NoReserve, + YesReserve + }; + + template + class LargeAlloc + { + void* reserved_start = nullptr; + void* reserved_end = nullptr; + + public: + // This will be a zero-size structure if stats are not enabled. + Stats stats; + + MemoryProvider& memory_provider; + + LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {} + + template + bool reserve_memory(size_t need, size_t add) + { + if (((size_t)reserved_start + need) > (size_t)reserved_end) + { + if (allow_reserve == YesReserve) + { + stats.segment_create(); + reserved_start = + memory_provider.template reserve(&add, SUPERSLAB_SIZE); + reserved_end = (void*)((size_t)reserved_start + add); + reserved_start = + (void*)bits::align_up((size_t)reserved_start, SUPERSLAB_SIZE); + + if (add < need) + return false; + } + else + { + return false; + } + } + + return true; + } + + template + void* alloc(size_t large_class, size_t size) + { + size_t rsize = ((size_t)1 << SUPERSLAB_BITS) << large_class; + if (size == 0) + size = rsize; + + void* p = memory_provider.large_stack[large_class].pop(); + + if (p == nullptr) + { + assert(reserved_start <= reserved_end); + size_t add; + + if ((rsize + SUPERSLAB_SIZE) < RESERVE_SIZE) + add = RESERVE_SIZE; + else + add = rsize + SUPERSLAB_SIZE; + + if (!reserve_memory(rsize, add)) + return nullptr; + + p = (void*)reserved_start; + reserved_start = (void*)((size_t)p + rsize); + + // All memory is zeroed since it comes from reserved space. + memory_provider.template notify_using(p, size); + } + else + { + if ((decommit_strategy != DecommitNone) || (large_class > 0)) + { + // Only the first page needs to be zeroed, as this was decommitted. + if (zero_mem == YesZero) + memory_provider.template zero(p, OS_PAGE_SIZE); + + memory_provider.template notify_using( + (void*)((size_t)p + OS_PAGE_SIZE), size - OS_PAGE_SIZE); + } + else + { + // This is a superslab that has not been decommitted. + if (zero_mem == YesZero) + memory_provider.template zero(p, size); + } + } + + return p; + } + + void dealloc(void* p, size_t large_class) + { + memory_provider.large_stack[large_class].push((Largeslab*)p); + } + }; + + using GlobalVirtual = MemoryProviderStateMixin; + /** + * The memory provider that will be used if no other provider is explicitly + * passed as an argument. + */ + HEADER_GLOBAL GlobalVirtual default_memory_provider; +} \ No newline at end of file diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h new file mode 100644 index 0000000..d9e3cbf --- /dev/null +++ b/src/mem/mediumslab.h @@ -0,0 +1,130 @@ +#pragma once + +#include "../ds/dllist.h" +#include "allocconfig.h" +#include "allocslab.h" +#include "sizeclass.h" + +namespace snmalloc +{ + class Mediumslab : public Allocslab + { + // This is the view of a 16 mb area when it is being used to allocate + // medium sized classes: 64 kb to 16 mb, non-inclusive. + private: + friend DLList; + + // Keep the allocator pointer on a separate cache line. It is read by + // other threads, and does not change, so we avoid false sharing. + alignas(CACHELINE_SIZE) Mediumslab* next; + Mediumslab* prev; + + uint16_t free; + uint8_t head; + uint8_t sizeclass; + uint16_t stack[SLAB_COUNT - 1]; + + public: + static constexpr uint32_t header_size() + { + static_assert( + sizeof(Mediumslab) < OS_PAGE_SIZE, + "Mediumslab header size must be less than the page size"); + static_assert( + sizeof(Mediumslab) < SLAB_SIZE, + "Mediumslab header size must be less than the slab size"); + + // Always use a full page as the header, in order to get page sized + // alignment of individual allocations. + return OS_PAGE_SIZE; + } + + static Mediumslab* get(void* p) + { + return (Mediumslab*)((size_t)p & SUPERSLAB_MASK); + } + + void init(RemoteAllocator* alloc, uint8_t sc, size_t rsize) + { + assert(sc >= NUM_SMALL_CLASSES); + assert((sc - NUM_SMALL_CLASSES) < NUM_MEDIUM_CLASSES); + + allocator = alloc; + head = 0; + + // If this was previously a Mediumslab of the same sizeclass, don't + // initialise the allocation stack. + if ((kind != Medium) || (sizeclass != sc)) + { + sizeclass = sc; + uint16_t ssize = (uint16_t)(rsize >> 8); + kind = Medium; + free = medium_slab_free(sc); + for (uint16_t i = free; i > 0; i--) + stack[free - i] = (uint16_t)((SUPERSLAB_SIZE >> 8) - (i * ssize)); + } + else + { + assert(free == medium_slab_free(sc)); + } + } + + uint8_t get_sizeclass() + { + return sizeclass; + } + + template + void* alloc(size_t size, MemoryProvider& memory_provider) + { + assert(!full()); + + uint16_t index = stack[head++]; + void* p = (void*)((size_t)this + ((size_t)index << 8)); + free--; + + assert(bits::is_aligned_block(p, OS_PAGE_SIZE)); + size = bits::align_up(size, OS_PAGE_SIZE); + + if (decommit_strategy == DecommitAll) + memory_provider.template notify_using(p, size); + else if (zero_mem == YesZero) + memory_provider.template zero(p, size); + + return p; + } + + template + bool dealloc(void* p, MemoryProvider& memory_provider) + { + assert(head > 0); + + // Returns true if the Mediumslab was full before this deallocation. + bool was_full = full(); + free++; + stack[--head] = pointer_to_index(p); + + if (decommit_strategy == DecommitAll) + memory_provider.notify_not_using(p, sizeclass_to_size(sizeclass)); + + return was_full; + } + + bool full() + { + return free == 0; + } + + bool empty() + { + return head == 0; + } + + private: + uint16_t pointer_to_index(void* p) + { + // Get the offset from the slab for a memory location. + return (uint16_t)(((size_t)p - (size_t)this) >> 8); + } + }; +} diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h new file mode 100644 index 0000000..1d24fae --- /dev/null +++ b/src/mem/metaslab.h @@ -0,0 +1,166 @@ +#pragma once + +#include "../ds/dllist.h" +#include "sizeclass.h" + +namespace snmalloc +{ + class Slab; + + struct SlabLink + { + SlabLink* prev; + SlabLink* next; + + Slab* get_slab() + { + return (Slab*)((size_t)this & SLAB_MASK); + } + }; + + using SlabList = DLList; + + static_assert( + sizeof(SlabLink) <= MIN_ALLOC_SIZE, + "Need to be able to pack a SlabLink into any free small alloc"); + + static constexpr uint16_t SLABLINK_INDEX = + (uint16_t)(SLAB_SIZE - sizeof(SlabLink)); + + // The Metaslab represent the status of a single slab. + // This can be either a short or a standard slab. + class Metaslab + { + private: + // How many entries are used in this slab. + uint16_t used; + + public: + // Bump free list of unused entries in this sizeclass. + // If the bottom bit is 1, then this represents a bump_ptr + // of where we have allocated up to in this slab. Otherwise, + // it represents the location of the first block in the free + // list. The free list is chained through deallocated blocks. + // It either terminates with a bump ptr, or if all the space is in + // the free list, then the last block will be also referenced by + // link. + // Note that, in the case that this is the first block in the size + // class list, where all the unused memory is in the free list, + // then the last block can both be interpreted as a final bump + // pointer entry, and the first entry in the doubly linked list. + // The terminal value in the free list, and the terminal value in + // the SlabLink previous field will alias. The SlabLink uses ~0 for + // its terminal value to be a valid terminal bump ptr. + uint16_t head; + // When a slab has free space it will be on the has space list for + // that size class. We use an empty block in this slab to be the + // doubly linked node into that size class's free list. + uint16_t link; + + union + { + uint8_t sizeclass; + uint8_t next; + }; + + void add_use() + { + used++; + } + + void sub_use() + { + used--; + } + + void set_unused() + { + used = 0; + } + + bool is_unused() + { + return used == 0; + } + + bool is_full() + { + return (head & 2) != 0; + } + + void set_full() + { + assert(head == 1); + head = (uint16_t)~0; + } + + SlabLink* get_link(Slab* slab) + { + return (SlabLink*)((size_t)slab + link); + } + + bool valid_head(bool is_short) + { + size_t size = sizeclass_to_size(sizeclass); + size_t offset = get_slab_offset(sizeclass, is_short); + return ((((head & ~(size_t)1) - (offset & ~(size_t)1)) % size) == 0); + } + + void debug_slab_invariant(bool is_short, Slab* slab) + { +#if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) + size_t size = sizeclass_to_size(sizeclass); + size_t offset = get_slab_offset(sizeclass, is_short) - 1; + + size_t accounted_for = used * size + offset; + + if (is_full()) + { + // All the blocks must be used. + assert(SLAB_SIZE == accounted_for); + // There is no free list to validate + // 'link' value is not important if full. + return; + } + // Block is not full + assert(SLAB_SIZE > accounted_for); + + // Walk bump-free-list-segment accounting for unused space + uint16_t curr = head; + while ((curr & 1) != 1) + { + // Check we are looking at a correctly aligned block + assert((curr - offset) % size == 0); + // Account for free elements in free list + accounted_for += size; + assert(SLAB_SIZE >= accounted_for); + // We are not guaranteed to hit a bump ptr unless + // we are the top element on the size class, so treat as + // a list segment. + if (curr == link) + break; + // Iterate bump/free list segment + curr = *(uint16_t*)((uintptr_t)slab + curr); + } + + // Check we terminated traversal on a correctly aligned block + assert(((curr & ~1) - offset) % size == 0); + + if (curr != link) + { + // The link should be at the special end location as we + // haven't completely filled this block at any point. + assert(link == SLABLINK_INDEX); + // Account for to be bump allocated space + accounted_for += SLAB_SIZE - (curr - 1); + } + + // All space accounted for + assert(SLAB_SIZE == accounted_for); +#else + UNUSED(slab); + UNUSED(is_short); +#endif + } + }; +} diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h new file mode 100644 index 0000000..d615e10 --- /dev/null +++ b/src/mem/pagemap.h @@ -0,0 +1,227 @@ +#pragma once + +#include "../ds/bits.h" + +#include +#include + +namespace snmalloc +{ + template + class Pagemap + { + private: + static constexpr size_t PAGEMAP_BITS = 16; + static constexpr size_t PAGEMAP_SIZE = 1 << PAGEMAP_BITS; + + static constexpr size_t COVERED_BITS = + bits::ADDRESS_BITS - GRANULARITY_BITS; + static constexpr size_t POINTER_BITS = + bits::next_pow2_bits_const(sizeof(void*)); + static constexpr size_t CONTENT_BITS = + bits::next_pow2_bits_const(sizeof(T)); + + static constexpr size_t BITS_FOR_LEAF = PAGEMAP_BITS - CONTENT_BITS; + static constexpr size_t ENTRIES_PER_LEAF = 1 << BITS_FOR_LEAF; + static constexpr size_t LEAF_MASK = ENTRIES_PER_LEAF - 1; + + static constexpr size_t BITS_PER_INDEX_LEVEL = PAGEMAP_BITS - POINTER_BITS; + static constexpr size_t ENTRIES_PER_INDEX_LEVEL = 1 << BITS_PER_INDEX_LEVEL; + static constexpr size_t ENTRIES_MASK = ENTRIES_PER_INDEX_LEVEL - 1; + + static constexpr size_t INDEX_BITS = + BITS_FOR_LEAF > COVERED_BITS ? 0 : COVERED_BITS - BITS_FOR_LEAF; + + static constexpr size_t INDEX_LEVELS = INDEX_BITS / BITS_PER_INDEX_LEVEL; + static constexpr size_t TOPLEVEL_BITS = + INDEX_BITS - (INDEX_LEVELS * BITS_PER_INDEX_LEVEL); + static constexpr size_t TOPLEVEL_ENTRIES = 1 << TOPLEVEL_BITS; + static constexpr size_t TOPLEVEL_SHIFT = + (INDEX_LEVELS * BITS_PER_INDEX_LEVEL) + BITS_FOR_LEAF + GRANULARITY_BITS; + + // Value used to represent when a node is being added too + static constexpr uintptr_t LOCKED_ENTRY = 1; + + struct Leaf + { + std::atomic values[ENTRIES_PER_LEAF]; + }; + + struct PagemapEntry + { + std::atomic entries[ENTRIES_PER_INDEX_LEVEL]; + }; + + static_assert( + sizeof(PagemapEntry) == sizeof(Leaf), "Should be the same size"); + + // Init removed as not required as this is only ever a global + // cl is generating a memset of zero, which will be a problem + // in libc/ucrt bring up. On ucrt this will run after the first + // allocation. + // TODO: This is fragile that it is not being memset, and we should review + // to ensure we don't get bitten by this in the future. + std::atomic top[TOPLEVEL_ENTRIES]; // = {nullptr}; + + template + inline PagemapEntry* get_node(std::atomic* e, bool& result) + { + // The page map nodes are all allocated directly from the OS zero + // initialised with a system call. We don't need any ordered to guarantee + // to see that correctly. + PagemapEntry* value = e->load(std::memory_order_relaxed); + + if ((uintptr_t)value <= LOCKED_ENTRY) + { + if constexpr (create_addr) + { + value = nullptr; + + if (e->compare_exchange_strong( + value, (PagemapEntry*)LOCKED_ENTRY, std::memory_order_relaxed)) + { + auto& v = default_memory_provider; + value = (PagemapEntry*)v.alloc_chunk(PAGEMAP_SIZE); + e->store(value, std::memory_order_release); + } + else + { + while ((uintptr_t)e->load(std::memory_order_relaxed) == + LOCKED_ENTRY) + { + bits::pause(); + } + value = e->load(std::memory_order_acquire); + } + } + else + { + result = false; + return nullptr; + } + } + result = true; + return value; + } + + template + inline std::pair get_leaf_index(void* p, bool& result) + { + size_t addr = (size_t)p; +#ifdef FreeBSD_KERNEL + // Zero the top 16 bits - kernel addresses all have them set, but the + // data structure assumes that they're zero. + addr &= 0xffffffffffffULL; +#endif + size_t ix = addr >> TOPLEVEL_SHIFT; + size_t shift = TOPLEVEL_SHIFT; + std::atomic* e = &top[ix]; + + for (size_t i = 0; i < INDEX_LEVELS; i++) + { + PagemapEntry* value = get_node(e, result); + if (!result) + return std::pair(nullptr, 0); + + shift -= BITS_PER_INDEX_LEVEL; + ix = (addr >> shift) & ENTRIES_MASK; + e = &value->entries[ix]; + + if constexpr (INDEX_LEVELS == 1) + { + UNUSED(i); + break; + } + i++; + + if (i == INDEX_LEVELS) + break; + } + + Leaf* leaf = (Leaf*)get_node(e, result); + + if (!result) + return std::pair(nullptr, 0); + + shift -= BITS_FOR_LEAF; + ix = (addr >> shift) & LEAF_MASK; + return std::pair(leaf, ix); + } + + template + inline std::atomic* get_addr(void* p, bool& success) + { + auto leaf_ix = get_leaf_index(p, success); + return &(leaf_ix.first->values[leaf_ix.second]); + } + + public: + static constexpr size_t GRANULARITY = 1 << GRANULARITY_BITS; + + /** + * Returns the index of a pagemap entry within a given page. This is used + * in code that propagates changes to the pagemap elsewhere. + */ + size_t index_for_address(void* p) + { + bool success; + return (OS_PAGE_SIZE - 1) & + reinterpret_cast(get_addr(p, success)); + } + + /** + * Returns the address of the page containing + */ + void* page_for_address(void* p) + { + bool success; + return reinterpret_cast( + ~(OS_PAGE_SIZE - 1) & + reinterpret_cast(get_addr(p, success))); + } + + std::atomic* get_ptr(void* p) + { + bool success; + return get_addr(p, success); + } + + T get(void* p) + { + bool success; + auto addr = get_addr(p, success); + if (!success) + return default_content; + return addr->load(std::memory_order_relaxed); + } + + void set(void* p, T x) + { + bool success; + auto addr = get_addr(p, success); + addr->store(x, std::memory_order_relaxed); + } + + void set_range(void* p, T x, size_t length) + { + bool success; + do + { + auto leaf_ix = get_leaf_index(p, success); + size_t ix = leaf_ix.second; + + auto last = std::min(LEAF_MASK + 1, ix + length); + + auto diff = last - ix; + + for (; ix < last; ix++) + { + leaf_ix.first->values[ix] = x; + } + + length = length - diff; + p = (void*)((uintptr_t)p + (diff << GRANULARITY_BITS)); + } while (length > 0); + } + }; +} diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h new file mode 100644 index 0000000..60d4856 --- /dev/null +++ b/src/mem/remoteallocator.h @@ -0,0 +1,64 @@ +#pragma once + +#include "../ds/mpscq.h" +#include "../mem/allocconfig.h" + +#include + +namespace snmalloc +{ + struct Remote + { + static const uint64_t SIZECLASS_SHIFT = 56; + static const uint64_t SIZECLASS_MASK = 0xffULL << SIZECLASS_SHIFT; + static const uint64_t TARGET_MASK = ~SIZECLASS_MASK; + + static_assert(SIZECLASS_MASK == 0xff00'0000'0000'0000ULL); + + using alloc_id_t = size_t; + union + { + std::atomic next; + Remote* non_atomic_next; + }; + + uint64_t value; + + void set_target_id(alloc_id_t id) + { + assert(id == (id & TARGET_MASK)); + value = (id & TARGET_MASK) | (value & SIZECLASS_MASK); + } + + void set_sizeclass(uint8_t sizeclass) + { + value = (value & TARGET_MASK) | + ((static_cast(sizeclass) << SIZECLASS_SHIFT) & + SIZECLASS_MASK); + } + + alloc_id_t target_id() + { + return value & TARGET_MASK; + } + + uint8_t sizeclass() + { + return (value & SIZECLASS_MASK) >> SIZECLASS_SHIFT; + } + }; + + struct RemoteAllocator + { + using alloc_id_t = Remote::alloc_id_t; + // Store the message queue on a separate cacheline. It is mutable data that + // is read by other threads. + alignas(CACHELINE_SIZE) MPSCQ message_queue; + + alloc_id_t id() + { + return static_cast( + reinterpret_cast(&message_queue)); + } + }; +} diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h new file mode 100644 index 0000000..1561df2 --- /dev/null +++ b/src/mem/sizeclass.h @@ -0,0 +1,148 @@ +#pragma once + +#include "allocconfig.h" + +namespace snmalloc +{ + constexpr static uint16_t get_slab_offset(uint8_t sc, bool is_short); + constexpr static size_t sizeclass_to_size(uint8_t sizeclass); + constexpr static uint16_t medium_slab_free(uint8_t sizeclass); + + static inline uint8_t size_to_sizeclass(size_t size) + { + // Don't use sizeclasses that are not a multiple of the alignment. + // For example, 24 byte allocations can be + // problematic for some data due to alignment issues. + return (uint8_t)bits::to_exp_mant(size); + } + + constexpr static inline uint8_t size_to_sizeclass_const(size_t size) + { + // Don't use sizeclasses that are not a multiple of the alignment. + // For example, 24 byte allocations can be + // problematic for some data due to alignment issues. + return (uint8_t)bits::to_exp_mant_const( + size); + } + + constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class) + { + return (size_t)1 << (large_class + SUPERSLAB_BITS); + } + + // Small classes range from [MIN, SLAB], i.e. inclusive. + static constexpr size_t NUM_SMALL_CLASSES = + size_to_sizeclass_const((size_t)1 << SLAB_BITS) + 1; + + static constexpr size_t NUM_SIZECLASSES = + size_to_sizeclass_const((size_t)1 << SUPERSLAB_BITS); + + // Medium classes range from (SLAB, SUPERSLAB), i.e. non-inclusive. + static constexpr size_t NUM_MEDIUM_CLASSES = + NUM_SIZECLASSES - NUM_SMALL_CLASSES; + + // Large classes range from [SUPERSLAB, ADDRESS_SPACE). + static constexpr size_t NUM_LARGE_CLASSES = + bits::ADDRESS_BITS - SUPERSLAB_BITS; + + template + constexpr void check_same() + { + static_assert(X == Y, "Values must be the same"); + } + + static_assert(size_to_sizeclass_const(SUPERSLAB_SIZE) == NUM_SIZECLASSES); + + inline static size_t round_by_sizeclass(size_t rsize, size_t offset) + { + // check_same(); + // Must be called with a rounded size. + assert(sizeclass_to_size(size_to_sizeclass(rsize)) == rsize); + // Only works up to certain offsets, exhaustively tested upto + // SUPERSLAB_SIZE. + assert(offset <= SUPERSLAB_SIZE); + + size_t align = bits::ctz(rsize); + size_t divider = rsize >> align; + // Maximum of 24 bits for 16MiB super/medium slab + if (INTERMEDIATE_BITS == 0 || divider == 1) + { + assert(divider == 1); + return offset & ~(rsize - 1); + } + + if constexpr (bits::is64() && INTERMEDIATE_BITS <= 2) + { + // Only works for 64 bit multiplication, as the following will overflow in + // 32bit. + // The code is using reciprocal division, with a shift of 26 bits, this + // is considerably more bits than we need in the result. If SUPERSLABS + // get larger then we should review this code. + static_assert(SUPERSLAB_BITS <= 24, "The following code assumes 24 bits"); + static constexpr size_t shift = 26; + size_t back_shift = shift + align; + static constexpr size_t mul_shift = 1ULL << shift; + static constexpr uint32_t constants[8] = {0, + mul_shift, + 0, + (mul_shift / 3) + 1, + 0, + (mul_shift / 5) + 1, + 0, + (mul_shift / 7) + 1}; + return ((constants[divider] * offset) >> back_shift) * rsize; + } + else + // Use 32-bit division as considerably faster than 64-bit, and + // everything fits into 32bits here. + return (uint32_t)(offset / rsize) * rsize; + } + + inline static bool is_multiple_of_sizeclass(size_t rsize, size_t offset) + { + // Must be called with a rounded size. + assert(sizeclass_to_size(size_to_sizeclass(rsize)) == rsize); + // Only works up to certain offsets, exhaustively tested upto + // SUPERSLAB_SIZE. + assert(offset <= SUPERSLAB_SIZE); + + size_t align = bits::ctz(rsize); + size_t divider = rsize >> align; + // Maximum of 24 bits for 16MiB super/medium slab + if (INTERMEDIATE_BITS == 0 || divider == 1) + { + assert(divider == 1); + return (offset & (rsize - 1)) == 0; + } + + if constexpr (bits::is64() && INTERMEDIATE_BITS <= 2) + { + // Only works for 64 bit multiplication, as the following will overflow in + // 32bit. + // The code is using reciprocal division, with a shift of 26 bits, this + // is considerably more bits than we need in the result. If SUPERSLABS + // get larger then we should review this code. + static_assert(SUPERSLAB_BITS <= 24, "The following code assumes 24 bits"); + static constexpr size_t shift = 31; + static constexpr size_t mul_shift = 1ULL << shift; + static constexpr uint32_t constants[8] = {0, + mul_shift, + 0, + (mul_shift / 3) + 1, + 0, + (mul_shift / 5) + 1, + 0, + (mul_shift / 7) + 1}; + + // There is a long chain of zeros after the backshift + // However, not all zero so just check a range. + // This is exhaustively tested for the current use case + return (((constants[divider] * offset)) & + (((1ULL << (align + 3)) - 1) << (shift - 3))) == 0; + } + else + // Use 32-bit division as considerably faster than 64-bit, and + // everything fits into 32bits here. + return (uint32_t)(offset % rsize) == 0; + } +}; diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h new file mode 100644 index 0000000..099b465 --- /dev/null +++ b/src/mem/sizeclasstable.h @@ -0,0 +1,71 @@ +#pragma once + +#include "superslab.h" + +namespace snmalloc +{ + struct SizeClassTable + { + size_t size[NUM_SIZECLASSES]; + uint16_t bump_ptr_start[NUM_SMALL_CLASSES]; + uint16_t short_bump_ptr_start[NUM_SMALL_CLASSES]; + uint16_t count_per_slab[NUM_SMALL_CLASSES]; + uint16_t medium_slab_slots[NUM_MEDIUM_CLASSES]; + + constexpr SizeClassTable() + : size(), + bump_ptr_start(), + short_bump_ptr_start(), + count_per_slab(), + medium_slab_slots() + { + for (uint8_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) + { + size[sizeclass] = + bits::from_exp_mant(sizeclass); + } + + size_t header_size = sizeof(Superslab); + size_t short_slab_size = SLAB_SIZE - header_size; + + for (uint8_t i = 0; i < NUM_SMALL_CLASSES; i++) + { + short_bump_ptr_start[i] = + (uint16_t)(1 + (short_slab_size % size[i]) + header_size); + bump_ptr_start[i] = (uint16_t)(1 + (SLAB_SIZE % size[i])); + count_per_slab[i] = (uint16_t)(SLAB_SIZE / size[i]); + } + + for (uint8_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) + { + medium_slab_slots[i - NUM_SMALL_CLASSES] = + (uint16_t)((SUPERSLAB_SIZE - Mediumslab::header_size()) / size[i]); + } + } + }; + + static constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); + + static inline constexpr uint16_t get_slab_offset(uint8_t sc, bool is_short) + { + if (is_short) + return sizeclass_metadata.short_bump_ptr_start[sc]; + else + return sizeclass_metadata.bump_ptr_start[sc]; + } + + constexpr static inline size_t sizeclass_to_size(uint8_t sizeclass) + { + return sizeclass_metadata.size[sizeclass]; + } + + constexpr static inline size_t sizeclass_to_count(uint8_t sizeclass) + { + return sizeclass_metadata.count_per_slab[sizeclass]; + } + + constexpr static inline uint16_t medium_slab_free(uint8_t sizeclass) + { + return sizeclass_metadata.medium_slab_slots[sizeclass - NUM_SMALL_CLASSES]; + } +} diff --git a/src/mem/slab.h b/src/mem/slab.h new file mode 100644 index 0000000..33138d9 --- /dev/null +++ b/src/mem/slab.h @@ -0,0 +1,165 @@ +#pragma once + +#include "superslab.h" + +namespace snmalloc +{ + class Slab + { + private: + uint16_t pointer_to_index(void* p) + { + // Get the offset from the slab for a memory location. + return (uint16_t)((size_t)p - (size_t)this); + } + + public: + static Slab* get(void* p) + { + return (Slab*)((size_t)p & SLAB_MASK); + } + + Metaslab* get_meta() + { + Superslab* super = Superslab::get(this); + return super->get_meta(this); + } + + SlabLink* get_link() + { + return get_meta()->get_link(this); + } + + template + void* alloc(SlabList* sc, size_t rsize, MemoryProvider& memory_provider) + { + // Read the head from the metadata stored in the superslab. + Metaslab* meta = get_meta(); + uint16_t head = meta->head; + + assert(rsize == sizeclass_to_size(meta->sizeclass)); + meta->debug_slab_invariant(is_short(), this); + assert(sc->get_head() == (SlabLink*)((size_t)this + meta->link)); + assert(!meta->is_full()); + + meta->add_use(); + + void* p; + + if ((head & 1) == 0) + { + p = (void*)((size_t)this + head); + + // Read the next slot from the memory that's about to be allocated. + uint16_t next = *(uint16_t*)p; + meta->head = next; + } + else + { + // This slab is being bump allocated. + p = (void*)((size_t)this + head - 1); + meta->head = head + (uint16_t)rsize; + if (meta->head == 1) + { + meta->set_full(); + } + } + + // If we're full, we're no longer the current slab for this sizeclass + if (meta->is_full()) + sc->pop(); + + meta->debug_slab_invariant(is_short(), this); + + if (zero_mem == YesZero) + { + if (rsize < PAGE_ALIGNED_SIZE) + memory_provider.zero(p, rsize); + else + memory_provider.template zero(p, rsize); + } + + return p; + } + + // Returns true, if it alters get_status. + template + inline typename Superslab::Action dealloc( + SlabList* sc, Superslab* super, void* p, MemoryProvider& memory_provider) + { + Metaslab* meta = super->get_meta(this); + + bool was_full = meta->is_full(); + meta->debug_slab_invariant(is_short(), this); + meta->sub_use(); + +#ifndef SNMALLOC_SAFE_CLIENT + if (!is_multiple_of_sizeclass( + sizeclass_to_size(meta->sizeclass), + (uintptr_t)this + SLAB_SIZE - (uintptr_t)p)) + { + error("Not deallocating start of an object"); + } +#endif + + if (was_full) + { + // We are not on the sizeclass list. + if (!meta->is_unused()) + { + // Update the head and the sizeclass link. + uint16_t index = pointer_to_index(p); + meta->head = index; + assert(meta->valid_head(is_short())); + meta->link = index; + + // Push on the list of slabs for this sizeclass. + sc->insert(meta->get_link(this)); + meta->debug_slab_invariant(is_short(), this); + } + else + { + // Dealloc on the superslab. + if (is_short()) + return super->dealloc_short_slab(memory_provider); + else + return super->dealloc_slab(this, memory_provider); + } + } + else if (meta->is_unused()) + { + // Remove from the sizeclass list and dealloc on the superslab. + sc->remove(meta->get_link(this)); + + if (is_short()) + return super->dealloc_short_slab(memory_provider); + else + return super->dealloc_slab(this, memory_provider); + } + else + { +#ifndef NDEBUG + sc->debug_check_contains(meta->get_link(this)); +#endif + + // Update the head and the next pointer in the free list. + uint16_t head = meta->head; + uint16_t current = pointer_to_index(p); + + // Set the head to the memory being deallocated. + meta->head = current; + assert(meta->valid_head(is_short())); + + // Set the next pointer to the previous head. + *(uint16_t*)p = head; + meta->debug_slab_invariant(is_short(), this); + } + return Superslab::NoSlabReturn; + } + + bool is_short() + { + return ((size_t)this & SUPERSLAB_MASK) == (size_t)this; + } + }; +} diff --git a/src/mem/superslab.h b/src/mem/superslab.h new file mode 100644 index 0000000..5a1c901 --- /dev/null +++ b/src/mem/superslab.h @@ -0,0 +1,233 @@ +#pragma once + +#include "metaslab.h" + +#include + +namespace snmalloc +{ + class Superslab : public Allocslab + { + // This is the view of a 16 mb superslab when it is being used to allocate + // 64 kb slabs. + private: + friend DLList; + + // Keep the allocator pointer on a separate cache line. It is read by + // other threads, and does not change, so we avoid false sharing. + alignas(CACHELINE_SIZE) + // The superslab is kept on a doubly linked list of superslabs which + // have some space. + Superslab* next; + Superslab* prev; + + // This is a reference to the first unused slab in the free slab list + // It is does not contain the short slab, which is handled using a bit + // in the "used" field below. The list is terminated by pointing to + // the short slab. + // The head linked list has an absolute pointer for head, but the next + // pointers stores in the metaslabs are relative pointers, that is they + // are the relative offset to the next entry minus 1. This means that + // all zeros is a list that chains through all the blocks, so the zero + // initialised memory requires no more work. + uint8_t head; + + // Represents twice the number of full size slabs used + // plus 1 for the short slab. i.e. using 3 slabs and the + // short slab would be 6 + 1 = 7 + uint16_t used; + + Metaslab meta[SLAB_COUNT]; + + // Used size_t as results in better code in MSVC + size_t slab_to_index(Slab* slab) + { + auto res = (((size_t)slab - (size_t)this) >> SLAB_BITS); + assert(res == (uint8_t)res); + return res; + } + + public: + enum Status + { + Full, + Available, + OnlyShortSlabAvailable, + Empty + }; + + enum Action + { + NoSlabReturn = 0, + NoStatusChange = 1, + StatusChange = 2 + }; + + static Superslab* get(void* p) + { + return (Superslab*)((size_t)p & SUPERSLAB_MASK); + } + + static bool is_short_sizeclass(uint8_t sizeclass) + { + constexpr uint8_t h = size_to_sizeclass_const(sizeof(Superslab)); + return sizeclass <= h; + } + + template + void init(RemoteAllocator* alloc, MemoryProvider& memory_provider) + { + allocator = alloc; + + if (kind != Super) + { + // If this wasn't previously a Superslab, we need to set up the + // header. + kind = Super; + // Point head at the first non-short slab. + head = 1; + + if (kind != Fresh) + { + // If this wasn't previously Fresh, we need to zero some things. + used = 0; + memory_provider.zero(meta, SLAB_COUNT * sizeof(Metaslab)); + } + + meta[0].set_unused(); + } + } + + bool is_empty() + { + return used == 0; + } + + bool is_full() + { + return (used == (((SLAB_COUNT - 1) << 1) + 1)); + } + + bool is_almost_full() + { + return (used >= ((SLAB_COUNT - 1) << 1)); + } + + Status get_status() + { + if (!is_almost_full()) + { + if (!is_empty()) + { + return Available; + } + else + { + return Empty; + } + } + else + { + if (!is_full()) + { + return OnlyShortSlabAvailable; + } + else + { + return Full; + } + } + } + + Metaslab* get_meta(Slab* slab) + { + return &meta[slab_to_index(slab)]; + } + + template + Slab* alloc_short_slab(uint8_t sizeclass, MemoryProvider& memory_provider) + { + if ((used & 1) == 1) + return alloc_slab(sizeclass, memory_provider); + + meta[0].head = get_slab_offset(sizeclass, true); + meta[0].sizeclass = sizeclass; + meta[0].link = SLABLINK_INDEX; + + if (decommit_strategy == DecommitAll) + { + memory_provider.template notify_using( + (void*)((size_t)this + OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE); + } + + used++; + return (Slab*)this; + } + + template + Slab* alloc_slab(uint8_t sizeclass, MemoryProvider& memory_provider) + { + Slab* slab = (Slab*)((size_t)this + ((size_t)head << SLAB_BITS)); + + uint8_t n = meta[head].next; + + meta[head].head = get_slab_offset(sizeclass, false); + meta[head].sizeclass = sizeclass; + meta[head].link = SLABLINK_INDEX; + + head = head + n + 1; + used += 2; + + if (decommit_strategy == DecommitAll) + { + memory_provider.template notify_using(slab, SLAB_SIZE); + } + + return slab; + } + + // Returns true, if this alters the value of get_status + template + Action dealloc_slab(Slab* slab, MemoryProvider& memory_provider) + { + // This is not the short slab. + uint8_t index = (uint8_t)slab_to_index(slab); + uint8_t n = head - index - 1; + + meta[index].next = n; + head = index; + bool was_almost_full = is_almost_full(); + used -= 2; + + if (decommit_strategy == DecommitAll) + memory_provider.notify_not_using(slab, SLAB_SIZE); + + assert(meta[index].is_unused()); + if (was_almost_full || is_empty()) + return StatusChange; + + return NoStatusChange; + } + + // Returns true, if this alters the value of get_status + template + Action dealloc_short_slab(MemoryProvider& memory_provider) + { + // This is the short slab. + if (decommit_strategy == DecommitAll) + { + memory_provider.notify_not_using( + (void*)((size_t)this + OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE); + } + + bool was_full = is_full(); + used--; + + assert(meta[0].is_unused()); + if (was_full || is_empty()) + return StatusChange; + + return NoStatusChange; + } + }; +} diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h new file mode 100644 index 0000000..80fd4da --- /dev/null +++ b/src/mem/threadalloc.h @@ -0,0 +1,265 @@ +#pragma once + +#include "../ds/helpers.h" +#include "globalalloc.h" +#if defined(SNMALLOC_USE_THREAD_DESTRUCTOR) && \ + defined(SNMALLOC_USE_THREAD_CLEANUP) +#error At most one out of SNMALLOC_USE_THREAD_CLEANUP and SNMALLOC_USE_THREAD_DESTRUCTOR may be defined. +#endif + +namespace snmalloc +{ + extern "C" void _malloc_thread_cleanup(void); + +#ifdef SNMALLOC_EXTERNAL_THREAD_ALLOC + /** + * Version of the `ThreadAlloc` interface that does no managment of thread + * local state, and just assumes that "ThreadAllocUntyped::get" has been + * declared before including snmalloc.h. As it is included before, it cannot + * know the allocator type, hence the casting. + * + * This class is used only when snmalloc is compiled as part of a runtime, + * which has its own managment of the thread local allocator pointer. + */ + class ThreadAllocUntypedWrapper + { + public: + static inline Alloc*& get() + { + return (Alloc*&)ThreadAllocUntyped::get(); + } + }; +#endif + + /** + * Version of the `ThreadAlloc` interface that uses a hook provided by libc + * to destroy thread-local state. This is the ideal option, because it + * enforces ordering of destruction such that the malloc state is destroyed + * after anything that can allocate memory. + * + * This class is used only when snmalloc is compiled as part of a compatible + * libc (for example, FreeBSD libc). + */ + class ThreadAllocLibcCleanup + { + /** + * Libc will call `_malloc_thread_cleanup` just before a thread terminates. + * This function must be allowed to call back into this class to destroy + * the state. + */ + friend void _malloc_thread_cleanup(void); + + /** + * Function called when the thread exits. This is guaranteed to be called + * precisely once per thread and releases the current allocator. + */ + static inline void exit() + { + if (auto* per_thread = get(false)) + { + current_alloc_pool()->release(per_thread); + per_thread = nullptr; + } + } + + public: + /** + * Returns a pointer to the allocator associated with this thread. If + * `create` is true, it will create an allocator if one does not exist, + * otherwise it will return `nullptr` in this case. This should be called + * with `create == false` only during thread teardown. + * + * The non-create case exists so that the `per_thread` variable can be a + * local static and not a global, allowing ODR to deduplicate it. + */ + static inline Alloc*& get(bool create = true) + { + static thread_local Alloc* per_thread; + if (!per_thread && create) + { + per_thread = current_alloc_pool()->acquire(); + } + return per_thread; + } + }; + /** + * Version of the `ThreadAlloc` interface that uses C++ `thread_local` + * destructors for cleanup. If a per-thread allocator is used during the + * destruction of other per-thread data, this class will create a new + * instance and register its destructor, so should eventually result in + * cleanup, but may result in allocators being returned to the global pool + * and then reacquired multiple times. + * + * This implementation depends on nothing outside of a working C++ + * environment and so should be the simplest for initial bringup on an + * unsupported platform. It is currently used in the FreeBSD kernel version. + */ + class ThreadAllocThreadDestructor + { + /** + * A pointer to the allocator owned by this thread. + */ + Alloc* alloc; + + /** + * Constructor. Acquires a new allocator and associates it with this + * object. There should be only one instance of this class per thread. + */ + ThreadAllocThreadDestructor() : alloc(current_alloc_pool()->acquire()) {} + + /** + * Destructor. Releases the allocator owned by this thread. + */ + ~ThreadAllocThreadDestructor() + { + current_alloc_pool()->release(alloc); + } + + public: + /** + * Public interface, returns the allocator for this thread, constructing + * one if necessary. + */ + static inline Alloc*& get() + { + static thread_local ThreadAllocThreadDestructor per_thread; + return per_thread.alloc; + } + }; + // When targeting the FreeBSD kernel, the pthread header exists, but the + // pthread symbols do not, so don't compile this because it will fail to + // link. +#ifndef FreeBSD_KERNEL + /** + * Version of the `ThreadAlloc` interface that uses thread-specific (POSIX + * threads) or Fiber-local (Windows) storage with an explicit destructor. + * Neither of the underlying mechanisms guarantee ordering, so the cleanup + * may be called before other cleanup functions or thread-local destructors. + * + * This implementation is used when using snmalloc as a library + * implementation of malloc, but not embedding it in C standard library. + * Using this implementation removes the dependency on a C++ runtime library. + */ + class ThreadAllocExplicitTLSCleanup + { + /** + * Cleanup function. This is registered with the operating system's + * thread- or fibre-local storage subsystem to clean up the per-thread + * allocator. + */ + static inline void +# ifdef _WIN32 + NTAPI +# endif + thread_alloc_release(void* p) + { + Alloc** pp = (Alloc**)p; + current_alloc_pool()->release(*pp); + *pp = nullptr; + } + +# ifdef _WIN32 + /** + * Key type used to identify fibre-local storage. + */ + using tls_key_t = DWORD; + + /** + * On Windows, construct a new fibre-local storage allocation. This + * function must not be called more than once. + */ + static inline tls_key_t tls_key_create() noexcept + { + return FlsAlloc(thread_alloc_release); + } + + /** + * On Windows, store a pointer to a `thread_local` pointer to an allocator + * into fibre-local storage. This function takes a pointer to the + * `thread_local` allocation, rather than to the pointee, so that the + * cleanup function can zero the pointer. + * + * This must not be called until after `tls_key_create` has returned. + */ + static inline void tls_set_value(tls_key_t key, Alloc** value) + { + FlsSetValue(key, static_cast(value)); + } +# else + /** + * Key type used for thread-specific storage. + */ + using tls_key_t = pthread_key_t; + + /** + * On POSIX systems, construct a new thread-specific storage allocation. + * This function must not be called more than once. + */ + static inline tls_key_t tls_key_create() noexcept + { + tls_key_t key; + pthread_key_create(&key, thread_alloc_release); + return key; + } + + /** + * On POSIX systems, store a pointer to a `thread_local` pointer to an + * allocator into fibre-local storage. This function takes a pointer to + * the `thread_local` allocation, rather than to the pointee, so that the + * cleanup function can zero the pointer. + * + * This must not be called until after `tls_key_create` has returned. + */ + static inline void tls_set_value(tls_key_t key, Alloc** value) + { + pthread_setspecific(key, static_cast(value)); + } +# endif + public: + /** + * Public interface, returns the allocator for the current thread, + * constructing it if necessary. + */ + static inline Alloc*& get() + { + static thread_local Alloc* per_thread; + + // If we don't have an allocator, construct one. + if (!per_thread) + { + // Construct the allocator and assign it to `per_thread` *before* doing + // anything else. This is important because `tls_key_create` may + // allocate memory and if we are providing the `malloc` implementation + // then this function must be re-entrant within a single thread. In + // this case, the second call to this function will simply return the + // allocator. + per_thread = current_alloc_pool()->acquire(); + + tls_key_t key = Singleton::get(); + // Associate the new allocator with the destructor. + tls_set_value(key, &per_thread); + } + + return per_thread; + } + }; +#endif + +#ifdef SNMALLOC_USE_THREAD_CLEANUP + /** + * Entry point the allows libc to call into the allocator for per-thread + * cleanup. + */ + extern "C" void _malloc_thread_cleanup(void) + { + ThreadAllocLibcCleanup::exit(); + } + using ThreadAlloc = ThreadAllocLibcCleanup; +#elif defined(SNMALLOC_USE_THREAD_DESTRUCTOR) + using ThreadAlloc = ThreadAllocThreadDestructor; +#elif defined(SNMALLOC_EXTERNAL_THREAD_ALLOC) + using ThreadAlloc = ThreadAllocUntypedWrapper; +#else + using ThreadAlloc = ThreadAllocExplicitTLSCleanup; +#endif +} diff --git a/src/mem/typealloc.h b/src/mem/typealloc.h new file mode 100644 index 0000000..e7e656b --- /dev/null +++ b/src/mem/typealloc.h @@ -0,0 +1,86 @@ +#pragma once + +#include "../ds/flaglock.h" +#include "../ds/mpmcstack.h" +#include "typeallocated.h" + +namespace snmalloc +{ + template + class TypeAlloc + { + private: + friend TypeAllocated; + + std::atomic_flag lock = ATOMIC_FLAG_INIT; + MPMCStack stack; + T* list = nullptr; + + TypeAlloc(MemoryProvider& m) : memory_provider(m) {} + + public: + MemoryProvider& memory_provider; + + static TypeAlloc* make(MemoryProvider& memory_provider) noexcept + { + auto r = memory_provider.alloc_chunk(sizeof(TypeAlloc)); + return new (r) TypeAlloc(memory_provider); + } + + static TypeAlloc* make() noexcept + { + return make(default_memory_provider); + } + + template + T* alloc(Args&&... args) + { + T* p = stack.pop(); + + if (p != nullptr) + return p; + + p = (T*)memory_provider.alloc_chunk(sizeof(T)); + + new (p) T(std::forward(args)...); + + FlagLock f(lock); + p->list_next = list; + list = p; + + return p; + } + + void dealloc(T* p) + { + // The object's destructor is not run. If the object is "reallocated", it + // is returned without the constructor being run, so the object is reused + // without re-initialisation. + stack.push(p); + } + + T* extract(T* p = nullptr) + { + // Returns a linked list of all objects in the stack, emptying the stack. + if (p == nullptr) + return stack.pop_all(); + else + return p->next; + } + + void restore(T* first, T* last) + { + // Pushes a linked list of objects onto the stack. Use to put a linked + // list returned by extract back onto the stack. + stack.push(first, last); + } + + T* iterate(T* p = nullptr) + { + if (p == nullptr) + return list; + else + return p->list_next; + } + }; +} diff --git a/src/mem/typeallocated.h b/src/mem/typeallocated.h new file mode 100644 index 0000000..c920647 --- /dev/null +++ b/src/mem/typeallocated.h @@ -0,0 +1,19 @@ +#pragma once + +#include "../ds/bits.h" + +namespace snmalloc +{ + template + class TypeAllocated + { + private: + template + friend class TypeAlloc; + template + friend class MPMCStack; + + std::atomic next = nullptr; + T* list_next; + }; +} \ No newline at end of file diff --git a/src/override/malloc.cc b/src/override/malloc.cc new file mode 100644 index 0000000..2bdd552 --- /dev/null +++ b/src/override/malloc.cc @@ -0,0 +1,188 @@ +#include "../snmalloc.h" + +#include + +using namespace snmalloc; + +#ifndef SNMALLOC_NAME_MANGLE +# define SNMALLOC_NAME_MANGLE(a) a +#endif + +extern "C" +{ + void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr) + { + return Alloc::external_pointer(ptr); + } + + void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) + { + // Include size 0 in the first sizeclass. + size = ((size - 1) >> (bits::BITS - 1)) + size; + + return ThreadAlloc::get()->alloc(size); + } + + void SNMALLOC_NAME_MANGLE(free)(void* ptr) + { + if (ptr == nullptr) + return; + + ThreadAlloc::get()->dealloc(ptr); + } + + void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size) + { + bool overflow = false; + size_t sz = bits::umul(size, nmemb, overflow); + if (overflow) + { + errno = ENOMEM; + return 0; + } + // Include size 0 in the first sizeclass. + sz = ((sz - 1) >> (bits::BITS - 1)) + sz; + return ThreadAlloc::get()->alloc(sz); + } + + size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)(void* ptr) + { + return Alloc::alloc_size(ptr); + } + + void* SNMALLOC_NAME_MANGLE(realloc)(void* ptr, size_t size) + { + if (size == (size_t)-1) + { + errno = ENOMEM; + return nullptr; + } + if (ptr == nullptr) + { + return SNMALLOC_NAME_MANGLE(malloc)(size); + } + if (size == 0) + { + SNMALLOC_NAME_MANGLE(free)(ptr); + return nullptr; + } +#ifndef NDEBUG + // This check is redundant, because the check in memcpy will fail if this + // is skipped, but it's useful for debugging. + if (Alloc::external_pointer(ptr) != ptr) + { + error( + "Calling realloc on pointer that is not to the start of an allocation"); + } +#endif + void* p = SNMALLOC_NAME_MANGLE(malloc)(size); + if (p) + { + assert(p == Alloc::external_pointer(p)); + size_t sz = + (std::min)(size, SNMALLOC_NAME_MANGLE(malloc_usable_size)(ptr)); + memcpy(p, ptr, sz); + SNMALLOC_NAME_MANGLE(free)(ptr); + } + return p; + } + +#ifndef __FreeBSD__ + void* SNMALLOC_NAME_MANGLE(reallocarray)(void* ptr, size_t nmemb, size_t size) + { + bool overflow = false; + size_t sz = bits::umul(size, nmemb, overflow); + if (overflow) + { + errno = ENOMEM; + return nullptr; + } + return SNMALLOC_NAME_MANGLE(realloc)(ptr, sz); + } +#endif + + void* SNMALLOC_NAME_MANGLE(aligned_alloc)(size_t alignment, size_t size) + { + assert((size % alignment) == 0); + (void)alignment; + return SNMALLOC_NAME_MANGLE(malloc)(size); + } + + void* SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size) + { + if ( + (alignment == 0) || (alignment == size_t(-1)) || + (alignment > SUPERSLAB_SIZE)) + { + errno = EINVAL; + return nullptr; + } + if ((size + alignment) < size) + { + errno = ENOMEM; + return nullptr; + } + + uint8_t sc = size_to_sizeclass((std::max)(size, alignment)); + if (sc >= NUM_SIZECLASSES) + { + // large allocs are 16M aligned. + return SNMALLOC_NAME_MANGLE(malloc)(size); + } + for (; sc < NUM_SIZECLASSES; sc++) + { + size = sizeclass_to_size(sc); + if ((size & -size) >= alignment) + { + return SNMALLOC_NAME_MANGLE(aligned_alloc)(alignment, size); + } + } + assert(false); + return nullptr; + } + + int SNMALLOC_NAME_MANGLE(posix_memalign)( + void** memptr, size_t alignment, size_t size) + { + if ( + ((alignment % sizeof(void*)) != 0) || + ((alignment & (alignment - 1)) != 0) || (alignment == 0)) + { + return EINVAL; + } + + void* p = SNMALLOC_NAME_MANGLE(memalign)(alignment, size); + if (p == nullptr) + { + return ENOMEM; + } + *memptr = p; + return 0; + } + +#ifndef __FreeBSD__ + void* SNMALLOC_NAME_MANGLE(valloc)(size_t size) + { + return SNMALLOC_NAME_MANGLE(memalign)(OS_PAGE_SIZE, size); + } +#endif + + void* SNMALLOC_NAME_MANGLE(pvalloc)(size_t size) + { + if (size == size_t(-1)) + { + errno = ENOMEM; + return nullptr; + } + return SNMALLOC_NAME_MANGLE(memalign)( + OS_PAGE_SIZE, (size + OS_PAGE_SIZE - 1) & ~(OS_PAGE_SIZE - 1)); + } + + void SNMALLOC_NAME_MANGLE(_malloc_prefork)(void) {} + void SNMALLOC_NAME_MANGLE(_malloc_postfork)(void) {} + void SNMALLOC_NAME_MANGLE(_malloc_first_thread)(void) {} + int SNMALLOC_NAME_MANGLE(mallctl)(const char*, void*, size_t*, void*, size_t) + { + return ENOENT; + } +} diff --git a/src/override/new.cc b/src/override/new.cc new file mode 100644 index 0000000..c89b313 --- /dev/null +++ b/src/override/new.cc @@ -0,0 +1,67 @@ +#include "../mem/alloc.h" +#include "../mem/threadalloc.h" +#include "../snmalloc.h" + +#ifdef _WIN32 +# define EXCEPTSPEC +#else +# ifdef _GLIBCXX_USE_NOEXCEPT +# define EXCEPTSPEC _GLIBCXX_USE_NOEXCEPT +# elif defined(_NOEXCEPT) +# define EXCEPTSPEC _NOEXCEPT +# else +# define EXCEPTSPEC +# endif +#endif + +using namespace snmalloc; + +void* operator new(size_t size) +{ + return ThreadAlloc::get()->alloc(size); +} + +void* operator new[](size_t size) +{ + return ThreadAlloc::get()->alloc(size); +} + +void* operator new(size_t size, std::nothrow_t&) +{ + return ThreadAlloc::get()->alloc(size); +} + +void* operator new[](size_t size, std::nothrow_t&) +{ + return ThreadAlloc::get()->alloc(size); +} + +void operator delete(void* p)EXCEPTSPEC +{ + ThreadAlloc::get()->dealloc(p); +} + +void operator delete(void* p, size_t size)EXCEPTSPEC +{ + ThreadAlloc::get()->dealloc(p, size); +} + +void operator delete(void* p, std::nothrow_t&) +{ + ThreadAlloc::get()->dealloc(p); +} + +void operator delete[](void* p) EXCEPTSPEC +{ + ThreadAlloc::get()->dealloc(p); +} + +void operator delete[](void* p, size_t size) EXCEPTSPEC +{ + ThreadAlloc::get()->dealloc(p, size); +} + +void operator delete[](void* p, std::nothrow_t&) +{ + ThreadAlloc::get()->dealloc(p); +} diff --git a/src/pal/pal.h b/src/pal/pal.h new file mode 100644 index 0000000..56c4414 --- /dev/null +++ b/src/pal/pal.h @@ -0,0 +1,46 @@ +#pragma once + +namespace snmalloc +{ + void error(const char* const str); +} + +// If simultating OE, then we need the underlying platform +#if !defined(OPEN_ENCLAVE) || defined(OPEN_ENCLAVE_SIMULATION) +# include "pal_free_bsd_kernel.h" +# include "pal_freebsd.h" +# include "pal_linux.h" +# include "pal_windows.h" +#endif +#include "pal_open_enclave.h" +#include "pal_plain.h" + +namespace snmalloc +{ +#if !defined(OPEN_ENCLAVE) || defined(OPEN_ENCLAVE_SIMULATION) + using DefaultPal = +# if defined(_WIN32) + PALWindows; +# elif defined(__linux__) + PALLinux; +# elif defined(FreeBSD_KERNEL) + PALFreeBSDKernel; +# elif defined(__FreeBSD__) + PALFBSD; +# endif +#endif + + using Pal = +#ifdef OPEN_ENCLAVE + PALPlainMixin; +#elif defined(SNMALLOC_MEMORY_PROVIDER) + PALPlainMixin +#else + DefaultPal; +#endif + + inline void error(const char* const str) + { + Pal::error(str); + } +} diff --git a/src/pal/pal_free_bsd_kernel.h b/src/pal/pal_free_bsd_kernel.h new file mode 100644 index 0000000..1ae5d2c --- /dev/null +++ b/src/pal/pal_free_bsd_kernel.h @@ -0,0 +1,90 @@ +#pragma once + +#include "../ds/bits.h" +#include "../mem/allocconfig.h" + +#if defined(FreeBSD_KERNEL) +extern "C" +{ +# include +# include +# include +# include +# include +# include +} + +namespace snmalloc +{ + class PALFreeBSDKernel + { + vm_offset_t get_vm_offset(uint_ptr_t p) + { + return static_cast(reinterpret_cast(p)); + } + + public: + void error(const char* const str) + { + panic("snmalloc error: %s", str); + } + + /// Notify platform that we will not be using these pages + void notify_not_using(void* p, size_t size) + { + vm_offset_t addr = get_vm_offset(p); + kmem_unback(kernel_object, addr, size); + } + + /// Notify platform that we will not be using these pages + template + void notify_using(void* p, size_t size) + { + vm_offset_t addr = get_vm_offset(p); + int flags = M_WAITOK | ((zero_mem == YesZero) ? M_ZERO : 0); + if (kmem_back(kernel_object, addr, size, flags) != KERN_SUCCESS) + { + error("Out of memory"); + } + } + + /// OS specific function for zeroing memory + template + void zero(void* p, size_t size) + { + ::bzero(p, size); + } + + template + void* reserve(size_t* size, size_t align) + { + size_t request = *size; + vm_offset_t addr; + if (vmem_xalloc( + kernel_arena, + request, + align, + 0, + 0, + VMEM_ADDR_MIN, + VMEM_ADDR_MAX, + M_BESTFIT, + &addr)) + { + return nullptr; + } + if (committed) + { + if ( + kmem_back(kernel_object, addr, request, M_ZERO | M_WAITOK) != + KERN_SUCCESS) + { + vmem_xfree(kernel_arena, addr, request); + return nullptr; + } + } + return get_vm_offset(addr); + } + }; +} +#endif diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h new file mode 100644 index 0000000..efa8978 --- /dev/null +++ b/src/pal/pal_freebsd.h @@ -0,0 +1,88 @@ +#pragma once + +#if defined(__FreeBSD__) && !defined(_KERNEL) +# include "../ds/bits.h" +# include "../mem/allocconfig.h" + +# include + +namespace snmalloc +{ + class PALFBSD + { + public: + static void error(const char* const str) + { + puts(str); + abort(); + } + + /// Notify platform that we will not be using these pages + void notify_not_using(void* p, size_t size) noexcept + { + assert(bits::is_aligned_block(p, size)); + madvise(p, size, MADV_FREE); + } + + /// Notify platform that we will not be using these pages + template + void notify_using(void* p, size_t size) noexcept + { + assert( + bits::is_aligned_block(p, size) || (zero_mem == NoZero)); + if (zero_mem == YesZero) + zero(p, size); + } + + /// OS specific function for zeroing memory + template + void zero(void* p, size_t size) noexcept + { + if (page_aligned || bits::is_aligned_block(p, size)) + { + assert(bits::is_aligned_block(p, size)); + void* r = mmap( + p, + size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, + -1, + 0); + + if (r != MAP_FAILED) + return; + } + + bzero(p, size); + } + + template + void* reserve(size_t* size, size_t align) noexcept + { + size_t request = *size; + // Alignment must be a power of 2. + assert(align == bits::next_pow2(align)); + + if (align == 0) + { + align = 1; + } + + size_t log2align = bits::next_pow2_bits(align); + + void* p = mmap( + NULL, + request, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED(log2align), + -1, + 0); + + if (p == MAP_FAILED) + error("Out of memory"); + + return p; + } + }; +} +#endif diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h new file mode 100644 index 0000000..711d774 --- /dev/null +++ b/src/pal/pal_linux.h @@ -0,0 +1,93 @@ +#pragma once + +#if defined(__linux__) +# include "../ds/bits.h" +# include "../mem/allocconfig.h" + +# include +# include +# include + +namespace snmalloc +{ + class PALLinux + { + public: + static void error(const char* const str) + { + puts(str); + abort(); + } + + /// Notify platform that we will not be using these pages + void notify_not_using(void* p, size_t size) noexcept + { + assert(bits::is_aligned_block(p, size)); + // Do nothing. Don't call madvise here, as the system call slows the + // allocator down too much. + UNUSED(p); + UNUSED(size); + } + + /// Notify platform that we will not be using these pages + template + void notify_using(void* p, size_t size) noexcept + { + assert( + bits::is_aligned_block(p, size) || (zero_mem == NoZero)); + + if (zero_mem == YesZero) + zero(p, size); + } + + /// OS specific function for zeroing memory + template + void zero(void* p, size_t size) noexcept + { + if (page_aligned || bits::is_aligned_block(p, size)) + { + assert(bits::is_aligned_block(p, size)); + madvise(p, size, MADV_DONTNEED); + } + else + { + ::memset(p, 0, size); + } + } + + template + void* reserve(size_t* size, size_t align) noexcept + { + size_t request = *size; + // Add align, so we can guarantee to provide at least size. + request += align; + // Alignment must be a power of 2. + assert(align == bits::next_pow2(align)); + + void* p = mmap( + NULL, + request, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, + 0); + + if (p == MAP_FAILED) + error("Out of memory"); + *size = request; + uintptr_t p0 = (uintptr_t)p; + uintptr_t start = bits::align_up(p0, align); + + if (start > (uintptr_t)p0) + { + uintptr_t end = bits::align_down(p0 + request, align); + *size = end - start; + munmap(p, start - p0); + munmap((void*)end, (p0 + request) - end); + p = (void*)start; + } + return p; + } + }; +} +#endif diff --git a/src/pal/pal_open_enclave.h b/src/pal/pal_open_enclave.h new file mode 100644 index 0000000..4b62f8b --- /dev/null +++ b/src/pal/pal_open_enclave.h @@ -0,0 +1,58 @@ +#pragma once + +#include "pal_plain.h" +#ifdef OPEN_ENCLAVE +extern "C" const void* __oe_get_heap_base(); +extern "C" const void* __oe_get_heap_end(); +extern "C" void* oe_memset(void* p, int c, size_t size); +extern "C" void oe_abort(); + +namespace snmalloc +{ + class PALOpenEnclave + { + std::atomic oe_base; + + public: + static void error(const char* const str) + { + UNUSED(str); + oe_abort(); + } + + template + void* reserve(size_t* size, size_t align) noexcept + { + if (oe_base == 0) + { + uintptr_t dummy = 0; + oe_base.compare_exchange_strong(dummy, (uintptr_t)__oe_get_heap_base()); + } + + uintptr_t old_base = oe_base; + uintptr_t old_base2 = old_base; + uintptr_t next_base; + auto end = (uintptr_t)__oe_get_heap_end(); + do + { + old_base2 = old_base; + auto new_base = bits::align_up(old_base, align); + next_base = new_base + *size; + + if (next_base > end) + error("Out of memory"); + + } while (oe_base.compare_exchange_strong(old_base, next_base)); + + *size = next_base - old_base2; + return (void*)old_base; + } + + template + void zero(void* p, size_t size) noexcept + { + oe_memset(p, 0, size); + } + }; +} +#endif \ No newline at end of file diff --git a/src/pal/pal_plain.h b/src/pal/pal_plain.h new file mode 100644 index 0000000..905f32a --- /dev/null +++ b/src/pal/pal_plain.h @@ -0,0 +1,34 @@ +#pragma once + +#include "../ds/bits.h" +#include "../mem/allocconfig.h" + +namespace snmalloc +{ + // Can be extended + // Will require a reserve method in subclasses. + template + class PALPlainMixin : public State + { + public: + PALPlainMixin() : State() {} + + // Notify platform that we will not be using these pages + void notify_not_using(void*, size_t) noexcept {} + + // Notify platform that we will not be using these pages + template + void notify_using(void* p, size_t size) noexcept + { + if constexpr (zero_mem == YesZero) + { + State::zero(p, size); + } + else + { + UNUSED(p); + UNUSED(size); + } + } + }; +} diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h new file mode 100644 index 0000000..7fcd00e --- /dev/null +++ b/src/pal/pal_windows.h @@ -0,0 +1,111 @@ +#pragma once + +#include "../ds/bits.h" +#include "../mem/allocconfig.h" + +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# define NOMINMAX +# include + +namespace snmalloc +{ + class PALWindows + { + public: + static void error(const char* const str) + { + puts(str); + abort(); + } + + /// Notify platform that we will not be using these pages + void notify_not_using(void* p, size_t size) noexcept + { + assert(bits::is_aligned_block(p, size)); + + BOOL ok = VirtualFree(p, size, MEM_DECOMMIT); + + if (!ok) + error("VirtualFree failed"); + } + + /// Notify platform that we will not be using these pages + template + void notify_using(void* p, size_t size) noexcept + { + assert( + bits::is_aligned_block(p, size) || (zero_mem == NoZero)); + + void* r = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE); + + if (r == nullptr) + error("out of memory"); + } + + /// OS specific function for zeroing memory + template + void zero(void* p, size_t size) noexcept + { + if (page_aligned || bits::is_aligned_block(p, size)) + { + assert(bits::is_aligned_block(p, size)); + notify_not_using(p, size); + notify_using(p, size); + } + else + ::memset(p, 0, size); + } + +# ifdef USE_SYSTEMATIC_TESTING + size_t& systematic_bump_ptr() + { + static size_t bump_ptr = (size_t)0x4000'0000'0000; + return bump_ptr; + } +# endif + + template + void* reserve(size_t* size, size_t align) noexcept + { + // Add align, so we can guarantee to provide at least size. + size_t request = *size + align; + + // Alignment must be a power of 2. + assert(align == bits::next_pow2(align)); + + DWORD flags = MEM_RESERVE; + + if (committed) + flags |= MEM_COMMIT; + + void* p; +# ifdef USE_SYSTEMATIC_TESTING + size_t retries = 1000; + do + { + p = VirtualAlloc( + (void*)systematic_bump_ptr(), request, flags, PAGE_READWRITE); + + systematic_bump_ptr() += request; + retries--; + } while (p == nullptr && retries > 0); +# else + p = VirtualAlloc(nullptr, request, flags, PAGE_READWRITE); +# endif + + uintptr_t aligned_p = bits::align_up((size_t)p, align); + + if (aligned_p != (uintptr_t)p) + { + auto extra_bit = aligned_p - (uintptr_t)p; + uintptr_t end = (uintptr_t)p + request; + // Attempt to align end of the block. + VirtualAlloc((void*)end, extra_bit, flags, PAGE_READWRITE); + } + *size = request; + return p; + } + }; +} +#endif \ No newline at end of file diff --git a/src/snmalloc.h b/src/snmalloc.h new file mode 100644 index 0000000..8c22647 --- /dev/null +++ b/src/snmalloc.h @@ -0,0 +1,3 @@ +#pragma once + +#include "mem/threadalloc.h" diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc new file mode 100644 index 0000000..b00a684 --- /dev/null +++ b/src/test/func/fixed_region/fixed_region.cc @@ -0,0 +1,52 @@ + +#define OPEN_ENCLAVE +#define OPEN_ENCLAVE_SIMULATION +#define USE_RESERVE_MULTIPLE 1 +#include +#include + +void* oe_base; +void* oe_end; +extern "C" const void* __oe_get_heap_base() +{ + return oe_base; +} + +extern "C" const void* __oe_get_heap_end() +{ + return oe_end; +} + +extern "C" void* oe_memset(void* p, int c, size_t size) +{ + return memset(p, c, size); +} + +extern "C" void oe_abort() +{ + abort(); +} + +using namespace snmalloc; +int main() +{ + DefaultPal pal; + + size_t size = 1ULL << 28; + oe_base = pal.reserve(&size, 0); + oe_end = (uint8_t*)oe_base + size; + std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl; + + auto a = ThreadAlloc::get(); + + for (size_t i = 0; i < 1000; i++) + { + auto r1 = a->alloc(100); + std::cout << "Allocated object " << r1 << std::endl; + + if (oe_base > r1) + abort(); + if (oe_end < r1) + abort(); + } +} \ No newline at end of file diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc new file mode 100644 index 0000000..c4d6ca5 --- /dev/null +++ b/src/test/func/memory/memory.cc @@ -0,0 +1,275 @@ +#include +#include +#include +#include + +using namespace snmalloc; + +void test_alloc_dealloc_64k() +{ + auto* alloc = ThreadAlloc::get(); + + constexpr size_t count = 1 << 12; + constexpr size_t outer_count = 12; + void* garbage[count]; + void* keep_alive[outer_count]; + + for (size_t j = 0; j < outer_count; j++) + { + // Allocate 64k of 16byte allocs + // This will fill the short slab, and then start a new slab. + for (size_t i = 0; i < count; i++) + { + garbage[i] = alloc->alloc(16); + } + + // Allocate one object on the second slab + keep_alive[j] = alloc->alloc(16); + + for (size_t i = 0; i < count; i++) + { + alloc->dealloc(garbage[i]); + } + } + for (size_t j = 0; j < outer_count; j++) + { + alloc->dealloc(keep_alive[j]); + } +} + +void test_random_allocation() +{ + auto* alloc = ThreadAlloc::get(); + std::unordered_set allocated; + + constexpr size_t count = 10000; + constexpr size_t outer_count = 10; + void* objects[count]; + for (size_t i = 0; i < count; i++) + objects[i] = nullptr; + + // Randomly allocate and deallocate objects + xoroshiro::p128r32 r; + size_t alloc_count = 0; + for (size_t j = 0; j < outer_count; j++) + { + auto just_dealloc = r.next() % 2 == 1; + auto duration = r.next() % count; + for (size_t i = 0; i < duration; i++) + { + auto index = r.next(); + auto& cell = objects[index % count]; + if (cell != nullptr) + { + alloc->dealloc(cell); + allocated.erase(cell); + cell = nullptr; + alloc_count--; + } + if (!just_dealloc) + { + cell = alloc->alloc(16); + auto pair = allocated.insert(cell); + // Check not already allocated + assert(pair.second); + UNUSED(pair); + alloc_count++; + } + else + { + if (alloc_count == 0 && just_dealloc) + break; + } + } + } + + // Deallocate all the remaining objects + for (size_t i = 0; i < count; i++) + if (objects[i] != nullptr) + alloc->dealloc(objects[i]); +} + +void test_calloc() +{ + auto* alloc = ThreadAlloc::get(); + + for (size_t size = 16; size <= (1 << 24); size <<= 1) + { + void* p = alloc->alloc(size); + memset(p, 0xFF, size); + alloc->dealloc(p, size); + + p = alloc->alloc(size); + + for (size_t i = 0; i < size; i++) + { + if (((char*)p)[i] != 0) + abort(); + } + + alloc->dealloc(p, size); + } + + current_alloc_pool()->debug_check_empty(); +} + +void test_double_alloc() +{ + auto* a1 = current_alloc_pool()->acquire(); + auto* a2 = current_alloc_pool()->acquire(); + + const size_t n = (1 << 16) / 32; + + for (size_t k = 0; k < 4; k++) + { + std::unordered_set set1; + std::unordered_set set2; + + for (size_t i = 0; i < (n * 2); i++) + { + void* p = a1->alloc(20); + assert(set1.find(p) == set1.end()); + set1.insert(p); + } + + for (size_t i = 0; i < (n * 2); i++) + { + void* p = a2->alloc(20); + assert(set2.find(p) == set2.end()); + set2.insert(p); + } + + while (!set1.empty()) + { + auto it = set1.begin(); + a2->dealloc(*it, 20); + set1.erase(it); + } + + while (!set2.empty()) + { + auto it = set2.begin(); + a1->dealloc(*it, 20); + set2.erase(it); + } + } + + current_alloc_pool()->release(a1); + current_alloc_pool()->release(a2); + current_alloc_pool()->debug_check_empty(); +} + +void test_external_pointer() +{ + // Malloc does not have an external pointer querying mechanism. + auto* alloc = ThreadAlloc::get(); + + for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++) + { + size_t size = sizeclass_to_size(sc); + void* p1 = alloc->alloc(size); + + for (size_t offset = 0; offset < size; offset += 17) + { + void* p2 = (void*)((size_t)p1 + offset); + void* p3 = Alloc::external_pointer(p2); + void* p4 = Alloc::external_pointer(p2); + UNUSED(p3); + UNUSED(p4); + assert(p1 == p3); + assert((size_t)p4 == (size_t)p1 + size - 1); + } + + alloc->dealloc(p1, size); + } + + current_alloc_pool()->debug_check_empty(); +}; + +void check_offset(void* base, void* interior) +{ + void* calced_base = Alloc::external_pointer((void*)interior); + if (calced_base != (void*)base) + abort(); +} + +void check_external_pointer_large(size_t* base) +{ + size_t size = *base; + char* curr = (char*)base; + for (size_t offset = 0; offset < size; offset += 1 << 24) + { + check_offset(base, (void*)(curr + offset)); + check_offset(base, (void*)(curr + offset + (1 << 24) - 1)); + } +} + +void test_external_pointer_large() +{ + xoroshiro::p128r64 r; + + auto* alloc = ThreadAlloc::get(); + + constexpr size_t count_log = 5; + constexpr size_t count = 1 << count_log; + // Pre allocate all the objects + size_t* objects[count]; + + for (size_t i = 0; i < count; i++) + { + size_t rand = r.next() & ((1 << 28) - 1); + size_t size = (1 << 24) + rand; + // store object + objects[i] = (size_t*)alloc->alloc(size); + // Store allocators size for this object + *objects[i] = Alloc::alloc_size(objects[i]); + + check_external_pointer_large(objects[i]); + if (i > 0) + check_external_pointer_large(objects[i - 1]); + } + + for (size_t i = 0; i < count; i++) + { + check_external_pointer_large(objects[i]); + } + + // Deallocate everything + for (size_t i = 0; i < count; i++) + { + alloc->dealloc(objects[i]); + } +} + +void test_alloc_16M() +{ + auto* alloc = ThreadAlloc::get(); + // sizes >= 16M use large_alloc + const size_t size = 16'000'000; + + void* p1 = alloc->alloc(size); + assert(Alloc::alloc_size(Alloc::external_pointer(p1)) >= size); + alloc->dealloc(p1); +} + +int main(int argc, char** argv) +{ +#ifdef USE_SYSTEMATIC_TESTING + opt::Opt opt(argc, argv); + size_t seed = opt.is("--seed", 0); + Virtual::systematic_bump_ptr() += seed << 17; +#else + UNUSED(argc); + UNUSED(argv); +#endif + + test_external_pointer_large(); + test_alloc_dealloc_64k(); + test_random_allocation(); + test_calloc(); + test_double_alloc(); + test_external_pointer(); + test_alloc_16M(); + + return 0; +} diff --git a/src/test/func/rounding/rounding.cc b/src/test/func/rounding/rounding.cc new file mode 100644 index 0000000..e15cfd6 --- /dev/null +++ b/src/test/func/rounding/rounding.cc @@ -0,0 +1,36 @@ +#include +#include +#include +#include + +using namespace snmalloc; + +// Check for all sizeclass that we correctly round every offset within +// a superslab to the correct value, by comparing with the standard +// unoptimised version using division. +// Also check we correctly determine multiples using optimized check. + +int main(int argc, char** argv) +{ + UNUSED(argc); + UNUSED(argv); + + for (size_t size_class = 0; size_class < NUM_SIZECLASSES; size_class++) + { + size_t rsize = sizeclass_to_size((uint8_t)size_class); + for (size_t offset = 0; offset < SUPERSLAB_SIZE; offset++) + { + size_t rounded = (offset / rsize) * rsize; + bool mod_0 = (offset % rsize) == 0; + + size_t opt_rounded = round_by_sizeclass(rsize, offset); + if (rounded != opt_rounded) + abort(); + + bool opt_mod_0 = is_multiple_of_sizeclass(rsize, offset); + if (opt_mod_0 != mod_0) + abort(); + } + } + return 0; +} diff --git a/src/test/histogram.h b/src/test/histogram.h new file mode 100644 index 0000000..8203c79 --- /dev/null +++ b/src/test/histogram.h @@ -0,0 +1,249 @@ +#pragma once + +#ifdef USE_MEASURE +# include "../ds/flaglock.h" + +# include +# include +# include +# define MEASURE_TIME_MARKERS(id, minbits, maxbits, markers) \ + static constexpr const char* const id##_time_markers[] = markers; \ + static histogram::Global> \ + id##_time_global(#id, __FILE__, __LINE__, id##_time_markers); \ + static thread_local histogram::Histogram \ + id##_time_local(id##_time_global); \ + histogram::MeasureTime> \ + id##_time(id##_time_local); + +# define MEASURE_TIME(id, minbits, maxbits) \ + MEASURE_TIME_MARKERS(id, minbits, maxbits, {nullptr}) + +# define MARKERS(...) \ + { \ + __VA_ARGS__, nullptr \ + } + +namespace histogram +{ + using namespace snmalloc; + + template + class Global; + + template< + class V, + size_t LOW_BITS, + size_t HIGH_BITS, + size_t INTERMEDIATE_BITS = LOW_BITS> + class Histogram + { + public: + using This = Histogram; + friend Global; + + static_assert(LOW_BITS < HIGH_BITS, "LOW_BITS must be less than HIGH_BITS"); + + static constexpr V LOW = (V)((size_t)1 << LOW_BITS); + static constexpr V HIGH = (V)((size_t)1 << HIGH_BITS); + static constexpr size_t BUCKETS = + ((HIGH_BITS - LOW_BITS) << INTERMEDIATE_BITS) + 2; + + private: + V high = (std::numeric_limits::min)(); + size_t overflow; + size_t count[BUCKETS]; + + Global* global; + + public: + Histogram() : global(nullptr) {} + Histogram(Global& g) : global(&g) {} + + ~Histogram() + { + if (global != nullptr) + global->add(*this); + } + + void record(V value) + { + if (value > high) + high = value; + + if (value >= HIGH) + { + overflow++; + } + else + { + auto i = get_index(value); + assert(i < BUCKETS); + count[i]++; + } + } + + V get_high() + { + return high; + } + + size_t get_overflow() + { + return overflow; + } + + size_t get_buckets() + { + return BUCKETS; + } + + size_t get_count(size_t index) + { + if (index >= BUCKETS) + return 0; + + return count[index]; + } + + static std::pair get_range(size_t index) + { + if (index >= BUCKETS) + return std::make_pair(HIGH, HIGH); + + if (index == 0) + return std::make_pair(0, get_value(index)); + + return std::make_pair(get_value(index - 1) + 1, get_value(index)); + } + + void add(This& that) + { + high = (std::max)(high, that.high); + overflow += that.overflow; + + for (size_t i = 0; i < BUCKETS; i++) + count[i] += that.count[i]; + } + + void print(std::ostream& o) + { + o << "\tHigh: " << high << std::endl + << "\tOverflow: " << overflow << std::endl; + + size_t grand_total = overflow; + for (size_t i = 0; i < BUCKETS; i++) + grand_total += count[i]; + + size_t old_percentage = 0; + size_t cumulative_total = 0; + for (size_t i = 0; i < BUCKETS; i++) + { + auto r = get_range(i); + + cumulative_total += count[i]; + + o << "\t" << std::setfill(' ') << std::setw(6) << std::get<0>(r) << ".." + << std::setfill(' ') << std::setw(6) << std::get<1>(r) << ": " + << std::setfill(' ') << std::setw(10) << count[i]; + + auto percentage = (cumulative_total * 100 / grand_total); + if (percentage != old_percentage) + { + old_percentage = percentage; + o << std::setfill(' ') << std::setw(20) + << (cumulative_total * 100 / grand_total) << "%"; + } + + o << std::endl; + } + } + + static size_t get_index(V value) + { + return bits::to_exp_mant( + value); + } + + static V get_value(size_t index) + { + return bits:: + from_exp_mant(index); + } + }; + + template + class Global + { + private: + const char* name; + const char* file; + size_t line; + const char* const* markers; + + std::atomic_flag lock = ATOMIC_FLAG_INIT; + H aggregate; + + public: + Global( + const char* name_, + const char* file_, + size_t line_, + const char* const* markers) + : name(name_), file(file_), line(line_), markers(markers) + {} + + ~Global() + { + print(); + } + + void add(H& histogram) + { + FlagLock f(lock); + aggregate.add(histogram); + } + + private: + void print() + { + std::cout << name; + + if (markers != nullptr) + { + std::cout << ": "; + size_t i = 0; + + while (markers[i] != nullptr) + std::cout << markers[i++] << " "; + } + + std::cout << std::endl << file << ":" << line << std::endl; + + aggregate.print(std::cout); + } + }; + + template + class MeasureTime + { + private: + H& histogram; + uint64_t t; + + public: + MeasureTime(H& histogram_) : histogram(histogram_) + { + t = bits::benchmark_time_start(); + } + + ~MeasureTime() + { + histogram.record(bits::benchmark_time_end() - t); + } + }; +} + +#else +# define MEASURE_TIME(id, minbits, maxbits) +# define MEASURE_TIME_MARKERS(id, minbits, maxbits, markers) +#endif diff --git a/src/test/measuretime.h b/src/test/measuretime.h new file mode 100644 index 0000000..d2b6182 --- /dev/null +++ b/src/test/measuretime.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include +#include + +#define DO_TIME(name, code) \ + { \ + auto start__ = std::chrono::high_resolution_clock::now(); \ + code auto finish__ = std::chrono::high_resolution_clock::now(); \ + auto diff__ = finish__ - start__; \ + std::cout << name << ": " << std::setw(12) << diff__.count() << " ns" \ + << std::endl; \ + } \ No newline at end of file diff --git a/src/test/opt.h b/src/test/opt.h new file mode 100644 index 0000000..1d48e7a --- /dev/null +++ b/src/test/opt.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include +#include + +namespace opt +{ + class Opt + { + private: + int argc; + char** argv; + + public: + Opt(int argc, char** argv) : argc(argc), argv(argv) {} + + bool has(const char* opt) + { + for (int i = 1; i < argc; i++) + { + if (!strcmp(opt, argv[i])) + return true; + } + + return false; + } + + template + T is(const char* opt, T def) + { + size_t len = strlen(opt); + + for (int i = 1; i < argc; i++) + { + const char* p = param(opt, len, i); + + if (p != nullptr) + { + char* end = nullptr; + T r; + + if (std::is_unsigned::value) + r = (T)strtoull(p, &end, 10); + else + r = (T)strtoll(p, &end, 10); + + if ((r == 0) && (end == p)) + return def; + + return r; + } + } + + return def; + } + + const char* is(const char* opt, const char* def) + { + size_t len = strlen(opt); + + for (int i = 1; i < argc; i++) + { + const char* p = param(opt, len, i); + + if (p != nullptr) + return p; + } + + return def; + } + + private: + const char* param(const char* opt, size_t len, int i) + { + if (strncmp(opt, argv[i], len)) + return nullptr; + + switch (argv[i][len]) + { + case '\0': + return (i < (argc - 1)) ? argv[i + 1] : nullptr; + case '=': + return &argv[i][len + 1]; + default: + return nullptr; + } + } + }; +} diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc new file mode 100644 index 0000000..d799baf --- /dev/null +++ b/src/test/perf/contention/contention.cc @@ -0,0 +1,176 @@ +#include "test/measuretime.h" +#include "test/opt.h" +#include "test/usage.h" +#include "test/xoroshiro.h" + +#include +#include +#include +#include +#include + +using namespace snmalloc; + +bool use_malloc = false; + +template +class ParallelTest +{ +private: + std::atomic flag = false; + std::atomic ready = 0; + uint64_t start; + uint64_t end; + std::atomic complete = 0; + + size_t cores; + + void run(size_t id) + { + auto prev = ready.fetch_add(1); + if (prev + 1 == cores) + { + start = bits::tick(); + flag = true; + } + while (!flag) + bits::pause(); + + f(id); + + prev = complete.fetch_add(1); + if (prev + 1 == cores) + { + end = bits::tick(); + } + } + +public: + ParallelTest(size_t cores) : cores(cores) + { + std::thread* t = new std::thread[cores]; + + for (size_t i = 0; i < cores; i++) + { + t[i] = std::thread(&ParallelTest::run, this, i); + } + // Wait for all the threads. + for (size_t i = 0; i < cores; i++) + { + t[i].join(); + } + + delete[] t; + } + + uint64_t time() + { + return end - start; + } +}; + +std::atomic* contention; +size_t swapsize; +size_t swapcount; + +void test_tasks_f(size_t id) +{ + Alloc* a = ThreadAlloc::get(); + xoroshiro::p128r32 r(id + 5000); + + for (size_t n = 0; n < swapcount; n++) + { + size_t size = 16 + (r.next() % 1024); + size_t* res = (size_t*)(use_malloc ? malloc(size) : a->alloc(size)); + + *res = size; + size_t* out = + contention[n % swapsize].exchange(res, std::memory_order_relaxed); + + if (out != nullptr) + { + size = *out; + if (use_malloc) + free(out); + else + a->dealloc(out, size); + } + } +}; + +void test_tasks(size_t num_tasks, size_t count, size_t size) +{ + Alloc* a = ThreadAlloc::get(); + + contention = new std::atomic[size]; + xoroshiro::p128r32 r; + + for (size_t n = 0; n < size; n++) + { + size_t alloc_size = 16 + (r.next() % 1024); + size_t* res = + (size_t*)(use_malloc ? malloc(alloc_size) : a->alloc(alloc_size)); + *res = alloc_size; + contention[n] = res; + } + swapcount = count; + swapsize = size; + +#ifdef USE_SNMALLOC_STATS + Stats s0; + current_alloc_pool()->aggregate_stats(s0); +#endif + + { + ParallelTest test(num_tasks); + + std::cout << "Task test, " << num_tasks << " threads, " << count + << " swaps per thread " << test.time() << "ticks" << std::endl; + + for (size_t n = 0; n < swapsize; n++) + { + if (contention[n] != nullptr) + { + if (use_malloc) + free(contention[n]); + else + a->dealloc(contention[n], *contention[n]); + } + } + + delete[] contention; + } + +#ifndef NDEBUG + current_alloc_pool()->debug_check_empty(); +#endif +}; + +int main(int argc, char** argv) +{ + opt::Opt opt(argc, argv); + size_t cores = opt.is("--cores", 8); + + size_t count = opt.is("--swapcount", 1 << 20); + size_t size = opt.is("--swapsize", 1 << 18); + use_malloc = opt.has("--use_malloc"); + + std::cout << "Allocator is " << (use_malloc ? "System" : "snmalloc") + << std::endl; + + for (size_t i = cores; i > 0; i >>= 1) + test_tasks(i, count, size); + + if (opt.has("--stats")) + { +#ifdef USE_SNMALLOC_STATS + Stats s; + current_alloc_pool()->aggregate_stats(s); + s.print(std::cout); +#endif + + usage::print_memory(); + } + + return 0; +} diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc new file mode 100644 index 0000000..9ec8cce --- /dev/null +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -0,0 +1,78 @@ +#include +#include +#include +#include + +using namespace snmalloc; + +constexpr size_t count_log = 20; +constexpr size_t count = 1 << count_log; +// Pre allocate all the objects +size_t* objects[count]; + +NOINLINE void setup(xoroshiro::p128r64& r, Alloc* alloc) +{ + for (size_t i = 0; i < count; i++) + { + size_t rand = (size_t)r.next(); + size_t offset = bits::clz(rand); + if (offset > 30) + offset = 30; + size_t size = (rand & 15) << offset; + if (size < 16) + size = 16; + // store object + objects[i] = (size_t*)alloc->alloc(size); + // Store allocators size for this object + *objects[i] = Alloc::alloc_size(objects[i]); + } +} + +NOINLINE void teardown(Alloc* alloc) +{ + // Deallocate everything + for (size_t i = 0; i < count; i++) + { + alloc->dealloc(objects[i]); + } + + current_alloc_pool()->debug_check_empty(); +} + +void test_external_pointer(xoroshiro::p128r64& r) +{ + auto* alloc = ThreadAlloc::get(); + + setup(r, alloc); + + DO_TIME("External pointer queries ", { + for (size_t i = 0; i < 10000000; i++) + { + size_t rand = (size_t)r.next(); + size_t oid = rand & (((size_t)1 << count_log) - 1); + size_t* external_ptr = objects[oid]; + size_t size = *external_ptr; + size_t offset = (size >> 4) * (rand & 15); + size_t interior_ptr = ((size_t)external_ptr) + offset; + void* calced_external = Alloc::external_pointer((void*)interior_ptr); + if (calced_external != external_ptr) + abort(); + } + }); + + teardown(alloc); +} + +int main(int, char**) +{ + xoroshiro::p128r64 r; +#if NDEBUG + size_t nn = 30; +#else + size_t nn = 3; +#endif + + for (size_t n = 0; n < nn; n++) + test_external_pointer(r); + return 0; +} diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc new file mode 100644 index 0000000..3b3700c --- /dev/null +++ b/src/test/perf/singlethread/singlethread.cc @@ -0,0 +1,83 @@ +#include +#include +#include + +using namespace snmalloc; + +template +void test_alloc_dealloc(size_t count, size_t size, bool write) +{ + auto* alloc = ThreadAlloc::get(); + + DO_TIME( + "Count: " << std::setw(6) << count << ", Size: " << std::setw(6) << size + << ", ZeroMem: " << (zero_mem == YesZero) << ", Write: " << write, + { + std::unordered_set set; + + // alloc 1.5x objects + for (size_t i = 0; i < ((count * 3) / 2); i++) + { + void* p = alloc->alloc(size); + assert(set.find(p) == set.end()); + + if (write) + *(int*)p = 4; + + set.insert(p); + } + + // free 0.25x of the objects + for (size_t i = 0; i < (count / 4); i++) + { + auto it = set.begin(); + void* p = *it; + alloc->dealloc(p, size); + set.erase(it); + assert(set.find(p) == set.end()); + } + + // alloc 1x objects + for (size_t i = 0; i < count; i++) + { + void* p = alloc->alloc(size); + assert(set.find(p) == set.end()); + + if (write) + *(int*)p = 4; + + set.insert(p); + } + + // free everything + while (!set.empty()) + { + auto it = set.begin(); + alloc->dealloc(*it, size); + set.erase(it); + } + }); + + current_alloc_pool()->debug_check_empty(); +} + +int main(int, char**) +{ + for (size_t size = 16; size <= 128; size <<= 1) + { + test_alloc_dealloc(1 << 15, size, false); + test_alloc_dealloc(1 << 15, size, true); + test_alloc_dealloc(1 << 15, size, false); + test_alloc_dealloc(1 << 15, size, true); + } + + for (size_t size = 1 << 12; size <= 1 << 17; size <<= 1) + { + test_alloc_dealloc(1 << 10, size, false); + test_alloc_dealloc(1 << 10, size, true); + test_alloc_dealloc(1 << 10, size, false); + test_alloc_dealloc(1 << 10, size, true); + } + + return 0; +} diff --git a/src/test/usage.h b/src/test/usage.h new file mode 100644 index 0000000..c48a631 --- /dev/null +++ b/src/test/usage.h @@ -0,0 +1,42 @@ +#pragma once + +#if defined(_WIN32) +# define WIN32_LEAN_AND_MEAN +# define NOMINMAX +# include +// Needs to be included after windows.h +# include +#endif + +#include +#include + +namespace usage +{ + void print_memory() + { +#if defined(_WIN32) + PROCESS_MEMORY_COUNTERS_EX pmc; + + if (!GetProcessMemoryInfo( + GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) + return; + + std::cout << "Memory info:" << std::endl + << "\tPageFaultCount: " << pmc.PageFaultCount << std::endl + << "\tPeakWorkingSetSize: " << pmc.PeakWorkingSetSize << std::endl + << "\tWorkingSetSize: " << pmc.WorkingSetSize << std::endl + << "\tQuotaPeakPagedPoolUsage: " << pmc.QuotaPeakPagedPoolUsage + << std::endl + << "\tQuotaPagedPoolUsage: " << pmc.QuotaPagedPoolUsage + << std::endl + << "\tQuotaPeakNonPagedPoolUsage: " + << pmc.QuotaPeakNonPagedPoolUsage << std::endl + << "\tQuotaNonPagedPoolUsage: " << pmc.QuotaNonPagedPoolUsage + << std::endl + << "\tPagefileUsage: " << pmc.PagefileUsage << std::endl + << "\tPeakPagefileUsage: " << pmc.PeakPagefileUsage << std::endl + << "\tPrivateUsage: " << pmc.PrivateUsage << std::endl; +#endif + } +}; diff --git a/src/test/xoroshiro.h b/src/test/xoroshiro.h new file mode 100644 index 0000000..092262b --- /dev/null +++ b/src/test/xoroshiro.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include + +namespace xoroshiro +{ + namespace detail + { + template + class XorOshiro + { + private: + static constexpr unsigned STATE_BITS = 8 * sizeof(STATE); + static constexpr unsigned RESULT_BITS = 8 * sizeof(RESULT); + + static_assert( + STATE_BITS >= RESULT_BITS, + "STATE must have at least as many bits as RESULT"); + + STATE x; + STATE y; + + static inline STATE rotl(STATE x, STATE k) + { + return (x << k) | (x >> (STATE_BITS - k)); + } + + public: + XorOshiro(STATE x_ = 5489, STATE y_ = 0) : x(x_), y(y_) + { + // If both zero, then this does not work + if (x_ == 0 && y_ == 0) + abort(); + + next(); + } + + void set_state(STATE x_, STATE y_ = 0) + { + // If both zero, then this does not work + if (x_ == 0 && y_ == 0) + abort(); + + x = x_; + y = y_; + next(); + } + + RESULT next() + { + STATE r = x + y; + y ^= x; + x = rotl(x, A) ^ y ^ (y << B); + y = rotl(y, C); + // If both zero, then this does not work + if (x == 0 && y == 0) + abort(); + return r >> (STATE_BITS - RESULT_BITS); + } + }; + } + + using p128r64 = detail::XorOshiro; + using p128r32 = detail::XorOshiro; + using p64r32 = detail::XorOshiro; + using p64r16 = detail::XorOshiro; + using p32r16 = detail::XorOshiro; + using p32r8 = detail::XorOshiro; + using p16r8 = detail::XorOshiro; +}