Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit of snmalloc

Browse Source 
History squashed from internal development.

Internal history has commit hash:
  e27a0e485c44a5003a802de2661ce3b21e120316
This commit is contained in:
Matthew Parkinson
2019-01-15 14:17:55 +00:00
parent e488c24784
commit 4f9d991449
53 changed files with 6940 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
.clang-format Normal file
View File

@@ -0,0 +1,110 @@
---
Language: Cpp
# BasedOnStyle: LLVM
AccessModifierOffset: -2
AlignAfterOpenBracket: AlwaysBreak
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: false
AlignEscapedNewlines: DontAlign
AlignOperands: false
AlignTrailingComments: false
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Empty
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: true
AlwaysBreakTemplateDeclarations: true
BinPackArguments: false
BinPackParameters: false
BraceWrapping:
AfterClass: true
AfterControlStatement: true
AfterEnum: true
AfterFunction: true
AfterNamespace: true
AfterObjCDeclaration: true
AfterStruct: true
AfterUnion: true
AfterExternBlock: true
BeforeCatch: true
BeforeElse: true
IndentBraces: false
SplitEmptyFunction: false
SplitEmptyRecord: false
SplitEmptyNamespace: false
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Custom
BreakBeforeInheritanceComma: false
BreakBeforeTernaryOperators: false
BreakConstructorInitializersBeforeComma: false
BreakConstructorInitializers: BeforeColon
BreakAfterJavaFieldAnnotations: false
BreakStringLiterals: true
ColumnLimit: 80
CommentPragmas: '^ IWYU pragma:'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 0
ContinuationIndentWidth: 2
Cpp11BracedListStyle: true
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: false
ForEachMacros:
- Q_FOREACH
- BOOST_FOREACH
IncludeBlocks: Regroup
IncludeCategories:
- Regex: '^"(llvm|llvm-c|clang|clang-c)/'
Priority: 2
- Regex: '^(<|"(gtest|gmock|isl|json)/)'
Priority: 3
- Regex: '.*'
Priority: 1
IncludeIsMainRegex: '(Test)?$'
IndentCaseLabels: true
IndentPPDirectives: AfterHash
IndentWidth: 2
IndentWrappedFunctionNames: false
JavaScriptQuotes: Leave
JavaScriptWrapImports: true
KeepEmptyLinesAtTheStartOfBlocks: false
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 1
NamespaceIndentation: All
ObjCBlockIndentWidth: 2
ObjCSpaceAfterProperty: false
ObjCSpaceBeforeProtocolList: true
PenaltyBreakAssignment: 2
PenaltyBreakBeforeFirstCallParameter: 19
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 60
PointerAlignment: Left
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: false
SpaceAfterTemplateKeyword: false
SpaceBeforeAssignmentOperators: true
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInContainerLiterals: false
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Cpp11
TabWidth: 2
UseTab: Never
...

333
.gitignore vendored
View File

@@ -1,330 +1,5 @@
## Ignore Visual Studio temporary files, build results, and build*/
## files generated by popular Visual Studio add-ons. CMakeFiles/
## .vscode/
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
# Visual Studio 2015/2017 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# Visual Studio 2017 auto generated files
Generated\ Files/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# Benchmark Results
BenchmarkDotNet.Artifacts/
# .NET Core
project.lock.json
project.fragment.lock.json
artifacts/
**/Properties/launchSettings.json
# StyleCop
StyleCopReport.xml
# Files built by Visual Studio
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.iobj
*.pch
*.pdb
*.ipdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# Visual Studio Trace Files
*.e2e
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# AxoCover is a Code Coverage Tool
.axoCover/*
!.axoCover/settings.json
# Visual Studio code coverage results
*.coverage
*.coveragexml
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# Note: Comment the next line if you want to checkin your web deploy settings,
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/[Pp]ackages/*
# except build/, which is used as an MSBuild target.
!**/[Pp]ackages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/[Pp]ackages/repositories.config
# NuGet v3's project.json files produces more ignorable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
*.appx
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
~$*
*~ *~
*.dbmdl *.sw?
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
orleans.codegen.cs
# Including strong name files can present a security risk
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
#*.snk
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
ServiceFabricBackup/
*.rptproj.bak
# SQL Server files
*.mdf
*.ldf
*.ndf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
*.rptproj.rsuser
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
node_modules/
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
*.vbw
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# JetBrains Rider
.idea/
*.sln.iml
# CodeRush
.cr/
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
# Cake - Uncomment if you are using it
# tools/**
# !tools/packages.config
# Tabs Studio
*.tss
# Telerik's JustMock configuration file
*.jmconfig
# BizTalk build output
*.btp.cs
*.btm.cs
*.odx.cs
*.xsd.cs
# OpenCover UI analysis results
OpenCover/
# Azure Stream Analytics local run output
ASALocalRun/
# MSBuild Binary and Structured Log
*.binlog
# NVidia Nsight GPU debugger configuration file
*.nvuser
# MFractors (Xamarin productivity tool) working folder
.mfractor/

92
.vsts-ci.yml Normal file
View File

@@ -0,0 +1,92 @@
resources:
- repo: self
phases:
- phase: Linux
queue:
name: 'Hosted Ubuntu 1604'
parallel: 2
matrix:
Debug:
BuildType: Debug
Release:
BuildType: Release
steps:
- script: |
sudo add-apt-repository ppa:ubuntu-toolchain-r/test
sudo apt-get update
sudo apt-get install -y ninja-build libc++-dev libc++abi-dev libc++abi1 libstdc++-7-dev
# sudo apt-get install clang-6.0 clang++-6.0
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
sudo update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100
sudo update-alternatives --set cc /usr/bin/clang
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100
sudo update-alternatives --set c++ /usr/bin/clang++
displayName: 'Install Build Dependencies'
- task: CMake@1
displayName: 'CMake .. -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="-stdlib=libstdc++ -std=c++17"'
inputs:
cmakeArgs: '.. -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="-stdlib=libstdc++ -std=c++17"'
- script: |
ninja
ctest -j 4 --output-on-failure
workingDirectory: build
failOnStderr: true
displayName: 'Compile & Test'
- script: |
sudo cp libsnmallocshim.so /usr/local/lib/
ninja clean
LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja
workingDirectory: build
failOnStderr: true
displayName: 'LD_PRELOAD Compile'
- phase: Windows
queue:
name: 'Hosted VS2017'
parallel: 2
matrix:
Debug:
BuildType: Debug
Release:
BuildType: Release
steps:
- task: CMake@1
displayName: 'CMake .. -G"Visual Studio 15 2017 Win64"'
inputs:
cmakeArgs: '.. -G"Visual Studio 15 2017 Win64"'
- task: MSBuild@1
displayName: 'Build solution build/snmalloc.sln'
inputs:
solution: build/snmalloc.sln
msbuildArguments: '/m /p:Configuration=$(BuildType)'
- script: 'ctest -j 4 --interactive-debug-mode 0 --output-on-failure'
workingDirectory: build
displayName: 'Run Ctest'
- phase: Format
queue:
name: 'Hosted Ubuntu 1604'
steps:
- script: |
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main"
sudo apt-get update
sudo apt-get install -y clang-format-6.0
sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-6.0 100
bash check-format.sh
displayName: 'Check Format'

107
CMakeLists.txt Normal file
View File

@@ -0,0 +1,107 @@
cmake_minimum_required(VERSION 3.8)
project(snmalloc C CXX)
option(USE_SNMALLOC_STATS "Track allocation stats" OFF)
option(USE_MEASURE "Measure performance with histograms" OFF)
option(USE_SBRK "Use sbrk instead of mmap" OFF)
macro(subdirlist result curdir)
file(GLOB children LIST_DIRECTORIES true RELATIVE ${curdir} ${curdir}/*)
set(dirlist "")
foreach(child ${children})
if(IS_DIRECTORY ${curdir}/${child})
list(APPEND dirlist ${child})
endif()
endforeach()
set(${result} ${dirlist})
endmacro()
macro(linklibs project)
if(NOT MSVC)
target_link_libraries(${project} ${CMAKE_THREAD_LIBS_INIT})
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
target_link_libraries(${project} atomic)
endif()
endif()
endmacro()
if(MSVC)
add_compile_options(/WX /W4 /wd4127 /wd4324 /wd4201 /std:c++latest)
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi")
set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG")
else()
find_package(Threads REQUIRED)
add_compile_options(-mcx16 -march=native -Wall -Wextra -Werror -g)
endif()
set(CMAKE_CXX_STANDARD 17)
if(USE_SNMALLOC_STATS)
add_definitions(-DUSE_SNMALLOC_STATS)
endif()
if(USE_MEASURE)
add_definitions(-DUSE_MEASURE)
endif()
if(USE_SBRK)
add_definitions(-DUSE_SBRK)
endif()
if(NOT MSVC)
add_library(snmallocshim SHARED src/override/malloc.cc)
target_link_libraries(snmallocshim -pthread)
target_include_directories(snmallocshim PRIVATE src)
endif()
enable_testing()
set(TESTDIR ${CMAKE_CURRENT_SOURCE_DIR}/src/test)
subdirlist(TEST_CATEGORIES ${TESTDIR})
foreach(TEST_CATEGORY ${TEST_CATEGORIES})
subdirlist(TESTS ${TESTDIR}/${TEST_CATEGORY})
foreach(TEST ${TESTS})
unset(SRC)
aux_source_directory(${TESTDIR}/${TEST_CATEGORY}/${TEST} SRC)
set(TESTNAME "${TEST_CATEGORY}-${TEST}")
add_executable(${TESTNAME} ${SRC} src/override/new.cc)
target_include_directories(${TESTNAME} PRIVATE src)
linklibs(${TESTNAME})
add_test(${TESTNAME} ${TESTNAME})
endforeach()
endforeach()
# The clang-format tool is installed under a variety of different names. Try
# to find a sensible one. Only look for 6.0 and 7.0 versions explicitly - we
# don't know whether our clang-format file will work with newer versions of the
# tool
set(CLANG_FORMAT_NAMES
clang-format-7.0
clang-format-6.0
clang-format70
clang-format60
clang-format)
# Loop over each of the possible names of clang-format and try to find one.
set(CLANG_FORMAT CLANG_FORMAT-NOTFOUND)
foreach (NAME IN ITEMS ${CLANG_FORMAT_NAMES})
if (${CLANG_FORMAT} STREQUAL "CLANG_FORMAT-NOTFOUND")
find_program(CLANG_FORMAT ${NAME})
endif ()
endforeach()
# If we've found a clang-format tool, generate a target for it, otherwise emit
# a warning.
if (${CLANG_FORMAT} STREQUAL "CLANG_FORMAT-NOTFOUND")
message(WARNING "Not generating clangformat target, no clang-format tool found")
else ()
message(STATUS "Generating clangformat target using ${CLANG_FORMAT}")
file(GLOB_RECURSE ALL_SOURCE_FILES *.cc *.h *.hh)
add_custom_target(
clangformat
COMMAND ${CLANG_FORMAT}
-i
${ALL_SOURCE_FILES}
)
endif()

78
README.md
View File

@@ -1,3 +1,81 @@
# snmalloc
snmalloc is a research allocator. Its key design features are:
* Memory that is freed by the same thread that allocated it does not require any
synchronising operations.
* Freeing memory in a different thread to initially allocated it, does not take
any locks and instead uses a novel message passing scheme to return the
memory to the original allocator, where it is recycled.
* The allocator uses large ranges of pages to reduce the amount of meta-data
required.
# Building on Windows
The Windows build currently depends on Visual Studio 2017.
To build with Visual Studio:
```
mkdir build
cd build
cmake -G "Visual Studio 15 2017 Win64" ..
cmake --build . --config Debug
cmake --build . --config Release
cmake --build . --config RelWithDebInfo
```
You can also omit the last three steps and build from the IDE.
Visual Studio builds use a separate directory to keep the binaries for each
build configuration.
Alternatively, you can follow the steps in the next section to build with Ninja
using the Visual Studio compiler.
# Building on Linux or FreeBSD
Snmalloc has very few dependencies, CMake, Ninja, Clang 6.0 or later and a C++17
standard library.
Building with GCC is currently not supported because GCC lacks support for the
`selectany` attribute to specify variables in a COMDAT.
To build a debug configuration:
```
mkdir build
cd build
cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Debug
ninja
```
To build a release configuration:
```
mkdir build
cd build
cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release
ninja
```
To build with optimizations on, but with debug information:
```
mkdir build
cd build
cmake -G Ninja .. -DCMAKE_BUILD_TYPE=RelWithDebInfo
ninja
```
The build produces a binary `libsnmallocshim.so`. This file can be
`LD_PRELOAD`ed to use the allocator in place of the system allocator, for
example, you can run the build script using the snmalloc as the allocator for
your toolchain:
```
LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja
```
# CMake Feature Flags
These can be added to your cmake command line.
```
-DUSE_SNMALLOC_STATS=ON // Track allocation stats
-DUSE_MEASURE=ON // Measure performance with histograms
```
# Contributing # Contributing

18
check-format.sh Normal file
View File

@@ -0,0 +1,18 @@
set -u
unformatted_files=""
for f in `find . -name *.h -o -name *.hh -o -name *.cc`; do
d=`clang-format -style=file $f | diff $f -`
if [ "$d" != "" ]; then
if [ "$unformatted_files" != "" ]; then
unformatted_files+=$'\n'
fi
unformatted_files+="$f"
fi
done
if [ "$unformatted_files" != "" ]; then
echo "$unformatted_files"
exit 1
fi

113
src/ds/aba.h Normal file
View File

@@ -0,0 +1,113 @@
#pragma once
#include "bits.h"
namespace snmalloc
{
template<typename T, Construction c = RequiresInit>
class ABA
{
public:
#ifdef PLATFORM_IS_X86
struct alignas(2 * sizeof(std::size_t)) Linked
{
T* ptr;
uintptr_t aba;
};
struct Independent
{
std::atomic<T*> ptr;
std::atomic<uintptr_t> aba;
};
static_assert(
sizeof(Linked) == sizeof(Independent),
"Expecting identical struct sizes in union");
static_assert(
sizeof(Linked) == (2 * sizeof(std::size_t)),
"Expecting ABA to be the size of two pointers");
using Cmp = Linked;
#else
using Cmp = T*;
#endif
private:
#ifdef PLATFORM_IS_X86
union
{
alignas(2 * sizeof(std::size_t)) std::atomic<Linked> linked;
Independent independent;
};
#else
std::atomic<T*> ptr;
#endif
public:
ABA()
{
if constexpr (c == RequiresInit)
init(nullptr);
}
void init(T* x)
{
#ifdef PLATFORM_IS_X86
independent.ptr.store(x, std::memory_order_relaxed);
independent.aba.store(0, std::memory_order_relaxed);
#else
ptr.store(x, std::memory_order_relaxed);
#endif
}
T* peek()
{
return independent.ptr.load(std::memory_order_relaxed);
}
Cmp read()
{
return
#ifdef PLATFORM_IS_X86
Cmp{independent.ptr.load(std::memory_order_relaxed),
independent.aba.load(std::memory_order_relaxed)};
#else
ptr.load(std::memory_order_relaxed);
#endif
}
static T* load(Cmp& from)
{
#ifdef PLATFORM_IS_X86
return from.ptr;
#else
return from;
#endif
}
bool compare_exchange(Cmp& expect, T* value)
{
#ifdef PLATFORM_IS_X86
# if defined(_MSC_VER) && defined(PLATFORM_BITS_64)
return _InterlockedCompareExchange128(
(volatile __int64*)&linked,
expect.aba + 1,
(__int64)value,
(__int64*)&expect);
# else
# if defined(__GNUC__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)
#error You must compile with -mcx16 to enable 16-bit atomic compare and swap.
# endif
Cmp xchg{value, expect.aba + 1};
return linked.compare_exchange_weak(
expect, xchg, std::memory_order_relaxed, std::memory_order_relaxed);
# endif
#else
return ptr.compare_exchange_weak(
expect, value, std::memory_order_relaxed, std::memory_order_relaxed);
#endif
}
};
}

466
src/ds/bits.h Normal file
View File

@@ -0,0 +1,466 @@
#pragma once
#include <stddef.h>
#ifdef _MSC_VER
# include <immintrin.h>
# include <intrin.h>
# define ALWAYSINLINE __forceinline
# define NOINLINE __declspec(noinline)
# define HEADER_GLOBAL __declspec(selectany)
#else
# include <cpuid.h>
# include <emmintrin.h>
# define ALWAYSINLINE __attribute__((always_inline))
# define NOINLINE __attribute__((noinline))
# define HEADER_GLOBAL __attribute__((selectany))
#endif
#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \
defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
defined(_M_AMD64)
# define PLATFORM_IS_X86
# if defined(__linux__) && !defined(OPEN_ENCLAVE)
# include <x86intrin.h>
# endif
# if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
defined(_M_AMD64)
# define PLATFORM_BITS_64
# else
# define PLATFORM_BITS_32
# endif
#endif
#if defined(_MSC_VER) && defined(PLATFORM_BITS_32)
# include <intsafe.h>
#endif
#ifndef __has_builtin
# define __has_builtin(x) 0
#endif
#define UNUSED(x) ((void)x)
// #define USE_LZCNT
#include <atomic>
#include <cassert>
#include <cstdint>
#include <type_traits>
#ifdef pause
# undef pause
#endif
namespace snmalloc
{
// Used to enable trivial constructors for
// class that zero init is sufficient.
// Supplying PreZeroed means the memory is pre-zeroed i.e. a global section
// RequiresInit is if the class needs to zero its fields.
enum Construction
{
PreZeroed,
RequiresInit
};
namespace bits
{
static constexpr size_t BITS = sizeof(size_t) * 8;
static constexpr bool is64()
{
return BITS == 64;
}
static constexpr size_t ADDRESS_BITS = is64() ? 48 : 32;
inline void pause()
{
#if defined(PLATFORM_IS_X86)
_mm_pause();
#else
# warning "Missing pause intrinsic"
#endif
}
inline uint64_t tick()
{
#if defined(PLATFORM_IS_X86)
# if defined(_MSC_VER)
return __rdtsc();
# elif defined(__clang__)
return __builtin_readcyclecounter();
# else
return __builtin_ia32_rdtsc();
# endif
#else
# error Define CPU tick for this platform
#endif
}
inline uint64_t tickp()
{
#if defined(PLATFORM_IS_X86)
# if defined(_MSC_VER)
unsigned int aux;
return __rdtscp(&aux);
# else
unsigned aux;
return __builtin_ia32_rdtscp(&aux);
# endif
#else
# error Define CPU tick for this platform
#endif
}
inline void halt_out_of_order()
{
#if defined(PLATFORM_IS_X86)
# if defined(_MSC_VER)
int cpu_info[4];
__cpuid(cpu_info, 0);
# else
unsigned int eax, ebx, ecx, edx;
__get_cpuid(0, &eax, &ebx, &ecx, &edx);
# endif
#else
# error Define CPU benchmark start time for this platform
#endif
}
inline uint64_t benchmark_time_start()
{
halt_out_of_order();
return tick();
}
inline uint64_t benchmark_time_end()
{
uint64_t t = tickp();
halt_out_of_order();
return t;
}
inline size_t clz(size_t x)
{
#if defined(_MSC_VER)
# ifdef USE_LZCNT
# ifdef PLATFORM_BITS_64
return __lzcnt64(x);
# else
return __lzcnt((uint32_t)x);
# endif
# else
unsigned long index;
# ifdef PLATFORM_BITS_64
_BitScanReverse64(&index, x);
# else
_BitScanReverse(&index, (unsigned long)x);
# endif
return BITS - index - 1;
# endif
#else
return (size_t)__builtin_clzl(x);
#endif
}
inline constexpr size_t rotr_const(size_t x, size_t n)
{
size_t nn = n & (BITS - 1);
return (x >> nn) | (x << (((size_t) - (int)nn) & (BITS - 1)));
}
inline constexpr size_t rotl_const(size_t x, size_t n)
{
size_t nn = n & (BITS - 1);
return (x << nn) | (x >> (((size_t) - (int)nn) & (BITS - 1)));
}
inline size_t rotr(size_t x, size_t n)
{
#if defined(_MSC_VER)
# ifdef PLATFORM_BITS_64
return _rotr64(x, (int)n);
# else
return _rotr((uint32_t)x, (int)n);
# endif
#else
return rotr_const(x, n);
#endif
}
inline size_t rotl(size_t x, size_t n)
{
#if defined(_MSC_VER)
# ifdef PLATFORM_BITS_64
return _rotl64(x, (int)n);
# else
return _rotl((uint32_t)x, (int)n);
# endif
#else
return rotl_const(x, n);
#endif
}
constexpr size_t clz_const(size_t x)
{
size_t n = 0;
for (int i = BITS - 1; i >= 0; i--)
{
size_t mask = (size_t)1 << i;
if ((x & mask) == mask)
return n;
n++;
}
return n;
}
inline size_t ctz(size_t x)
{
#if defined(_MSC_VER)
# ifdef PLATFORM_BITS_64
return _tzcnt_u64(x);
# else
return _tzcnt_u32((uint32_t)x);
# endif
#else
return (size_t)__builtin_ctzl(x);
#endif
}
constexpr size_t ctz_const(size_t x)
{
size_t n = 0;
for (size_t i = 0; i < BITS; i++)
{
size_t mask = (size_t)1 << i;
if ((x & mask) == mask)
return n;
n++;
}
return n;
}
inline size_t umul(size_t x, size_t y, bool& overflow)
{
#if __has_builtin(__builtin_mul_overflow)
size_t prod;
overflow = __builtin_mul_overflow(x, y, &prod);
return prod;
#elif defined(_MSC_VER)
# if defined(PLATFORM_BITS_64)
size_t high_prod;
size_t prod = _umul128(x, y, &high_prod);
overflow = high_prod != 0;
return prod;
# else
size_t prod;
overflow = S_OK == UIntMult(x, y, &prod);
return prod;
# endif
#else
size_t prod = x * y;
return y && (x > ((size_t)-1 / y));
#endif
}
inline size_t next_pow2(size_t x)
{
// Correct for numbers [0..MAX_SIZE >> 1).
// Returns 1 for x > (MAX_SIZE >> 1).
if (x <= 2)
return x;
return (size_t)1 << (BITS - clz(x - 1));
}
inline size_t next_pow2_bits(size_t x)
{
// Correct for numbers [1..MAX_SIZE].
// Returns 64 for 0. Approximately 2 cycles.
return BITS - clz(x - 1);
}
constexpr size_t next_pow2_const(size_t x)
{
if (x <= 2)
return x;
return (size_t)1 << (BITS - clz_const(x - 1));
}
constexpr size_t next_pow2_bits_const(size_t x)
{
return BITS - clz_const(x - 1);
}
inline static size_t hash(void* p)
{
size_t x = (size_t)p;
if (is64())
{
x = ~x + (x << 21);
x = x ^ (x >> 24);
x = (x + (x << 3)) + (x << 8);
x = x ^ (x >> 14);
x = (x + (x << 2)) + (x << 4);
x = x ^ (x >> 28);
x = x + (x << 31);
}
else
{
x = ~x + (x << 15);
x = x ^ (x >> 12);
x = x + (x << 2);
x = x ^ (x >> 4);
x = (x + (x << 3)) + (x << 11);
x = x ^ (x >> 16);
}
return x;
}
static inline size_t align_down(size_t value, size_t alignment)
{
assert(next_pow2(alignment) == alignment);
size_t align_1 = alignment - 1;
value &= ~align_1;
return value;
}
static inline size_t align_up(size_t value, size_t alignment)
{
assert(next_pow2(alignment) == alignment);
size_t align_1 = alignment - 1;
value += align_1;
value &= ~align_1;
return value;
}
template<size_t alignment>
static inline bool is_aligned_block(void* p, size_t size)
{
assert(next_pow2(alignment) == alignment);
return (((size_t)p | size) & (alignment - 1)) == 0;
}
template<class T>
constexpr T inc_mod(T v, T mod)
{
static_assert(
std::is_integral<T>::value, "inc_mod can only be used on integers");
using S = std::make_signed_t<T>;
constexpr S shift = (sizeof(S) * 8) - 1;
S a = (S)(v + 1);
S b = (S)(mod - a - 1);
return a & ~(b >> shift);
}
/************************************************
*
* Map large range of strictly positive integers
* into an exponent and mantissa pair.
*
* The reverse mapping is given as:
*
* e | m | value
* ---------------------------------
* 0 | x1 ... xm | 0..00 x1 .. xm
* 1 | x1 ... xm | 0..01 x1 .. xm
* 2 | x1 ... xm | 0..1 x1 .. xm 0
* 3 | x1 ... xm | 0.1 x1 .. xm 00
*
* The forward mapping maps a value to the
* smallest exponent and mantissa with a
* reverse mapping not less than the value.
*
* Does not work for value=0.
***********************************************/
template<size_t MANTISSA_BITS, size_t LOW_BITS = 0>
static size_t to_exp_mant(size_t value)
{
value += ((size_t)1 << (LOW_BITS)) - 1;
value >>= LOW_BITS;
if (MANTISSA_BITS > 0)
{
size_t LEADING_BIT = ((size_t)1 << MANTISSA_BITS) >> 1;
size_t MANTISSA_MASK = ((size_t)1 << MANTISSA_BITS) - 1;
value = value - 1;
size_t e = (bits::BITS - clz(value | LEADING_BIT)) - MANTISSA_BITS;
size_t shift_e = (e == 0) ? 0 : e - 1;
size_t m = (value >> shift_e) & MANTISSA_MASK;
return (e << MANTISSA_BITS) + m;
}
else
{
return bits::next_pow2_bits(value);
}
}
template<size_t MANTISSA_BITS, size_t LOW_BITS = 0>
constexpr static size_t to_exp_mant_const(size_t value)
{
value += ((size_t)1 << LOW_BITS) - 1;
value >>= LOW_BITS;
if (MANTISSA_BITS > 0)
{
size_t LEADING_BIT = (size_t)1 << (MANTISSA_BITS - 1);
size_t MANTISSA_MASK = ((size_t)1 << MANTISSA_BITS) - 1;
value = value - 1;
size_t e =
(bits::BITS - clz_const(value | LEADING_BIT)) - MANTISSA_BITS;
size_t shift_e = (e == 0) ? 0 : e - 1;
size_t m = (value >> shift_e) & MANTISSA_MASK;
return (e << MANTISSA_BITS) + m;
}
else
{
return bits::next_pow2_bits_const(value);
}
}
template<size_t MANTISSA_BITS, size_t LOW_BITS = 0>
constexpr static size_t from_exp_mant(size_t m_e)
{
if (MANTISSA_BITS > 0)
{
size_t MANTISSA_MASK = ((size_t)1 << MANTISSA_BITS) - 1;
size_t m = m_e & MANTISSA_MASK;
size_t e = m_e >> MANTISSA_BITS;
size_t b = e == 0 ? 0 : 1;
size_t shifted_e = e - b;
size_t extended_m = (m + ((size_t)b << MANTISSA_BITS)) + 1;
return extended_m << (shifted_e + LOW_BITS);
}
else
{
return (size_t)1 << (m_e + LOW_BITS);
}
}
}
}

55
src/ds/csv.h Normal file
View File

@@ -0,0 +1,55 @@
#pragma once
#include <iostream>
#include <string>
namespace snmalloc
{
class CSVStream
{
private:
std::ostream* out;
bool first = true;
public:
class Endl
{};
Endl endl;
CSVStream(std::ostream* o) : out(o) {}
void preprint()
{
if (!first)
{
*out << ", ";
}
else
{
first = false;
}
}
CSVStream& operator<<(const std::string& str)
{
preprint();
*out << str;
return *this;
}
CSVStream& operator<<(uint64_t u)
{
preprint();
*out << u;
return *this;
}
CSVStream& operator<<(Endl)
{
*out << std::endl;
first = true;
return *this;
}
};
}

120
src/ds/dllist.h Normal file
View File

@@ -0,0 +1,120 @@
#pragma once
#include <cassert>
#include <cstdint>
#include <type_traits>
namespace snmalloc
{
template<class T, uintptr_t terminator = 0>
class DLList
{
private:
static_assert(
std::is_same<decltype(((T*)0)->prev), T*>::value, "T->prev must be a T*");
static_assert(
std::is_same<decltype(((T*)0)->next), T*>::value, "T->next must be a T*");
T* head = (T*)terminator;
public:
T* get_head()
{
return head;
}
T* pop()
{
T* item = head;
if (item != (T*)terminator)
remove(item);
return item;
}
void insert(T* item)
{
#ifndef NDEBUG
debug_check_not_contains(item);
#endif
item->next = head;
item->prev = (T*)terminator;
if (head != (T*)terminator)
head->prev = item;
head = item;
#ifndef NDEBUG
debug_check();
#endif
}
void remove(T* item)
{
#ifndef NDEBUG
debug_check_contains(item);
#endif
if (item->next != (T*)terminator)
item->next->prev = item->prev;
if (item->prev != (T*)terminator)
item->prev->next = item->next;
else
head = item->next;
#ifndef NDEBUG
debug_check();
#endif
}
void debug_check_contains(T* item)
{
#ifndef NDEBUG
debug_check();
T* curr = head;
while (curr != item)
{
assert(curr != (T*)terminator);
curr = curr->next;
}
#else
UNUSED(item);
#endif
}
void debug_check_not_contains(T* item)
{
#ifndef NDEBUG
debug_check();
T* curr = head;
while (curr != (T*)terminator)
{
assert(curr != item);
curr = curr->next;
}
#else
UNUSED(item);
#endif
}
void debug_check()
{
#ifndef NDEBUG
T* item = head;
T* prev = (T*)terminator;
while (item != (T*)terminator)
{
assert(item->prev == prev);
prev = item;
item = item->next;
}
#endif
}
};
}

24
src/ds/flaglock.h Normal file
View File

@@ -0,0 +1,24 @@
#pragma once
#include "bits.h"
namespace snmalloc
{
class FlagLock
{
private:
std::atomic_flag& lock;
public:
FlagLock(std::atomic_flag& lock) : lock(lock)
{
while (lock.test_and_set(std::memory_order_acquire))
bits::pause();
}
~FlagLock()
{
lock.clear(std::memory_order_release);
}
};
}

33
src/ds/helpers.h Normal file
View File

@@ -0,0 +1,33 @@
#pragma once
#include "flaglock.h"
namespace snmalloc
{
/*
* In some use cases we need to run before any of the C++ runtime has been
* initialised. This singleton class is design to not depend on the runtime.
*/
template<class Object, Object init() noexcept>
class Singleton
{
public:
inline static Object& get()
{
static std::atomic_flag flag;
static std::atomic<bool> initialised;
static Object obj;
if (!initialised.load(std::memory_order_acquire))
{
FlagLock lock(flag);
if (!initialised)
{
obj = init();
initialised.store(true, std::memory_order_release);
}
}
return obj;
}
};
}

75
src/ds/mpmcstack.h Normal file
View File

@@ -0,0 +1,75 @@
#pragma once
#include "aba.h"
namespace snmalloc
{
template<class T, Construction c = RequiresInit>
class MPMCStack
{
using ABA = ABA<T, c>;
private:
static_assert(
std::is_same<decltype(((T*)0)->next), std::atomic<T*>>::value,
"T->next must be a std::atomic<T*>");
ABA stack;
public:
void push(T* item)
{
return push(item, item);
}
void push(T* first, T* last)
{
// Pushes an item on the stack.
auto cmp = stack.read();
do
{
T* top = ABA::load(cmp);
last->next.store(top, std::memory_order_release);
} while (!stack.compare_exchange(cmp, first));
}
T* pop()
{
// Returns the next item. If the returned value is decommitted, it is
// possible for the read of top->next to segfault.
auto cmp = stack.read();
T* top;
T* next;
do
{
top = ABA::load(cmp);
if (top == nullptr)
break;
next = top->next.load(std::memory_order_acquire);
} while (!stack.compare_exchange(cmp, next));
return top;
}
T* pop_all()
{
// Returns all items as a linked list, leaving an empty stack.
auto cmp = stack.read();
T* top;
do
{
top = ABA::load(cmp);
if (top == nullptr)
break;
} while (!stack.compare_exchange(cmp, nullptr));
return top;
}
};
}

99
src/ds/mpscq.h Normal file
View File

@@ -0,0 +1,99 @@
#pragma once
#include "bits.h"
#include <stdlib.h>
#include <utility>
namespace snmalloc
{
template<class T>
class MPSCQ
{
private:
static_assert(
std::is_same<decltype(((T*)0)->next), std::atomic<T*>>::value,
"T->next must be a std::atomic<T*>");
std::atomic<T*> head;
T* tail;
public:
void invariant()
{
#ifndef NDEBUG
assert(head != nullptr);
assert(tail != nullptr);
#endif
}
void init(T* stub)
{
stub->next.store(nullptr, std::memory_order_relaxed);
tail = stub;
head.store(stub, std::memory_order_relaxed);
invariant();
}
T* destroy()
{
T* tl = tail;
head.store(nullptr, std::memory_order_relaxed);
tail = nullptr;
return tl;
}
T* get_head()
{
return head.load(std::memory_order_relaxed);
}
inline void push(T* item)
{
push(item, item);
}
inline bool is_empty()
{
T* hd = head.load(std::memory_order_relaxed);
return hd == tail;
}
void push(T* first, T* last)
{
// Pushes a list of messages to the queue. Each message from first to
// last should be linked together through their next pointers.
invariant();
last->next.store(nullptr, std::memory_order_relaxed);
std::atomic_thread_fence(std::memory_order_release);
T* prev = head.exchange(last, std::memory_order_relaxed);
prev->next.store(first, std::memory_order_relaxed);
}
std::pair<T*, T*> pop()
{
// Returns the next message and the tail message. If the next message
// is not null, the tail message should be freed by the caller.
invariant();
T* tl = tail;
T* next = tl->next.load(std::memory_order_relaxed);
if (next != nullptr)
{
tail = next;
assert(tail);
std::atomic_thread_fence(std::memory_order_acquire);
}
invariant();
return std::make_pair(next, tl);
}
T* peek()
{
return tail->next.load(std::memory_order_relaxed);
}
};
}

1091
src/mem/alloc.h Normal file
View File

File diff suppressed because it is too large Load Diff

121
src/mem/allocconfig.h Normal file
View File

@@ -0,0 +1,121 @@
#pragma once
#include "../ds/bits.h"
namespace snmalloc
{
enum ZeroMem
{
NoZero,
YesZero
};
// 0 intermediate bits results in power of 2 small allocs. 1 intermediate
// bit gives additional sizeclasses at the midpoint between each power of 2.
// 2 intermediate bits gives 3 intermediate sizeclasses, etc.
static constexpr size_t INTERMEDIATE_BITS =
#ifdef USE_INTERMEDIATE_BITS
USE_INTERMEDIATE_BITS
#else
2
#endif
;
// Return remote small allocs when the local cache reaches this size.
static constexpr size_t REMOTE_CACHE =
#ifdef USE_REMOTE_CACHE
USE_REMOTE_CACHE
#else
1 << 20
#endif
;
// Handle at most this many object from the remote dealloc queue at a time.
static constexpr size_t REMOTE_BATCH =
#ifdef USE_REMOTE_BATCH
REMOTE_BATCH
#else
64
#endif
;
static constexpr size_t RESERVE_MULTIPLE =
#ifdef USE_RESERVE_MULTIPLE
USE_RESERVE_MULTIPLE
#else
bits::is64() ? 16 : 2
#endif
;
enum DecommitStrategy
{
DecommitNone,
DecommitSuper,
DecommitAll
};
static constexpr DecommitStrategy decommit_strategy =
#ifdef USE_DECOMMIT_STRATEGY
USE_DECOMMIT_STRATEGY
#else
DecommitSuper
#endif
;
// The remaining values are derived, not configurable.
// Used to isolate values on cache lines to prevent false sharing.
static constexpr size_t CACHELINE_SIZE = 64;
// Used to keep Superslab metadata committed.
static constexpr size_t OS_PAGE_SIZE = 0x1000;
static constexpr size_t PAGE_ALIGNED_SIZE = OS_PAGE_SIZE << INTERMEDIATE_BITS;
// Some system headers (e.g. Linux' sys/user.h, FreeBSD's machine/param.h)
// define `PAGE_SIZE` as a macro. We don't use `PAGE_SIZE` as our variable
// name, to avoid conflicts, but if we do see a macro definition then check
// that our value matches the platform's expected value.
#ifdef PAGE_SIZE
static_assert(
PAGE_SIZE == OS_PAGE_SIZE,
"Page size from system header does not match snmalloc config page size.");
#endif
// Minimum allocation size is space for two pointers.
static constexpr size_t MIN_ALLOC_BITS = bits::is64() ? 4 : 3;
static constexpr size_t MIN_ALLOC_SIZE = 1 << MIN_ALLOC_BITS;
// Slabs are 64 kb.
static constexpr size_t SLAB_BITS = 16;
static constexpr size_t SLAB_SIZE = 1 << SLAB_BITS;
static constexpr size_t SLAB_MASK = ~(SLAB_SIZE - 1);
// Superslabs are composed of this many slabs. Slab offsets are encoded as
// a byte, so the maximum count is 256. This must be a power of two to
// allow fast masking to find a superslab start address.
static constexpr size_t SLAB_COUNT_BITS = 8;
static constexpr size_t SLAB_COUNT = 1 << SLAB_COUNT_BITS;
static constexpr size_t SUPERSLAB_SIZE = SLAB_SIZE * SLAB_COUNT;
static constexpr size_t SUPERSLAB_MASK = ~(SUPERSLAB_SIZE - 1);
static constexpr size_t SUPERSLAB_BITS = SLAB_BITS + SLAB_COUNT_BITS;
static constexpr size_t RESERVE_SIZE = SUPERSLAB_SIZE * RESERVE_MULTIPLE;
// Number of slots for remote deallocation.
static constexpr size_t REMOTE_SLOT_BITS = 6;
static constexpr size_t REMOTE_SLOTS = 1 << REMOTE_SLOT_BITS;
static constexpr size_t REMOTE_MASK = REMOTE_SLOTS - 1;
static_assert(
INTERMEDIATE_BITS < MIN_ALLOC_BITS,
"INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS");
static_assert(
MIN_ALLOC_SIZE >= (sizeof(void*) * 2),
"MIN_ALLOC_SIZE must be sufficient for two pointers");
static_assert(
SLAB_BITS == (sizeof(uint16_t) * 8),
"SLAB_BITS must be the bits in a uint16_t");
static_assert(
SLAB_COUNT == bits::next_pow2_const(SLAB_COUNT),
"SLAB_COUNT must be a power of 2");
static_assert(
SLAB_COUNT <= (UINT8_MAX + 1), "SLAB_COUNT must fit in a uint8_t");
};

19
src/mem/allocslab.h Normal file
View File

@@ -0,0 +1,19 @@
#pragma once
#include "../mem/baseslab.h"
#include "remoteallocator.h"
namespace snmalloc
{
class Allocslab : public Baseslab
{
protected:
RemoteAllocator* allocator;
public:
RemoteAllocator* get_allocator()
{
return allocator;
}
};
}

391
src/mem/allocstats.h Normal file
View File

@@ -0,0 +1,391 @@
#pragma once
#include "../ds/bits.h"
#include <cstdint>
#ifdef USE_SNMALLOC_STATS
# include "../ds/csv.h"
# include "sizeclass.h"
# include <cstring>
# include <iostream>
#endif
namespace snmalloc
{
template<size_t N, size_t LARGE_N>
struct AllocStats
{
struct CurrentMaxPair
{
size_t current = 0;
size_t max = 0;
void inc()
{
current++;
if (current > max)
max++;
}
void dec()
{
current--;
}
bool is_empty()
{
return current == 0;
}
bool is_unused()
{
return max == 0;
}
void add(CurrentMaxPair& that)
{
current += that.current;
max += that.max;
}
#ifdef USE_SNMALLOC_STATS
void print(CSVStream& csv, size_t multiplier = 1)
{
csv << current * multiplier << max * multiplier;
}
#endif
};
struct Stats
{
CurrentMaxPair count;
CurrentMaxPair slab_count;
uint64_t time = bits::tick();
uint64_t ticks = 0;
double online_average = 0;
bool is_empty()
{
return count.is_empty();
}
void add(Stats& that)
{
count.add(that.count);
slab_count.add(that.slab_count);
}
void addToRunningAverage()
{
uint64_t now = bits::tick();
if (slab_count.current != 0)
{
double occupancy = (double)count.current / (double)slab_count.current;
uint64_t duration = now - time;
if (ticks == 0)
online_average = occupancy;
else
online_average += ((occupancy - online_average) * duration) / ticks;
ticks += duration;
}
time = now;
}
#ifdef USE_SNMALLOC_STATS
void
print(CSVStream& csv, size_t multiplier = 1, size_t slab_multiplier = 1)
{
// Keep in sync with header lower down
count.print(csv, multiplier);
slab_count.print(csv, slab_multiplier);
size_t average = (size_t)(online_average * multiplier);
csv << average << (slab_multiplier - average) * slab_count.max
<< csv.endl;
}
#endif
};
#ifdef USE_SNMALLOC_STATS
static constexpr size_t BUCKETS_BITS = 4;
static constexpr size_t BUCKETS = 1 << BUCKETS_BITS;
static constexpr size_t TOTAL_BUCKETS =
bits::to_exp_mant_const<BUCKETS_BITS>(
((size_t)1 << (bits::ADDRESS_BITS - 1)));
Stats sizeclass[N];
Stats large[LARGE_N];
size_t remote_freed = 0;
size_t remote_posted = 0;
size_t remote_received = 0;
size_t superslab_push_count = 0;
size_t superslab_pop_count = 0;
size_t superslab_fresh_count = 0;
size_t segment_count = 0;
size_t bucketed_requests[TOTAL_BUCKETS] = {};
#endif
void alloc_request(size_t size)
{
UNUSED(size);
#ifdef USE_SNMALLOC_STATS
bucketed_requests[bits::to_exp_mant<BUCKETS_BITS>(size)]++;
#endif
}
bool is_empty()
{
#ifdef USE_SNMALLOC_STATS
for (size_t i = 0; i < N; i++)
{
if (!sizeclass[i].is_empty())
return false;
}
for (size_t i = 0; i < LARGE_N; i++)
{
if (!large[i].is_empty())
return false;
}
return (remote_freed == remote_posted);
#else
return true;
#endif
}
void sizeclass_alloc(uint8_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
sizeclass[sc].addToRunningAverage();
sizeclass[sc].count.inc();
#endif
}
void sizeclass_dealloc(uint8_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
sizeclass[sc].addToRunningAverage();
sizeclass[sc].count.dec();
#endif
}
void large_alloc(size_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
large[sc].count.inc();
#endif
}
void sizeclass_alloc_slab(uint8_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
sizeclass[sc].addToRunningAverage();
sizeclass[sc].slab_count.inc();
#endif
}
void sizeclass_dealloc_slab(uint8_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
sizeclass[sc].addToRunningAverage();
sizeclass[sc].slab_count.dec();
#endif
}
void large_dealloc(size_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
large[sc].count.dec();
#endif
}
void segment_create()
{
#ifdef USE_SNMALLOC_STATS
segment_count++;
#endif
}
void superslab_pop()
{
#ifdef USE_SNMALLOC_STATS
superslab_pop_count++;
#endif
}
void superslab_push()
{
#ifdef USE_SNMALLOC_STATS
superslab_push_count++;
#endif
}
void superslab_fresh()
{
#ifdef USE_SNMALLOC_STATS
superslab_fresh_count++;
#endif
}
void remote_free(uint8_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
remote_freed += sizeclass_to_size(sc);
#endif
}
void remote_post()
{
#ifdef USE_SNMALLOC_STATS
remote_posted = remote_freed;
#endif
}
void remote_receive(uint8_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
remote_received += sizeclass_to_size(sc);
#endif
}
void add(AllocStats<N, LARGE_N>& that)
{
UNUSED(that);
#ifdef USE_SNMALLOC_STATS
for (size_t i = 0; i < N; i++)
sizeclass[i].add(that.sizeclass[i]);
for (size_t i = 0; i < LARGE_N; i++)
large[i].add(that.large[i]);
for (size_t i = 0; i < TOTAL_BUCKETS; i++)
bucketed_requests[i] += that.bucketed_requests[i];
remote_freed += that.remote_freed;
remote_posted += that.remote_posted;
remote_received += that.remote_received;
superslab_pop_count += that.superslab_pop_count;
superslab_push_count += that.superslab_push_count;
superslab_fresh_count += that.superslab_fresh_count;
segment_count += that.segment_count;
#endif
}
#ifdef USE_SNMALLOC_STATS
template<class Alloc>
void print(std::ostream& o, uint64_t dumpid = 0, uint64_t allocatorid = 0)
{
UNUSED(o);
UNUSED(dumpid);
UNUSED(allocatorid);
CSVStream csv(&o);
if (dumpid == 0)
{
// Output headers for initial dump
// Keep in sync with data dump
csv << "GlobalStats"
<< "DumpID"
<< "AllocatorID"
<< "Remote freed"
<< "Remote posted"
<< "Remote received"
<< "Superslab pop"
<< "Superslab push"
<< "Superslab fresh"
<< "Segments" << csv.endl;
csv << "BucketedStats"
<< "DumpID"
<< "AllocatorID"
<< "Size group"
<< "Size"
<< "Current bytes"
<< "Max bytes"
<< "Current Slab bytes"
<< "Max Slab bytes"
<< "Average Slab Usage"
<< "Average wasted space" << csv.endl;
csv << "AllocSizes"
<< "DumpID"
<< "AllocatorID"
<< "ClassID"
<< "Low size"
<< "High size"
<< "Count" << csv.endl;
}
for (uint8_t i = 0; i < N; i++)
{
if (sizeclass[i].count.is_unused())
continue;
sizeclass[i].addToRunningAverage();
csv << "BucketedStats" << dumpid << allocatorid << i
<< sizeclass_to_size(i);
sizeclass[i].print(csv, sizeclass_to_size(i), SLAB_SIZE);
}
for (uint8_t i = 0; i < LARGE_N; i++)
{
if (large[i].count.is_unused())
continue;
csv << "BucketedStats" << dumpid << allocatorid << (i + N)
<< large_sizeclass_to_size(i);
large[i].print(csv, large_sizeclass_to_size(i));
}
size_t low = 0;
size_t high = 0;
for (size_t i = 0; i < TOTAL_BUCKETS; i++)
{
low = high + 1;
high = bits::from_exp_mant<BUCKETS_BITS>(i);
if (bucketed_requests[i] == 0)
continue;
csv << "AllocSizes" << dumpid << allocatorid << i << low << high
<< bucketed_requests[i] << csv.endl;
}
csv << "GlobalStats" << dumpid << allocatorid << remote_freed
<< remote_posted << remote_received << superslab_pop_count
<< superslab_push_count << superslab_fresh_count << segment_count
<< csv.endl;
}
#endif
};
}

27
src/mem/baseslab.h Normal file
View File

@@ -0,0 +1,27 @@
#pragma once
#include "../ds/mpmcstack.h"
#include "allocconfig.h"
namespace snmalloc
{
enum SlabKind
{
Fresh = 0,
Large,
Medium,
Super
};
class Baseslab
{
protected:
SlabKind kind;
public:
SlabKind get_kind()
{
return kind;
}
};
}

171
src/mem/globalalloc.h Normal file
View File

@@ -0,0 +1,171 @@
#pragma once
#include "../ds/helpers.h"
#include "alloc.h"
#include "typealloc.h"
namespace snmalloc
{
template<class MemoryProvider>
class AllocPool : TypeAlloc<Allocator<MemoryProvider>, MemoryProvider>
{
using Alloc = Allocator<MemoryProvider>;
using Parent = TypeAlloc<Allocator<MemoryProvider>, MemoryProvider>;
public:
static AllocPool* make(MemoryProvider& mp)
{
static_assert(
sizeof(AllocPool) == sizeof(Parent),
"You cannot add fields to this class.");
// This cast is safe due to the static assert.
return (AllocPool*)Parent::make(mp);
}
static AllocPool* make() noexcept
{
return make(default_memory_provider);
}
Alloc* acquire()
{
return Parent::alloc(Parent::memory_provider);
}
void release(Alloc* a)
{
Parent::dealloc(a);
}
public:
void aggregate_stats(Stats& stats)
{
auto* alloc = Parent::iterate();
while (alloc != nullptr)
{
stats.add(alloc->stats());
alloc = Parent::iterate(alloc);
}
}
void print_all_stats(std::ostream& o, uint64_t dumpid = 0)
{
auto alloc = Parent::iterate();
while (alloc != nullptr)
{
alloc->stats().template print<Alloc>(o, dumpid, alloc->id());
alloc = Parent::iterate(alloc);
}
}
void cleanup_unused()
{
#ifndef USE_MALLOC
// Call this periodically to free and coalesce memory allocated by
// allocators that are not currently in use by any thread.
// One atomic operation to extract the stack, another to restore it.
// Handling the message queue for each stack is non-atomic.
auto* first = Parent::extract();
auto* alloc = first;
decltype(alloc) last;
if (alloc != nullptr)
{
while (alloc != nullptr)
{
alloc->handle_message_queue();
last = alloc;
alloc = Parent::extract(alloc);
}
restore(first, last);
}
#endif
}
void debug_check_empty()
{
#ifndef USE_MALLOC
// This is a debugging function. It checks that all memory from all
// allocators has been freed.
size_t alloc_count = 0;
auto* alloc = Parent::iterate();
// Count the linked allocators.
while (alloc != nullptr)
{
alloc = Parent::iterate(alloc);
alloc_count++;
}
bool done = false;
while (!done)
{
done = true;
alloc = Parent::iterate();
while (alloc != nullptr)
{
// Destroy the message queue so that it has no stub message.
Remote* p = alloc->message_queue().destroy();
while (p != nullptr)
{
Remote* next = p->non_atomic_next;
alloc->handle_dealloc_remote(p);
p = next;
}
// Place the static stub message on the queue.
alloc->init_message_queue();
// Post all remotes, including forwarded ones. If any allocator posts,
// repeat the loop.
if (alloc->remote.size > 0)
{
alloc->stats().remote_post();
alloc->remote.post(alloc->id());
done = false;
}
alloc = Parent::iterate(alloc);
}
}
alloc = Parent::iterate();
size_t empty_count = 0;
while (alloc != nullptr)
{
// Check that the allocator has freed all memory.
if (alloc->stats().is_empty())
empty_count++;
alloc = Parent::iterate(alloc);
}
if (alloc_count != empty_count)
error("Incorrect number of allocators");
#endif
}
};
inline AllocPool<GlobalVirtual>*& current_alloc_pool()
{
return Singleton<
AllocPool<GlobalVirtual>*,
AllocPool<GlobalVirtual>::make>::get();
}
template<class MemoryProvider>
inline AllocPool<MemoryProvider>* make_alloc_pool(MemoryProvider& mp)
{
return AllocPool<MemoryProvider>::make(mp);
}
using Alloc = Allocator<GlobalVirtual>;
}

207
src/mem/largealloc.h Normal file
View File

@@ -0,0 +1,207 @@
#pragma once
#include "../ds/flaglock.h"
#include "../ds/mpmcstack.h"
#include "../pal/pal.h"
#include "allocstats.h"
#include "baseslab.h"
#include "sizeclass.h"
#include <utility>
namespace snmalloc
{
class Largeslab : public Baseslab
{
// This is the view of a contiguous memory area when it is being kept
// in the global size-classed caches of available contiguous memory areas.
private:
template<class a, Construction c>
friend class MPMCStack;
std::atomic<Largeslab*> next;
public:
void init()
{
kind = Large;
}
};
// This represents the state that the large allcoator needs to add to the
// global state of the allocator. This is currently stored in the memory
// provider, so we add this in.
template<class MemoryProviderState>
class MemoryProviderStateMixin : public MemoryProviderState
{
std::atomic_flag lock = ATOMIC_FLAG_INIT;
size_t bump;
size_t remaining;
std::pair<void*, size_t> reserve_block() noexcept
{
size_t size = SUPERSLAB_SIZE;
void* r = ((MemoryProviderState*)this)
->template reserve<false>(&size, SUPERSLAB_SIZE);
if (size < SUPERSLAB_SIZE)
error("out of memory");
((MemoryProviderState*)this)
->template notify_using<NoZero>(r, OS_PAGE_SIZE);
return std::make_pair(r, size);
}
public:
/**
* Stack of large allocations that have been returned for reuse.
*/
MPMCStack<Largeslab, PreZeroed> large_stack[NUM_LARGE_CLASSES];
/**
* Primitive allocator for structure that are required before
* the allocator can be running.
***/
void* alloc_chunk(size_t size)
{
// Cache line align
size = bits::align_up(size, 64);
void* p;
{
FlagLock f(lock);
if (remaining < size)
{
auto r_size = reserve_block();
bump = (size_t)r_size.first;
remaining = r_size.second;
}
p = (void*)bump;
bump += size;
remaining -= size;
}
auto page_start = bits::align_down((size_t)p, OS_PAGE_SIZE);
auto page_end = bits::align_up((size_t)p + size, OS_PAGE_SIZE);
((MemoryProviderState*)this)
->template notify_using<NoZero>(
(void*)page_start, page_end - page_start);
return p;
}
};
using Stats = AllocStats<NUM_SIZECLASSES, NUM_LARGE_CLASSES>;
enum AllowReserve
{
NoReserve,
YesReserve
};
template<class MemoryProvider>
class LargeAlloc
{
void* reserved_start = nullptr;
void* reserved_end = nullptr;
public:
// This will be a zero-size structure if stats are not enabled.
Stats stats;
MemoryProvider& memory_provider;
LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {}
template<AllowReserve allow_reserve>
bool reserve_memory(size_t need, size_t add)
{
if (((size_t)reserved_start + need) > (size_t)reserved_end)
{
if (allow_reserve == YesReserve)
{
stats.segment_create();
reserved_start =
memory_provider.template reserve<false>(&add, SUPERSLAB_SIZE);
reserved_end = (void*)((size_t)reserved_start + add);
reserved_start =
(void*)bits::align_up((size_t)reserved_start, SUPERSLAB_SIZE);
if (add < need)
return false;
}
else
{
return false;
}
}
return true;
}
template<ZeroMem zero_mem = NoZero, AllowReserve allow_reserve = YesReserve>
void* alloc(size_t large_class, size_t size)
{
size_t rsize = ((size_t)1 << SUPERSLAB_BITS) << large_class;
if (size == 0)
size = rsize;
void* p = memory_provider.large_stack[large_class].pop();
if (p == nullptr)
{
assert(reserved_start <= reserved_end);
size_t add;
if ((rsize + SUPERSLAB_SIZE) < RESERVE_SIZE)
add = RESERVE_SIZE;
else
add = rsize + SUPERSLAB_SIZE;
if (!reserve_memory<allow_reserve>(rsize, add))
return nullptr;
p = (void*)reserved_start;
reserved_start = (void*)((size_t)p + rsize);
// All memory is zeroed since it comes from reserved space.
memory_provider.template notify_using<NoZero>(p, size);
}
else
{
if ((decommit_strategy != DecommitNone) || (large_class > 0))
{
// Only the first page needs to be zeroed, as this was decommitted.
if (zero_mem == YesZero)
memory_provider.template zero<true>(p, OS_PAGE_SIZE);
memory_provider.template notify_using<zero_mem>(
(void*)((size_t)p + OS_PAGE_SIZE), size - OS_PAGE_SIZE);
}
else
{
// This is a superslab that has not been decommitted.
if (zero_mem == YesZero)
memory_provider.template zero<true>(p, size);
}
}
return p;
}
void dealloc(void* p, size_t large_class)
{
memory_provider.large_stack[large_class].push((Largeslab*)p);
}
};
using GlobalVirtual = MemoryProviderStateMixin<Pal>;
/**
* The memory provider that will be used if no other provider is explicitly
* passed as an argument.
*/
HEADER_GLOBAL GlobalVirtual default_memory_provider;
}

130
src/mem/mediumslab.h Normal file
View File

@@ -0,0 +1,130 @@
#pragma once
#include "../ds/dllist.h"
#include "allocconfig.h"
#include "allocslab.h"
#include "sizeclass.h"
namespace snmalloc
{
class Mediumslab : public Allocslab
{
// This is the view of a 16 mb area when it is being used to allocate
// medium sized classes: 64 kb to 16 mb, non-inclusive.
private:
friend DLList<Mediumslab>;
// Keep the allocator pointer on a separate cache line. It is read by
// other threads, and does not change, so we avoid false sharing.
alignas(CACHELINE_SIZE) Mediumslab* next;
Mediumslab* prev;
uint16_t free;
uint8_t head;
uint8_t sizeclass;
uint16_t stack[SLAB_COUNT - 1];
public:
static constexpr uint32_t header_size()
{
static_assert(
sizeof(Mediumslab) < OS_PAGE_SIZE,
"Mediumslab header size must be less than the page size");
static_assert(
sizeof(Mediumslab) < SLAB_SIZE,
"Mediumslab header size must be less than the slab size");
// Always use a full page as the header, in order to get page sized
// alignment of individual allocations.
return OS_PAGE_SIZE;
}
static Mediumslab* get(void* p)
{
return (Mediumslab*)((size_t)p & SUPERSLAB_MASK);
}
void init(RemoteAllocator* alloc, uint8_t sc, size_t rsize)
{
assert(sc >= NUM_SMALL_CLASSES);
assert((sc - NUM_SMALL_CLASSES) < NUM_MEDIUM_CLASSES);
allocator = alloc;
head = 0;
// If this was previously a Mediumslab of the same sizeclass, don't
// initialise the allocation stack.
if ((kind != Medium) || (sizeclass != sc))
{
sizeclass = sc;
uint16_t ssize = (uint16_t)(rsize >> 8);
kind = Medium;
free = medium_slab_free(sc);
for (uint16_t i = free; i > 0; i--)
stack[free - i] = (uint16_t)((SUPERSLAB_SIZE >> 8) - (i * ssize));
}
else
{
assert(free == medium_slab_free(sc));
}
}
uint8_t get_sizeclass()
{
return sizeclass;
}
template<ZeroMem zero_mem, typename MemoryProvider>
void* alloc(size_t size, MemoryProvider& memory_provider)
{
assert(!full());
uint16_t index = stack[head++];
void* p = (void*)((size_t)this + ((size_t)index << 8));
free--;
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, OS_PAGE_SIZE));
size = bits::align_up(size, OS_PAGE_SIZE);
if (decommit_strategy == DecommitAll)
memory_provider.template notify_using<zero_mem>(p, size);
else if (zero_mem == YesZero)
memory_provider.template zero<true>(p, size);
return p;
}
template<typename MemoryProvider>
bool dealloc(void* p, MemoryProvider& memory_provider)
{
assert(head > 0);
// Returns true if the Mediumslab was full before this deallocation.
bool was_full = full();
free++;
stack[--head] = pointer_to_index(p);
if (decommit_strategy == DecommitAll)
memory_provider.notify_not_using(p, sizeclass_to_size(sizeclass));
return was_full;
}
bool full()
{
return free == 0;
}
bool empty()
{
return head == 0;
}
private:
uint16_t pointer_to_index(void* p)
{
// Get the offset from the slab for a memory location.
return (uint16_t)(((size_t)p - (size_t)this) >> 8);
}
};
}

166
src/mem/metaslab.h Normal file
View File

@@ -0,0 +1,166 @@
#pragma once
#include "../ds/dllist.h"
#include "sizeclass.h"
namespace snmalloc
{
class Slab;
struct SlabLink
{
SlabLink* prev;
SlabLink* next;
Slab* get_slab()
{
return (Slab*)((size_t)this & SLAB_MASK);
}
};
using SlabList = DLList<SlabLink, ~(uintptr_t)0>;
static_assert(
sizeof(SlabLink) <= MIN_ALLOC_SIZE,
"Need to be able to pack a SlabLink into any free small alloc");
static constexpr uint16_t SLABLINK_INDEX =
(uint16_t)(SLAB_SIZE - sizeof(SlabLink));
// The Metaslab represent the status of a single slab.
// This can be either a short or a standard slab.
class Metaslab
{
private:
// How many entries are used in this slab.
uint16_t used;
public:
// Bump free list of unused entries in this sizeclass.
// If the bottom bit is 1, then this represents a bump_ptr
// of where we have allocated up to in this slab. Otherwise,
// it represents the location of the first block in the free
// list. The free list is chained through deallocated blocks.
// It either terminates with a bump ptr, or if all the space is in
// the free list, then the last block will be also referenced by
// link.
// Note that, in the case that this is the first block in the size
// class list, where all the unused memory is in the free list,
// then the last block can both be interpreted as a final bump
// pointer entry, and the first entry in the doubly linked list.
// The terminal value in the free list, and the terminal value in
// the SlabLink previous field will alias. The SlabLink uses ~0 for
// its terminal value to be a valid terminal bump ptr.
uint16_t head;
// When a slab has free space it will be on the has space list for
// that size class. We use an empty block in this slab to be the
// doubly linked node into that size class's free list.
uint16_t link;
union
{
uint8_t sizeclass;
uint8_t next;
};
void add_use()
{
used++;
}
void sub_use()
{
used--;
}
void set_unused()
{
used = 0;
}
bool is_unused()
{
return used == 0;
}
bool is_full()
{
return (head & 2) != 0;
}
void set_full()
{
assert(head == 1);
head = (uint16_t)~0;
}
SlabLink* get_link(Slab* slab)
{
return (SlabLink*)((size_t)slab + link);
}
bool valid_head(bool is_short)
{
size_t size = sizeclass_to_size(sizeclass);
size_t offset = get_slab_offset(sizeclass, is_short);
return ((((head & ~(size_t)1) - (offset & ~(size_t)1)) % size) == 0);
}
void debug_slab_invariant(bool is_short, Slab* slab)
{
#if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS)
size_t size = sizeclass_to_size(sizeclass);
size_t offset = get_slab_offset(sizeclass, is_short) - 1;
size_t accounted_for = used * size + offset;
if (is_full())
{
// All the blocks must be used.
assert(SLAB_SIZE == accounted_for);
// There is no free list to validate
// 'link' value is not important if full.
return;
}
// Block is not full
assert(SLAB_SIZE > accounted_for);
// Walk bump-free-list-segment accounting for unused space
uint16_t curr = head;
while ((curr & 1) != 1)
{
// Check we are looking at a correctly aligned block
assert((curr - offset) % size == 0);
// Account for free elements in free list
accounted_for += size;
assert(SLAB_SIZE >= accounted_for);
// We are not guaranteed to hit a bump ptr unless
// we are the top element on the size class, so treat as
// a list segment.
if (curr == link)
break;
// Iterate bump/free list segment
curr = *(uint16_t*)((uintptr_t)slab + curr);
}
// Check we terminated traversal on a correctly aligned block
assert(((curr & ~1) - offset) % size == 0);
if (curr != link)
{
// The link should be at the special end location as we
// haven't completely filled this block at any point.
assert(link == SLABLINK_INDEX);
// Account for to be bump allocated space
accounted_for += SLAB_SIZE - (curr - 1);
}
// All space accounted for
assert(SLAB_SIZE == accounted_for);
#else
UNUSED(slab);
UNUSED(is_short);
#endif
}
};
}

227
src/mem/pagemap.h Normal file
View File

@@ -0,0 +1,227 @@
#pragma once
#include "../ds/bits.h"
#include <algorithm>
#include <atomic>
namespace snmalloc
{
template<size_t GRANULARITY_BITS, typename T, T default_content>
class Pagemap
{
private:
static constexpr size_t PAGEMAP_BITS = 16;
static constexpr size_t PAGEMAP_SIZE = 1 << PAGEMAP_BITS;
static constexpr size_t COVERED_BITS =
bits::ADDRESS_BITS - GRANULARITY_BITS;
static constexpr size_t POINTER_BITS =
bits::next_pow2_bits_const(sizeof(void*));
static constexpr size_t CONTENT_BITS =
bits::next_pow2_bits_const(sizeof(T));
static constexpr size_t BITS_FOR_LEAF = PAGEMAP_BITS - CONTENT_BITS;
static constexpr size_t ENTRIES_PER_LEAF = 1 << BITS_FOR_LEAF;
static constexpr size_t LEAF_MASK = ENTRIES_PER_LEAF - 1;
static constexpr size_t BITS_PER_INDEX_LEVEL = PAGEMAP_BITS - POINTER_BITS;
static constexpr size_t ENTRIES_PER_INDEX_LEVEL = 1 << BITS_PER_INDEX_LEVEL;
static constexpr size_t ENTRIES_MASK = ENTRIES_PER_INDEX_LEVEL - 1;
static constexpr size_t INDEX_BITS =
BITS_FOR_LEAF > COVERED_BITS ? 0 : COVERED_BITS - BITS_FOR_LEAF;
static constexpr size_t INDEX_LEVELS = INDEX_BITS / BITS_PER_INDEX_LEVEL;
static constexpr size_t TOPLEVEL_BITS =
INDEX_BITS - (INDEX_LEVELS * BITS_PER_INDEX_LEVEL);
static constexpr size_t TOPLEVEL_ENTRIES = 1 << TOPLEVEL_BITS;
static constexpr size_t TOPLEVEL_SHIFT =
(INDEX_LEVELS * BITS_PER_INDEX_LEVEL) + BITS_FOR_LEAF + GRANULARITY_BITS;
// Value used to represent when a node is being added too
static constexpr uintptr_t LOCKED_ENTRY = 1;
struct Leaf
{
std::atomic<T> values[ENTRIES_PER_LEAF];
};
struct PagemapEntry
{
std::atomic<PagemapEntry*> entries[ENTRIES_PER_INDEX_LEVEL];
};
static_assert(
sizeof(PagemapEntry) == sizeof(Leaf), "Should be the same size");
// Init removed as not required as this is only ever a global
// cl is generating a memset of zero, which will be a problem
// in libc/ucrt bring up. On ucrt this will run after the first
// allocation.
// TODO: This is fragile that it is not being memset, and we should review
// to ensure we don't get bitten by this in the future.
std::atomic<PagemapEntry*> top[TOPLEVEL_ENTRIES]; // = {nullptr};
template<bool create_addr>
inline PagemapEntry* get_node(std::atomic<PagemapEntry*>* e, bool& result)
{
// The page map nodes are all allocated directly from the OS zero
// initialised with a system call. We don't need any ordered to guarantee
// to see that correctly.
PagemapEntry* value = e->load(std::memory_order_relaxed);
if ((uintptr_t)value <= LOCKED_ENTRY)
{
if constexpr (create_addr)
{
value = nullptr;
if (e->compare_exchange_strong(
value, (PagemapEntry*)LOCKED_ENTRY, std::memory_order_relaxed))
{
auto& v = default_memory_provider;
value = (PagemapEntry*)v.alloc_chunk(PAGEMAP_SIZE);
e->store(value, std::memory_order_release);
}
else
{
while ((uintptr_t)e->load(std::memory_order_relaxed) ==
LOCKED_ENTRY)
{
bits::pause();
}
value = e->load(std::memory_order_acquire);
}
}
else
{
result = false;
return nullptr;
}
}
result = true;
return value;
}
template<bool create_addr>
inline std::pair<Leaf*, size_t> get_leaf_index(void* p, bool& result)
{
size_t addr = (size_t)p;
#ifdef FreeBSD_KERNEL
// Zero the top 16 bits - kernel addresses all have them set, but the
// data structure assumes that they're zero.
addr &= 0xffffffffffffULL;
#endif
size_t ix = addr >> TOPLEVEL_SHIFT;
size_t shift = TOPLEVEL_SHIFT;
std::atomic<PagemapEntry*>* e = &top[ix];
for (size_t i = 0; i < INDEX_LEVELS; i++)
{
PagemapEntry* value = get_node<create_addr>(e, result);
if (!result)
return std::pair(nullptr, 0);
shift -= BITS_PER_INDEX_LEVEL;
ix = (addr >> shift) & ENTRIES_MASK;
e = &value->entries[ix];
if constexpr (INDEX_LEVELS == 1)
{
UNUSED(i);
break;
}
i++;
if (i == INDEX_LEVELS)
break;
}
Leaf* leaf = (Leaf*)get_node<create_addr>(e, result);
if (!result)
return std::pair(nullptr, 0);
shift -= BITS_FOR_LEAF;
ix = (addr >> shift) & LEAF_MASK;
return std::pair(leaf, ix);
}
template<bool create_addr>
inline std::atomic<T>* get_addr(void* p, bool& success)
{
auto leaf_ix = get_leaf_index<create_addr>(p, success);
return &(leaf_ix.first->values[leaf_ix.second]);
}
public:
static constexpr size_t GRANULARITY = 1 << GRANULARITY_BITS;
/**
* Returns the index of a pagemap entry within a given page. This is used
* in code that propagates changes to the pagemap elsewhere.
*/
size_t index_for_address(void* p)
{
bool success;
return (OS_PAGE_SIZE - 1) &
reinterpret_cast<size_t>(get_addr<true>(p, success));
}
/**
* Returns the address of the page containing
*/
void* page_for_address(void* p)
{
bool success;
return reinterpret_cast<void*>(
~(OS_PAGE_SIZE - 1) &
reinterpret_cast<uintptr_t>(get_addr<true>(p, success)));
}
std::atomic<T>* get_ptr(void* p)
{
bool success;
return get_addr<true>(p, success);
}
T get(void* p)
{
bool success;
auto addr = get_addr<false>(p, success);
if (!success)
return default_content;
return addr->load(std::memory_order_relaxed);
}
void set(void* p, T x)
{
bool success;
auto addr = get_addr<true>(p, success);
addr->store(x, std::memory_order_relaxed);
}
void set_range(void* p, T x, size_t length)
{
bool success;
do
{
auto leaf_ix = get_leaf_index<true>(p, success);
size_t ix = leaf_ix.second;
auto last = std::min(LEAF_MASK + 1, ix + length);
auto diff = last - ix;
for (; ix < last; ix++)
{
leaf_ix.first->values[ix] = x;
}
length = length - diff;
p = (void*)((uintptr_t)p + (diff << GRANULARITY_BITS));
} while (length > 0);
}
};
}

64
src/mem/remoteallocator.h Normal file
View File

@@ -0,0 +1,64 @@
#pragma once
#include "../ds/mpscq.h"
#include "../mem/allocconfig.h"
#include <atomic>
namespace snmalloc
{
struct Remote
{
static const uint64_t SIZECLASS_SHIFT = 56;
static const uint64_t SIZECLASS_MASK = 0xffULL << SIZECLASS_SHIFT;
static const uint64_t TARGET_MASK = ~SIZECLASS_MASK;
static_assert(SIZECLASS_MASK == 0xff00'0000'0000'0000ULL);
using alloc_id_t = size_t;
union
{
std::atomic<Remote*> next;
Remote* non_atomic_next;
};
uint64_t value;
void set_target_id(alloc_id_t id)
{
assert(id == (id & TARGET_MASK));
value = (id & TARGET_MASK) | (value & SIZECLASS_MASK);
}
void set_sizeclass(uint8_t sizeclass)
{
value = (value & TARGET_MASK) |
((static_cast<uint64_t>(sizeclass) << SIZECLASS_SHIFT) &
SIZECLASS_MASK);
}
alloc_id_t target_id()
{
return value & TARGET_MASK;
}
uint8_t sizeclass()
{
return (value & SIZECLASS_MASK) >> SIZECLASS_SHIFT;
}
};
struct RemoteAllocator
{
using alloc_id_t = Remote::alloc_id_t;
// Store the message queue on a separate cacheline. It is mutable data that
// is read by other threads.
alignas(CACHELINE_SIZE) MPSCQ<Remote> message_queue;
alloc_id_t id()
{
return static_cast<alloc_id_t>(
reinterpret_cast<uintptr_t>(&message_queue));
}
};
}

148
src/mem/sizeclass.h Normal file
View File

@@ -0,0 +1,148 @@
#pragma once
#include "allocconfig.h"
namespace snmalloc
{
constexpr static uint16_t get_slab_offset(uint8_t sc, bool is_short);
constexpr static size_t sizeclass_to_size(uint8_t sizeclass);
constexpr static uint16_t medium_slab_free(uint8_t sizeclass);
static inline uint8_t size_to_sizeclass(size_t size)
{
// Don't use sizeclasses that are not a multiple of the alignment.
// For example, 24 byte allocations can be
// problematic for some data due to alignment issues.
return (uint8_t)bits::to_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(size);
}
constexpr static inline uint8_t size_to_sizeclass_const(size_t size)
{
// Don't use sizeclasses that are not a multiple of the alignment.
// For example, 24 byte allocations can be
// problematic for some data due to alignment issues.
return (uint8_t)bits::to_exp_mant_const<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(
size);
}
constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class)
{
return (size_t)1 << (large_class + SUPERSLAB_BITS);
}
// Small classes range from [MIN, SLAB], i.e. inclusive.
static constexpr size_t NUM_SMALL_CLASSES =
size_to_sizeclass_const((size_t)1 << SLAB_BITS) + 1;
static constexpr size_t NUM_SIZECLASSES =
size_to_sizeclass_const((size_t)1 << SUPERSLAB_BITS);
// Medium classes range from (SLAB, SUPERSLAB), i.e. non-inclusive.
static constexpr size_t NUM_MEDIUM_CLASSES =
NUM_SIZECLASSES - NUM_SMALL_CLASSES;
// Large classes range from [SUPERSLAB, ADDRESS_SPACE).
static constexpr size_t NUM_LARGE_CLASSES =
bits::ADDRESS_BITS - SUPERSLAB_BITS;
template<size_t X, size_t Y>
constexpr void check_same()
{
static_assert(X == Y, "Values must be the same");
}
static_assert(size_to_sizeclass_const(SUPERSLAB_SIZE) == NUM_SIZECLASSES);
inline static size_t round_by_sizeclass(size_t rsize, size_t offset)
{
// check_same<NUM_LARGE_CLASSES, Globals::num_large_classes>();
// Must be called with a rounded size.
assert(sizeclass_to_size(size_to_sizeclass(rsize)) == rsize);
// Only works up to certain offsets, exhaustively tested upto
// SUPERSLAB_SIZE.
assert(offset <= SUPERSLAB_SIZE);
size_t align = bits::ctz(rsize);
size_t divider = rsize >> align;
// Maximum of 24 bits for 16MiB super/medium slab
if (INTERMEDIATE_BITS == 0 || divider == 1)
{
assert(divider == 1);
return offset & ~(rsize - 1);
}
if constexpr (bits::is64() && INTERMEDIATE_BITS <= 2)
{
// Only works for 64 bit multiplication, as the following will overflow in
// 32bit.
// The code is using reciprocal division, with a shift of 26 bits, this
// is considerably more bits than we need in the result. If SUPERSLABS
// get larger then we should review this code.
static_assert(SUPERSLAB_BITS <= 24, "The following code assumes 24 bits");
static constexpr size_t shift = 26;
size_t back_shift = shift + align;
static constexpr size_t mul_shift = 1ULL << shift;
static constexpr uint32_t constants[8] = {0,
mul_shift,
0,
(mul_shift / 3) + 1,
0,
(mul_shift / 5) + 1,
0,
(mul_shift / 7) + 1};
return ((constants[divider] * offset) >> back_shift) * rsize;
}
else
// Use 32-bit division as considerably faster than 64-bit, and
// everything fits into 32bits here.
return (uint32_t)(offset / rsize) * rsize;
}
inline static bool is_multiple_of_sizeclass(size_t rsize, size_t offset)
{
// Must be called with a rounded size.
assert(sizeclass_to_size(size_to_sizeclass(rsize)) == rsize);
// Only works up to certain offsets, exhaustively tested upto
// SUPERSLAB_SIZE.
assert(offset <= SUPERSLAB_SIZE);
size_t align = bits::ctz(rsize);
size_t divider = rsize >> align;
// Maximum of 24 bits for 16MiB super/medium slab
if (INTERMEDIATE_BITS == 0 || divider == 1)
{
assert(divider == 1);
return (offset & (rsize - 1)) == 0;
}
if constexpr (bits::is64() && INTERMEDIATE_BITS <= 2)
{
// Only works for 64 bit multiplication, as the following will overflow in
// 32bit.
// The code is using reciprocal division, with a shift of 26 bits, this
// is considerably more bits than we need in the result. If SUPERSLABS
// get larger then we should review this code.
static_assert(SUPERSLAB_BITS <= 24, "The following code assumes 24 bits");
static constexpr size_t shift = 31;
static constexpr size_t mul_shift = 1ULL << shift;
static constexpr uint32_t constants[8] = {0,
mul_shift,
0,
(mul_shift / 3) + 1,
0,
(mul_shift / 5) + 1,
0,
(mul_shift / 7) + 1};
// There is a long chain of zeros after the backshift
// However, not all zero so just check a range.
// This is exhaustively tested for the current use case
return (((constants[divider] * offset)) &
(((1ULL << (align + 3)) - 1) << (shift - 3))) == 0;
}
else
// Use 32-bit division as considerably faster than 64-bit, and
// everything fits into 32bits here.
return (uint32_t)(offset % rsize) == 0;
}
};

71
src/mem/sizeclasstable.h Normal file
View File

@@ -0,0 +1,71 @@
#pragma once
#include "superslab.h"
namespace snmalloc
{
struct SizeClassTable
{
size_t size[NUM_SIZECLASSES];
uint16_t bump_ptr_start[NUM_SMALL_CLASSES];
uint16_t short_bump_ptr_start[NUM_SMALL_CLASSES];
uint16_t count_per_slab[NUM_SMALL_CLASSES];
uint16_t medium_slab_slots[NUM_MEDIUM_CLASSES];
constexpr SizeClassTable()
: size(),
bump_ptr_start(),
short_bump_ptr_start(),
count_per_slab(),
medium_slab_slots()
{
for (uint8_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++)
{
size[sizeclass] =
bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(sizeclass);
}
size_t header_size = sizeof(Superslab);
size_t short_slab_size = SLAB_SIZE - header_size;
for (uint8_t i = 0; i < NUM_SMALL_CLASSES; i++)
{
short_bump_ptr_start[i] =
(uint16_t)(1 + (short_slab_size % size[i]) + header_size);
bump_ptr_start[i] = (uint16_t)(1 + (SLAB_SIZE % size[i]));
count_per_slab[i] = (uint16_t)(SLAB_SIZE / size[i]);
}
for (uint8_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++)
{
medium_slab_slots[i - NUM_SMALL_CLASSES] =
(uint16_t)((SUPERSLAB_SIZE - Mediumslab::header_size()) / size[i]);
}
}
};
static constexpr SizeClassTable sizeclass_metadata = SizeClassTable();
static inline constexpr uint16_t get_slab_offset(uint8_t sc, bool is_short)
{
if (is_short)
return sizeclass_metadata.short_bump_ptr_start[sc];
else
return sizeclass_metadata.bump_ptr_start[sc];
}
constexpr static inline size_t sizeclass_to_size(uint8_t sizeclass)
{
return sizeclass_metadata.size[sizeclass];
}
constexpr static inline size_t sizeclass_to_count(uint8_t sizeclass)
{
return sizeclass_metadata.count_per_slab[sizeclass];
}
constexpr static inline uint16_t medium_slab_free(uint8_t sizeclass)
{
return sizeclass_metadata.medium_slab_slots[sizeclass - NUM_SMALL_CLASSES];
}
}

165
src/mem/slab.h Normal file
View File

@@ -0,0 +1,165 @@
#pragma once
#include "superslab.h"
namespace snmalloc
{
class Slab
{
private:
uint16_t pointer_to_index(void* p)
{
// Get the offset from the slab for a memory location.
return (uint16_t)((size_t)p - (size_t)this);
}
public:
static Slab* get(void* p)
{
return (Slab*)((size_t)p & SLAB_MASK);
}
Metaslab* get_meta()
{
Superslab* super = Superslab::get(this);
return super->get_meta(this);
}
SlabLink* get_link()
{
return get_meta()->get_link(this);
}
template<ZeroMem zero_mem, typename MemoryProvider>
void* alloc(SlabList* sc, size_t rsize, MemoryProvider& memory_provider)
{
// Read the head from the metadata stored in the superslab.
Metaslab* meta = get_meta();
uint16_t head = meta->head;
assert(rsize == sizeclass_to_size(meta->sizeclass));
meta->debug_slab_invariant(is_short(), this);
assert(sc->get_head() == (SlabLink*)((size_t)this + meta->link));
assert(!meta->is_full());
meta->add_use();
void* p;
if ((head & 1) == 0)
{
p = (void*)((size_t)this + head);
// Read the next slot from the memory that's about to be allocated.
uint16_t next = *(uint16_t*)p;
meta->head = next;
}
else
{
// This slab is being bump allocated.
p = (void*)((size_t)this + head - 1);
meta->head = head + (uint16_t)rsize;
if (meta->head == 1)
{
meta->set_full();
}
}
// If we're full, we're no longer the current slab for this sizeclass
if (meta->is_full())
sc->pop();
meta->debug_slab_invariant(is_short(), this);
if (zero_mem == YesZero)
{
if (rsize < PAGE_ALIGNED_SIZE)
memory_provider.zero(p, rsize);
else
memory_provider.template zero<true>(p, rsize);
}
return p;
}
// Returns true, if it alters get_status.
template<typename MemoryProvider>
inline typename Superslab::Action dealloc(
SlabList* sc, Superslab* super, void* p, MemoryProvider& memory_provider)
{
Metaslab* meta = super->get_meta(this);
bool was_full = meta->is_full();
meta->debug_slab_invariant(is_short(), this);
meta->sub_use();
#ifndef SNMALLOC_SAFE_CLIENT
if (!is_multiple_of_sizeclass(
sizeclass_to_size(meta->sizeclass),
(uintptr_t)this + SLAB_SIZE - (uintptr_t)p))
{
error("Not deallocating start of an object");
}
#endif
if (was_full)
{
// We are not on the sizeclass list.
if (!meta->is_unused())
{
// Update the head and the sizeclass link.
uint16_t index = pointer_to_index(p);
meta->head = index;
assert(meta->valid_head(is_short()));
meta->link = index;
// Push on the list of slabs for this sizeclass.
sc->insert(meta->get_link(this));
meta->debug_slab_invariant(is_short(), this);
}
else
{
// Dealloc on the superslab.
if (is_short())
return super->dealloc_short_slab(memory_provider);
else
return super->dealloc_slab(this, memory_provider);
}
}
else if (meta->is_unused())
{
// Remove from the sizeclass list and dealloc on the superslab.
sc->remove(meta->get_link(this));
if (is_short())
return super->dealloc_short_slab(memory_provider);
else
return super->dealloc_slab(this, memory_provider);
}
else
{
#ifndef NDEBUG
sc->debug_check_contains(meta->get_link(this));
#endif
// Update the head and the next pointer in the free list.
uint16_t head = meta->head;
uint16_t current = pointer_to_index(p);
// Set the head to the memory being deallocated.
meta->head = current;
assert(meta->valid_head(is_short()));
// Set the next pointer to the previous head.
*(uint16_t*)p = head;
meta->debug_slab_invariant(is_short(), this);
}
return Superslab::NoSlabReturn;
}
bool is_short()
{
return ((size_t)this & SUPERSLAB_MASK) == (size_t)this;
}
};
}

233
src/mem/superslab.h Normal file
View File

@@ -0,0 +1,233 @@
#pragma once
#include "metaslab.h"
#include <cstring>
namespace snmalloc
{
class Superslab : public Allocslab
{
// This is the view of a 16 mb superslab when it is being used to allocate
// 64 kb slabs.
private:
friend DLList<Superslab>;
// Keep the allocator pointer on a separate cache line. It is read by
// other threads, and does not change, so we avoid false sharing.
alignas(CACHELINE_SIZE)
// The superslab is kept on a doubly linked list of superslabs which
// have some space.
Superslab* next;
Superslab* prev;
// This is a reference to the first unused slab in the free slab list
// It is does not contain the short slab, which is handled using a bit
// in the "used" field below. The list is terminated by pointing to
// the short slab.
// The head linked list has an absolute pointer for head, but the next
// pointers stores in the metaslabs are relative pointers, that is they
// are the relative offset to the next entry minus 1. This means that
// all zeros is a list that chains through all the blocks, so the zero
// initialised memory requires no more work.
uint8_t head;
// Represents twice the number of full size slabs used
// plus 1 for the short slab. i.e. using 3 slabs and the
// short slab would be 6 + 1 = 7
uint16_t used;
Metaslab meta[SLAB_COUNT];
// Used size_t as results in better code in MSVC
size_t slab_to_index(Slab* slab)
{
auto res = (((size_t)slab - (size_t)this) >> SLAB_BITS);
assert(res == (uint8_t)res);
return res;
}
public:
enum Status
{
Full,
Available,
OnlyShortSlabAvailable,
Empty
};
enum Action
{
NoSlabReturn = 0,
NoStatusChange = 1,
StatusChange = 2
};
static Superslab* get(void* p)
{
return (Superslab*)((size_t)p & SUPERSLAB_MASK);
}
static bool is_short_sizeclass(uint8_t sizeclass)
{
constexpr uint8_t h = size_to_sizeclass_const(sizeof(Superslab));
return sizeclass <= h;
}
template<typename MemoryProvider>
void init(RemoteAllocator* alloc, MemoryProvider& memory_provider)
{
allocator = alloc;
if (kind != Super)
{
// If this wasn't previously a Superslab, we need to set up the
// header.
kind = Super;
// Point head at the first non-short slab.
head = 1;
if (kind != Fresh)
{
// If this wasn't previously Fresh, we need to zero some things.
used = 0;
memory_provider.zero(meta, SLAB_COUNT * sizeof(Metaslab));
}
meta[0].set_unused();
}
}
bool is_empty()
{
return used == 0;
}
bool is_full()
{
return (used == (((SLAB_COUNT - 1) << 1) + 1));
}
bool is_almost_full()
{
return (used >= ((SLAB_COUNT - 1) << 1));
}
Status get_status()
{
if (!is_almost_full())
{
if (!is_empty())
{
return Available;
}
else
{
return Empty;
}
}
else
{
if (!is_full())
{
return OnlyShortSlabAvailable;
}
else
{
return Full;
}
}
}
Metaslab* get_meta(Slab* slab)
{
return &meta[slab_to_index(slab)];
}
template<typename MemoryProvider>
Slab* alloc_short_slab(uint8_t sizeclass, MemoryProvider& memory_provider)
{
if ((used & 1) == 1)
return alloc_slab(sizeclass, memory_provider);
meta[0].head = get_slab_offset(sizeclass, true);
meta[0].sizeclass = sizeclass;
meta[0].link = SLABLINK_INDEX;
if (decommit_strategy == DecommitAll)
{
memory_provider.template notify_using<NoZero>(
(void*)((size_t)this + OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE);
}
used++;
return (Slab*)this;
}
template<typename MemoryProvider>
Slab* alloc_slab(uint8_t sizeclass, MemoryProvider& memory_provider)
{
Slab* slab = (Slab*)((size_t)this + ((size_t)head << SLAB_BITS));
uint8_t n = meta[head].next;
meta[head].head = get_slab_offset(sizeclass, false);
meta[head].sizeclass = sizeclass;
meta[head].link = SLABLINK_INDEX;
head = head + n + 1;
used += 2;
if (decommit_strategy == DecommitAll)
{
memory_provider.template notify_using<NoZero>(slab, SLAB_SIZE);
}
return slab;
}
// Returns true, if this alters the value of get_status
template<typename MemoryProvider>
Action dealloc_slab(Slab* slab, MemoryProvider& memory_provider)
{
// This is not the short slab.
uint8_t index = (uint8_t)slab_to_index(slab);
uint8_t n = head - index - 1;
meta[index].next = n;
head = index;
bool was_almost_full = is_almost_full();
used -= 2;
if (decommit_strategy == DecommitAll)
memory_provider.notify_not_using(slab, SLAB_SIZE);
assert(meta[index].is_unused());
if (was_almost_full || is_empty())
return StatusChange;
return NoStatusChange;
}
// Returns true, if this alters the value of get_status
template<typename MemoryProvider>
Action dealloc_short_slab(MemoryProvider& memory_provider)
{
// This is the short slab.
if (decommit_strategy == DecommitAll)
{
memory_provider.notify_not_using(
(void*)((size_t)this + OS_PAGE_SIZE), SLAB_SIZE - OS_PAGE_SIZE);
}
bool was_full = is_full();
used--;
assert(meta[0].is_unused());
if (was_full || is_empty())
return StatusChange;
return NoStatusChange;
}
};
}

265
src/mem/threadalloc.h Normal file
View File

@@ -0,0 +1,265 @@
#pragma once
#include "../ds/helpers.h"
#include "globalalloc.h"
#if defined(SNMALLOC_USE_THREAD_DESTRUCTOR) && \
defined(SNMALLOC_USE_THREAD_CLEANUP)
#error At most one out of SNMALLOC_USE_THREAD_CLEANUP and SNMALLOC_USE_THREAD_DESTRUCTOR may be defined.
#endif
namespace snmalloc
{
extern "C" void _malloc_thread_cleanup(void);
#ifdef SNMALLOC_EXTERNAL_THREAD_ALLOC
/**
* Version of the `ThreadAlloc` interface that does no managment of thread
* local state, and just assumes that "ThreadAllocUntyped::get" has been
* declared before including snmalloc.h. As it is included before, it cannot
* know the allocator type, hence the casting.
*
* This class is used only when snmalloc is compiled as part of a runtime,
* which has its own managment of the thread local allocator pointer.
*/
class ThreadAllocUntypedWrapper
{
public:
static inline Alloc*& get()
{
return (Alloc*&)ThreadAllocUntyped::get();
}
};
#endif
/**
* Version of the `ThreadAlloc` interface that uses a hook provided by libc
* to destroy thread-local state. This is the ideal option, because it
* enforces ordering of destruction such that the malloc state is destroyed
* after anything that can allocate memory.
*
* This class is used only when snmalloc is compiled as part of a compatible
* libc (for example, FreeBSD libc).
*/
class ThreadAllocLibcCleanup
{
/**
* Libc will call `_malloc_thread_cleanup` just before a thread terminates.
* This function must be allowed to call back into this class to destroy
* the state.
*/
friend void _malloc_thread_cleanup(void);
/**
* Function called when the thread exits. This is guaranteed to be called
* precisely once per thread and releases the current allocator.
*/
static inline void exit()
{
if (auto* per_thread = get(false))
{
current_alloc_pool()->release(per_thread);
per_thread = nullptr;
}
}
public:
/**
* Returns a pointer to the allocator associated with this thread. If
* `create` is true, it will create an allocator if one does not exist,
* otherwise it will return `nullptr` in this case. This should be called
* with `create == false` only during thread teardown.
*
* The non-create case exists so that the `per_thread` variable can be a
* local static and not a global, allowing ODR to deduplicate it.
*/
static inline Alloc*& get(bool create = true)
{
static thread_local Alloc* per_thread;
if (!per_thread && create)
{
per_thread = current_alloc_pool()->acquire();
}
return per_thread;
}
};
/**
* Version of the `ThreadAlloc` interface that uses C++ `thread_local`
* destructors for cleanup. If a per-thread allocator is used during the
* destruction of other per-thread data, this class will create a new
* instance and register its destructor, so should eventually result in
* cleanup, but may result in allocators being returned to the global pool
* and then reacquired multiple times.
*
* This implementation depends on nothing outside of a working C++
* environment and so should be the simplest for initial bringup on an
* unsupported platform. It is currently used in the FreeBSD kernel version.
*/
class ThreadAllocThreadDestructor
{
/**
* A pointer to the allocator owned by this thread.
*/
Alloc* alloc;
/**
* Constructor. Acquires a new allocator and associates it with this
* object. There should be only one instance of this class per thread.
*/
ThreadAllocThreadDestructor() : alloc(current_alloc_pool()->acquire()) {}
/**
* Destructor. Releases the allocator owned by this thread.
*/
~ThreadAllocThreadDestructor()
{
current_alloc_pool()->release(alloc);
}
public:
/**
* Public interface, returns the allocator for this thread, constructing
* one if necessary.
*/
static inline Alloc*& get()
{
static thread_local ThreadAllocThreadDestructor per_thread;
return per_thread.alloc;
}
};
// When targeting the FreeBSD kernel, the pthread header exists, but the
// pthread symbols do not, so don't compile this because it will fail to
// link.
#ifndef FreeBSD_KERNEL
/**
* Version of the `ThreadAlloc` interface that uses thread-specific (POSIX
* threads) or Fiber-local (Windows) storage with an explicit destructor.
* Neither of the underlying mechanisms guarantee ordering, so the cleanup
* may be called before other cleanup functions or thread-local destructors.
*
* This implementation is used when using snmalloc as a library
* implementation of malloc, but not embedding it in C standard library.
* Using this implementation removes the dependency on a C++ runtime library.
*/
class ThreadAllocExplicitTLSCleanup
{
/**
* Cleanup function. This is registered with the operating system's
* thread- or fibre-local storage subsystem to clean up the per-thread
* allocator.
*/
static inline void
# ifdef _WIN32
NTAPI
# endif
thread_alloc_release(void* p)
{
Alloc** pp = (Alloc**)p;
current_alloc_pool()->release(*pp);
*pp = nullptr;
}
# ifdef _WIN32
/**
* Key type used to identify fibre-local storage.
*/
using tls_key_t = DWORD;
/**
* On Windows, construct a new fibre-local storage allocation. This
* function must not be called more than once.
*/
static inline tls_key_t tls_key_create() noexcept
{
return FlsAlloc(thread_alloc_release);
}
/**
* On Windows, store a pointer to a `thread_local` pointer to an allocator
* into fibre-local storage. This function takes a pointer to the
* `thread_local` allocation, rather than to the pointee, so that the
* cleanup function can zero the pointer.
*
* This must not be called until after `tls_key_create` has returned.
*/
static inline void tls_set_value(tls_key_t key, Alloc** value)
{
FlsSetValue(key, static_cast<void*>(value));
}
# else
/**
* Key type used for thread-specific storage.
*/
using tls_key_t = pthread_key_t;
/**
* On POSIX systems, construct a new thread-specific storage allocation.
* This function must not be called more than once.
*/
static inline tls_key_t tls_key_create() noexcept
{
tls_key_t key;
pthread_key_create(&key, thread_alloc_release);
return key;
}
/**
* On POSIX systems, store a pointer to a `thread_local` pointer to an
* allocator into fibre-local storage. This function takes a pointer to
* the `thread_local` allocation, rather than to the pointee, so that the
* cleanup function can zero the pointer.
*
* This must not be called until after `tls_key_create` has returned.
*/
static inline void tls_set_value(tls_key_t key, Alloc** value)
{
pthread_setspecific(key, static_cast<void*>(value));
}
# endif
public:
/**
* Public interface, returns the allocator for the current thread,
* constructing it if necessary.
*/
static inline Alloc*& get()
{
static thread_local Alloc* per_thread;
// If we don't have an allocator, construct one.
if (!per_thread)
{
// Construct the allocator and assign it to `per_thread` *before* doing
// anything else. This is important because `tls_key_create` may
// allocate memory and if we are providing the `malloc` implementation
// then this function must be re-entrant within a single thread. In
// this case, the second call to this function will simply return the
// allocator.
per_thread = current_alloc_pool()->acquire();
tls_key_t key = Singleton<tls_key_t, tls_key_create>::get();
// Associate the new allocator with the destructor.
tls_set_value(key, &per_thread);
}
return per_thread;
}
};
#endif
#ifdef SNMALLOC_USE_THREAD_CLEANUP
/**
* Entry point the allows libc to call into the allocator for per-thread
* cleanup.
*/
extern "C" void _malloc_thread_cleanup(void)
{
ThreadAllocLibcCleanup::exit();
}
using ThreadAlloc = ThreadAllocLibcCleanup;
#elif defined(SNMALLOC_USE_THREAD_DESTRUCTOR)
using ThreadAlloc = ThreadAllocThreadDestructor;
#elif defined(SNMALLOC_EXTERNAL_THREAD_ALLOC)
using ThreadAlloc = ThreadAllocUntypedWrapper;
#else
using ThreadAlloc = ThreadAllocExplicitTLSCleanup;
#endif
}

86
src/mem/typealloc.h Normal file
View File

@@ -0,0 +1,86 @@
#pragma once
#include "../ds/flaglock.h"
#include "../ds/mpmcstack.h"
#include "typeallocated.h"
namespace snmalloc
{
template<class T, class MemoryProvider = GlobalVirtual>
class TypeAlloc
{
private:
friend TypeAllocated<T>;
std::atomic_flag lock = ATOMIC_FLAG_INIT;
MPMCStack<T, PreZeroed> stack;
T* list = nullptr;
TypeAlloc(MemoryProvider& m) : memory_provider(m) {}
public:
MemoryProvider& memory_provider;
static TypeAlloc* make(MemoryProvider& memory_provider) noexcept
{
auto r = memory_provider.alloc_chunk(sizeof(TypeAlloc));
return new (r) TypeAlloc(memory_provider);
}
static TypeAlloc* make() noexcept
{
return make(default_memory_provider);
}
template<typename... Args>
T* alloc(Args&&... args)
{
T* p = stack.pop();
if (p != nullptr)
return p;
p = (T*)memory_provider.alloc_chunk(sizeof(T));
new (p) T(std::forward<Args...>(args)...);
FlagLock f(lock);
p->list_next = list;
list = p;
return p;
}
void dealloc(T* p)
{
// The object's destructor is not run. If the object is "reallocated", it
// is returned without the constructor being run, so the object is reused
// without re-initialisation.
stack.push(p);
}
T* extract(T* p = nullptr)
{
// Returns a linked list of all objects in the stack, emptying the stack.
if (p == nullptr)
return stack.pop_all();
else
return p->next;
}
void restore(T* first, T* last)
{
// Pushes a linked list of objects onto the stack. Use to put a linked
// list returned by extract back onto the stack.
stack.push(first, last);
}
T* iterate(T* p = nullptr)
{
if (p == nullptr)
return list;
else
return p->list_next;
}
};
}

19
src/mem/typeallocated.h Normal file
View File

@@ -0,0 +1,19 @@
#pragma once
#include "../ds/bits.h"
namespace snmalloc
{
template<class T>
class TypeAllocated
{
private:
template<class TT, class MemoryProvider>
friend class TypeAlloc;
template<class TT, Construction c>
friend class MPMCStack;
std::atomic<T*> next = nullptr;
T* list_next;
};
}

188
src/override/malloc.cc Normal file
View File

@@ -0,0 +1,188 @@
#include "../snmalloc.h"
#include <errno.h>
using namespace snmalloc;
#ifndef SNMALLOC_NAME_MANGLE
# define SNMALLOC_NAME_MANGLE(a) a
#endif
extern "C"
{
void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr)
{
return Alloc::external_pointer<End>(ptr);
}
void* SNMALLOC_NAME_MANGLE(malloc)(size_t size)
{
// Include size 0 in the first sizeclass.
size = ((size - 1) >> (bits::BITS - 1)) + size;
return ThreadAlloc::get()->alloc(size);
}
void SNMALLOC_NAME_MANGLE(free)(void* ptr)
{
if (ptr == nullptr)
return;
ThreadAlloc::get()->dealloc(ptr);
}
void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size)
{
bool overflow = false;
size_t sz = bits::umul(size, nmemb, overflow);
if (overflow)
{
errno = ENOMEM;
return 0;
}
// Include size 0 in the first sizeclass.
sz = ((sz - 1) >> (bits::BITS - 1)) + sz;
return ThreadAlloc::get()->alloc<ZeroMem::YesZero>(sz);
}
size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)(void* ptr)
{
return Alloc::alloc_size(ptr);
}
void* SNMALLOC_NAME_MANGLE(realloc)(void* ptr, size_t size)
{
if (size == (size_t)-1)
{
errno = ENOMEM;
return nullptr;
}
if (ptr == nullptr)
{
return SNMALLOC_NAME_MANGLE(malloc)(size);
}
if (size == 0)
{
SNMALLOC_NAME_MANGLE(free)(ptr);
return nullptr;
}
#ifndef NDEBUG
// This check is redundant, because the check in memcpy will fail if this
// is skipped, but it's useful for debugging.
if (Alloc::external_pointer<Start>(ptr) != ptr)
{
error(
"Calling realloc on pointer that is not to the start of an allocation");
}
#endif
void* p = SNMALLOC_NAME_MANGLE(malloc)(size);
if (p)
{
assert(p == Alloc::external_pointer<Start>(p));
size_t sz =
(std::min)(size, SNMALLOC_NAME_MANGLE(malloc_usable_size)(ptr));
memcpy(p, ptr, sz);
SNMALLOC_NAME_MANGLE(free)(ptr);
}
return p;
}
#ifndef __FreeBSD__
void* SNMALLOC_NAME_MANGLE(reallocarray)(void* ptr, size_t nmemb, size_t size)
{
bool overflow = false;
size_t sz = bits::umul(size, nmemb, overflow);
if (overflow)
{
errno = ENOMEM;
return nullptr;
}
return SNMALLOC_NAME_MANGLE(realloc)(ptr, sz);
}
#endif
void* SNMALLOC_NAME_MANGLE(aligned_alloc)(size_t alignment, size_t size)
{
assert((size % alignment) == 0);
(void)alignment;
return SNMALLOC_NAME_MANGLE(malloc)(size);
}
void* SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size)
{
if (
(alignment == 0) || (alignment == size_t(-1)) ||
(alignment > SUPERSLAB_SIZE))
{
errno = EINVAL;
return nullptr;
}
if ((size + alignment) < size)
{
errno = ENOMEM;
return nullptr;
}
uint8_t sc = size_to_sizeclass((std::max)(size, alignment));
if (sc >= NUM_SIZECLASSES)
{
// large allocs are 16M aligned.
return SNMALLOC_NAME_MANGLE(malloc)(size);
}
for (; sc < NUM_SIZECLASSES; sc++)
{
size = sizeclass_to_size(sc);
if ((size & -size) >= alignment)
{
return SNMALLOC_NAME_MANGLE(aligned_alloc)(alignment, size);
}
}
assert(false);
return nullptr;
}
int SNMALLOC_NAME_MANGLE(posix_memalign)(
void** memptr, size_t alignment, size_t size)
{
if (
((alignment % sizeof(void*)) != 0) ||
((alignment & (alignment - 1)) != 0) || (alignment == 0))
{
return EINVAL;
}
void* p = SNMALLOC_NAME_MANGLE(memalign)(alignment, size);
if (p == nullptr)
{
return ENOMEM;
}
*memptr = p;
return 0;
}
#ifndef __FreeBSD__
void* SNMALLOC_NAME_MANGLE(valloc)(size_t size)
{
return SNMALLOC_NAME_MANGLE(memalign)(OS_PAGE_SIZE, size);
}
#endif
void* SNMALLOC_NAME_MANGLE(pvalloc)(size_t size)
{
if (size == size_t(-1))
{
errno = ENOMEM;
return nullptr;
}
return SNMALLOC_NAME_MANGLE(memalign)(
OS_PAGE_SIZE, (size + OS_PAGE_SIZE - 1) & ~(OS_PAGE_SIZE - 1));
}
void SNMALLOC_NAME_MANGLE(_malloc_prefork)(void) {}
void SNMALLOC_NAME_MANGLE(_malloc_postfork)(void) {}
void SNMALLOC_NAME_MANGLE(_malloc_first_thread)(void) {}
int SNMALLOC_NAME_MANGLE(mallctl)(const char*, void*, size_t*, void*, size_t)
{
return ENOENT;
}
}

67
src/override/new.cc Normal file
View File

@@ -0,0 +1,67 @@
#include "../mem/alloc.h"
#include "../mem/threadalloc.h"
#include "../snmalloc.h"
#ifdef _WIN32
# define EXCEPTSPEC
#else
# ifdef _GLIBCXX_USE_NOEXCEPT
# define EXCEPTSPEC _GLIBCXX_USE_NOEXCEPT
# elif defined(_NOEXCEPT)
# define EXCEPTSPEC _NOEXCEPT
# else
# define EXCEPTSPEC
# endif
#endif
using namespace snmalloc;
void* operator new(size_t size)
{
return ThreadAlloc::get()->alloc(size);
}
void* operator new[](size_t size)
{
return ThreadAlloc::get()->alloc(size);
}
void* operator new(size_t size, std::nothrow_t&)
{
return ThreadAlloc::get()->alloc(size);
}
void* operator new[](size_t size, std::nothrow_t&)
{
return ThreadAlloc::get()->alloc(size);
}
void operator delete(void* p)EXCEPTSPEC
{
ThreadAlloc::get()->dealloc(p);
}
void operator delete(void* p, size_t size)EXCEPTSPEC
{
ThreadAlloc::get()->dealloc(p, size);
}
void operator delete(void* p, std::nothrow_t&)
{
ThreadAlloc::get()->dealloc(p);
}
void operator delete[](void* p) EXCEPTSPEC
{
ThreadAlloc::get()->dealloc(p);
}
void operator delete[](void* p, size_t size) EXCEPTSPEC
{
ThreadAlloc::get()->dealloc(p, size);
}
void operator delete[](void* p, std::nothrow_t&)
{
ThreadAlloc::get()->dealloc(p);
}

46
src/pal/pal.h Normal file
View File

@@ -0,0 +1,46 @@
#pragma once
namespace snmalloc
{
void error(const char* const str);
}
// If simultating OE, then we need the underlying platform
#if !defined(OPEN_ENCLAVE) || defined(OPEN_ENCLAVE_SIMULATION)
# include "pal_free_bsd_kernel.h"
# include "pal_freebsd.h"
# include "pal_linux.h"
# include "pal_windows.h"
#endif
#include "pal_open_enclave.h"
#include "pal_plain.h"
namespace snmalloc
{
#if !defined(OPEN_ENCLAVE) || defined(OPEN_ENCLAVE_SIMULATION)
using DefaultPal =
# if defined(_WIN32)
PALWindows;
# elif defined(__linux__)
PALLinux;
# elif defined(FreeBSD_KERNEL)
PALFreeBSDKernel;
# elif defined(__FreeBSD__)
PALFBSD;
# endif
#endif
using Pal =
#ifdef OPEN_ENCLAVE
PALPlainMixin<PALOpenEnclave>;
#elif defined(SNMALLOC_MEMORY_PROVIDER)
PALPlainMixin<SNMALLOC_MEMORY_PROVIDER>
#else
DefaultPal;
#endif
inline void error(const char* const str)
{
Pal::error(str);
}
}

90
src/pal/pal_free_bsd_kernel.h Normal file
View File

@@ -0,0 +1,90 @@
#pragma once
#include "../ds/bits.h"
#include "../mem/allocconfig.h"
#if defined(FreeBSD_KERNEL)
extern "C"
{
# include <sys/vmem.h>
# include <vm/vm.h>
# include <vm/vm_extern.h>
# include <vm/vm_kern.h>
# include <vm/vm_object.h>
# include <vm/vm_param.h>
}
namespace snmalloc
{
class PALFreeBSDKernel
{
vm_offset_t get_vm_offset(uint_ptr_t p)
{
return static_cast<vm_offset_t>(reinterpret_cast<uintptr_t>(p));
}
public:
void error(const char* const str)
{
panic("snmalloc error: %s", str);
}
/// Notify platform that we will not be using these pages
void notify_not_using(void* p, size_t size)
{
vm_offset_t addr = get_vm_offset(p);
kmem_unback(kernel_object, addr, size);
}
/// Notify platform that we will not be using these pages
template<ZeroMem zero_mem>
void notify_using(void* p, size_t size)
{
vm_offset_t addr = get_vm_offset(p);
int flags = M_WAITOK | ((zero_mem == YesZero) ? M_ZERO : 0);
if (kmem_back(kernel_object, addr, size, flags) != KERN_SUCCESS)
{
error("Out of memory");
}
}
/// OS specific function for zeroing memory
template<bool page_aligned = false>
void zero(void* p, size_t size)
{
::bzero(p, size);
}
template<bool committed>
void* reserve(size_t* size, size_t align)
{
size_t request = *size;
vm_offset_t addr;
if (vmem_xalloc(
kernel_arena,
request,
align,
0,
0,
VMEM_ADDR_MIN,
VMEM_ADDR_MAX,
M_BESTFIT,
&addr))
{
return nullptr;
}
if (committed)
{
if (
kmem_back(kernel_object, addr, request, M_ZERO | M_WAITOK) !=
KERN_SUCCESS)
{
vmem_xfree(kernel_arena, addr, request);
return nullptr;
}
}
return get_vm_offset(addr);
}
};
}
#endif

88
src/pal/pal_freebsd.h Normal file
View File

@@ -0,0 +1,88 @@
#pragma once
#if defined(__FreeBSD__) && !defined(_KERNEL)
# include "../ds/bits.h"
# include "../mem/allocconfig.h"
# include <sys/mman.h>
namespace snmalloc
{
class PALFBSD
{
public:
static void error(const char* const str)
{
puts(str);
abort();
}
/// Notify platform that we will not be using these pages
void notify_not_using(void* p, size_t size) noexcept
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
madvise(p, size, MADV_FREE);
}
/// Notify platform that we will not be using these pages
template<ZeroMem zero_mem>
void notify_using(void* p, size_t size) noexcept
{
assert(
bits::is_aligned_block<OS_PAGE_SIZE>(p, size) || (zero_mem == NoZero));
if (zero_mem == YesZero)
zero(p, size);
}
/// OS specific function for zeroing memory
template<bool page_aligned = false>
void zero(void* p, size_t size) noexcept
{
if (page_aligned || bits::is_aligned_block<OS_PAGE_SIZE>(p, size))
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
void* r = mmap(
p,
size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED,
-1,
0);
if (r != MAP_FAILED)
return;
}
bzero(p, size);
}
template<bool committed>
void* reserve(size_t* size, size_t align) noexcept
{
size_t request = *size;
// Alignment must be a power of 2.
assert(align == bits::next_pow2(align));
if (align == 0)
{
align = 1;
}
size_t log2align = bits::next_pow2_bits(align);
void* p = mmap(
NULL,
request,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED(log2align),
-1,
0);
if (p == MAP_FAILED)
error("Out of memory");
return p;
}
};
}
#endif

93
src/pal/pal_linux.h Normal file
View File

@@ -0,0 +1,93 @@
#pragma once
#if defined(__linux__)
# include "../ds/bits.h"
# include "../mem/allocconfig.h"
# include <stdio.h>
# include <string.h>
# include <sys/mman.h>
namespace snmalloc
{
class PALLinux
{
public:
static void error(const char* const str)
{
puts(str);
abort();
}
/// Notify platform that we will not be using these pages
void notify_not_using(void* p, size_t size) noexcept
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
// Do nothing. Don't call madvise here, as the system call slows the
// allocator down too much.
UNUSED(p);
UNUSED(size);
}
/// Notify platform that we will not be using these pages
template<ZeroMem zero_mem>
void notify_using(void* p, size_t size) noexcept
{
assert(
bits::is_aligned_block<OS_PAGE_SIZE>(p, size) || (zero_mem == NoZero));
if (zero_mem == YesZero)
zero<true>(p, size);
}
/// OS specific function for zeroing memory
template<bool page_aligned = false>
void zero(void* p, size_t size) noexcept
{
if (page_aligned || bits::is_aligned_block<OS_PAGE_SIZE>(p, size))
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
madvise(p, size, MADV_DONTNEED);
}
else
{
::memset(p, 0, size);
}
}
template<bool committed>
void* reserve(size_t* size, size_t align) noexcept
{
size_t request = *size;
// Add align, so we can guarantee to provide at least size.
request += align;
// Alignment must be a power of 2.
assert(align == bits::next_pow2(align));
void* p = mmap(
NULL,
request,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
-1,
0);
if (p == MAP_FAILED)
error("Out of memory");
*size = request;
uintptr_t p0 = (uintptr_t)p;
uintptr_t start = bits::align_up(p0, align);
if (start > (uintptr_t)p0)
{
uintptr_t end = bits::align_down(p0 + request, align);
*size = end - start;
munmap(p, start - p0);
munmap((void*)end, (p0 + request) - end);
p = (void*)start;
}
return p;
}
};
}
#endif

58
src/pal/pal_open_enclave.h Normal file
View File

@@ -0,0 +1,58 @@
#pragma once
#include "pal_plain.h"
#ifdef OPEN_ENCLAVE
extern "C" const void* __oe_get_heap_base();
extern "C" const void* __oe_get_heap_end();
extern "C" void* oe_memset(void* p, int c, size_t size);
extern "C" void oe_abort();
namespace snmalloc
{
class PALOpenEnclave
{
std::atomic<uintptr_t> oe_base;
public:
static void error(const char* const str)
{
UNUSED(str);
oe_abort();
}
template<bool committed>
void* reserve(size_t* size, size_t align) noexcept
{
if (oe_base == 0)
{
uintptr_t dummy = 0;
oe_base.compare_exchange_strong(dummy, (uintptr_t)__oe_get_heap_base());
}
uintptr_t old_base = oe_base;
uintptr_t old_base2 = old_base;
uintptr_t next_base;
auto end = (uintptr_t)__oe_get_heap_end();
do
{
old_base2 = old_base;
auto new_base = bits::align_up(old_base, align);
next_base = new_base + *size;
if (next_base > end)
error("Out of memory");
} while (oe_base.compare_exchange_strong(old_base, next_base));
*size = next_base - old_base2;
return (void*)old_base;
}
template<bool page_aligned = false>
void zero(void* p, size_t size) noexcept
{
oe_memset(p, 0, size);
}
};
}
#endif

34
src/pal/pal_plain.h Normal file
View File

@@ -0,0 +1,34 @@
#pragma once
#include "../ds/bits.h"
#include "../mem/allocconfig.h"
namespace snmalloc
{
// Can be extended
// Will require a reserve method in subclasses.
template<class State>
class PALPlainMixin : public State
{
public:
PALPlainMixin() : State() {}
// Notify platform that we will not be using these pages
void notify_not_using(void*, size_t) noexcept {}
// Notify platform that we will not be using these pages
template<ZeroMem zero_mem>
void notify_using(void* p, size_t size) noexcept
{
if constexpr (zero_mem == YesZero)
{
State::zero(p, size);
}
else
{
UNUSED(p);
UNUSED(size);
}
}
};
}

111
src/pal/pal_windows.h Normal file
View File

@@ -0,0 +1,111 @@
#pragma once
#include "../ds/bits.h"
#include "../mem/allocconfig.h"
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
# define NOMINMAX
# include <windows.h>
namespace snmalloc
{
class PALWindows
{
public:
static void error(const char* const str)
{
puts(str);
abort();
}
/// Notify platform that we will not be using these pages
void notify_not_using(void* p, size_t size) noexcept
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
BOOL ok = VirtualFree(p, size, MEM_DECOMMIT);
if (!ok)
error("VirtualFree failed");
}
/// Notify platform that we will not be using these pages
template<ZeroMem zero_mem>
void notify_using(void* p, size_t size) noexcept
{
assert(
bits::is_aligned_block<OS_PAGE_SIZE>(p, size) || (zero_mem == NoZero));
void* r = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE);
if (r == nullptr)
error("out of memory");
}
/// OS specific function for zeroing memory
template<bool page_aligned = false>
void zero(void* p, size_t size) noexcept
{
if (page_aligned || bits::is_aligned_block<OS_PAGE_SIZE>(p, size))
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
notify_not_using(p, size);
notify_using<YesZero>(p, size);
}
else
::memset(p, 0, size);
}
# ifdef USE_SYSTEMATIC_TESTING
size_t& systematic_bump_ptr()
{
static size_t bump_ptr = (size_t)0x4000'0000'0000;
return bump_ptr;
}
# endif
template<bool committed>
void* reserve(size_t* size, size_t align) noexcept
{
// Add align, so we can guarantee to provide at least size.
size_t request = *size + align;
// Alignment must be a power of 2.
assert(align == bits::next_pow2(align));
DWORD flags = MEM_RESERVE;
if (committed)
flags |= MEM_COMMIT;
void* p;
# ifdef USE_SYSTEMATIC_TESTING
size_t retries = 1000;
do
{
p = VirtualAlloc(
(void*)systematic_bump_ptr(), request, flags, PAGE_READWRITE);
systematic_bump_ptr() += request;
retries--;
} while (p == nullptr && retries > 0);
# else
p = VirtualAlloc(nullptr, request, flags, PAGE_READWRITE);
# endif
uintptr_t aligned_p = bits::align_up((size_t)p, align);
if (aligned_p != (uintptr_t)p)
{
auto extra_bit = aligned_p - (uintptr_t)p;
uintptr_t end = (uintptr_t)p + request;
// Attempt to align end of the block.
VirtualAlloc((void*)end, extra_bit, flags, PAGE_READWRITE);
}
*size = request;
return p;
}
};
}
#endif

3
src/snmalloc.h Normal file
View File

@@ -0,0 +1,3 @@
#pragma once
#include "mem/threadalloc.h"

52
src/test/func/fixed_region/fixed_region.cc Normal file
View File

@@ -0,0 +1,52 @@
#define OPEN_ENCLAVE
#define OPEN_ENCLAVE_SIMULATION
#define USE_RESERVE_MULTIPLE 1
#include <iostream>
#include <snmalloc.h>
void* oe_base;
void* oe_end;
extern "C" const void* __oe_get_heap_base()
{
return oe_base;
}
extern "C" const void* __oe_get_heap_end()
{
return oe_end;
}
extern "C" void* oe_memset(void* p, int c, size_t size)
{
return memset(p, c, size);
}
extern "C" void oe_abort()
{
abort();
}
using namespace snmalloc;
int main()
{
DefaultPal pal;
size_t size = 1ULL << 28;
oe_base = pal.reserve<true>(&size, 0);
oe_end = (uint8_t*)oe_base + size;
std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;
auto a = ThreadAlloc::get();
for (size_t i = 0; i < 1000; i++)
{
auto r1 = a->alloc(100);
std::cout << "Allocated object " << r1 << std::endl;
if (oe_base > r1)
abort();
if (oe_end < r1)
abort();
}
}

275
src/test/func/memory/memory.cc Normal file
View File

@@ -0,0 +1,275 @@
#include <snmalloc.h>
#include <test/opt.h>
#include <test/xoroshiro.h>
#include <unordered_set>
using namespace snmalloc;
void test_alloc_dealloc_64k()
{
auto* alloc = ThreadAlloc::get();
constexpr size_t count = 1 << 12;
constexpr size_t outer_count = 12;
void* garbage[count];
void* keep_alive[outer_count];
for (size_t j = 0; j < outer_count; j++)
{
// Allocate 64k of 16byte allocs
// This will fill the short slab, and then start a new slab.
for (size_t i = 0; i < count; i++)
{
garbage[i] = alloc->alloc(16);
}
// Allocate one object on the second slab
keep_alive[j] = alloc->alloc(16);
for (size_t i = 0; i < count; i++)
{
alloc->dealloc(garbage[i]);
}
}
for (size_t j = 0; j < outer_count; j++)
{
alloc->dealloc(keep_alive[j]);
}
}
void test_random_allocation()
{
auto* alloc = ThreadAlloc::get();
std::unordered_set<void*> allocated;
constexpr size_t count = 10000;
constexpr size_t outer_count = 10;
void* objects[count];
for (size_t i = 0; i < count; i++)
objects[i] = nullptr;
// Randomly allocate and deallocate objects
xoroshiro::p128r32 r;
size_t alloc_count = 0;
for (size_t j = 0; j < outer_count; j++)
{
auto just_dealloc = r.next() % 2 == 1;
auto duration = r.next() % count;
for (size_t i = 0; i < duration; i++)
{
auto index = r.next();
auto& cell = objects[index % count];
if (cell != nullptr)
{
alloc->dealloc(cell);
allocated.erase(cell);
cell = nullptr;
alloc_count--;
}
if (!just_dealloc)
{
cell = alloc->alloc(16);
auto pair = allocated.insert(cell);
// Check not already allocated
assert(pair.second);
UNUSED(pair);
alloc_count++;
}
else
{
if (alloc_count == 0 && just_dealloc)
break;
}
}
}
// Deallocate all the remaining objects
for (size_t i = 0; i < count; i++)
if (objects[i] != nullptr)
alloc->dealloc(objects[i]);
}
void test_calloc()
{
auto* alloc = ThreadAlloc::get();
for (size_t size = 16; size <= (1 << 24); size <<= 1)
{
void* p = alloc->alloc(size);
memset(p, 0xFF, size);
alloc->dealloc(p, size);
p = alloc->alloc<YesZero>(size);
for (size_t i = 0; i < size; i++)
{
if (((char*)p)[i] != 0)
abort();
}
alloc->dealloc(p, size);
}
current_alloc_pool()->debug_check_empty();
}
void test_double_alloc()
{
auto* a1 = current_alloc_pool()->acquire();
auto* a2 = current_alloc_pool()->acquire();
const size_t n = (1 << 16) / 32;
for (size_t k = 0; k < 4; k++)
{
std::unordered_set<void*> set1;
std::unordered_set<void*> set2;
for (size_t i = 0; i < (n * 2); i++)
{
void* p = a1->alloc(20);
assert(set1.find(p) == set1.end());
set1.insert(p);
}
for (size_t i = 0; i < (n * 2); i++)
{
void* p = a2->alloc(20);
assert(set2.find(p) == set2.end());
set2.insert(p);
}
while (!set1.empty())
{
auto it = set1.begin();
a2->dealloc(*it, 20);
set1.erase(it);
}
while (!set2.empty())
{
auto it = set2.begin();
a1->dealloc(*it, 20);
set2.erase(it);
}
}
current_alloc_pool()->release(a1);
current_alloc_pool()->release(a2);
current_alloc_pool()->debug_check_empty();
}
void test_external_pointer()
{
// Malloc does not have an external pointer querying mechanism.
auto* alloc = ThreadAlloc::get();
for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++)
{
size_t size = sizeclass_to_size(sc);
void* p1 = alloc->alloc(size);
for (size_t offset = 0; offset < size; offset += 17)
{
void* p2 = (void*)((size_t)p1 + offset);
void* p3 = Alloc::external_pointer(p2);
void* p4 = Alloc::external_pointer<End>(p2);
UNUSED(p3);
UNUSED(p4);
assert(p1 == p3);
assert((size_t)p4 == (size_t)p1 + size - 1);
}
alloc->dealloc(p1, size);
}
current_alloc_pool()->debug_check_empty();
};
void check_offset(void* base, void* interior)
{
void* calced_base = Alloc::external_pointer((void*)interior);
if (calced_base != (void*)base)
abort();
}
void check_external_pointer_large(size_t* base)
{
size_t size = *base;
char* curr = (char*)base;
for (size_t offset = 0; offset < size; offset += 1 << 24)
{
check_offset(base, (void*)(curr + offset));
check_offset(base, (void*)(curr + offset + (1 << 24) - 1));
}
}
void test_external_pointer_large()
{
xoroshiro::p128r64 r;
auto* alloc = ThreadAlloc::get();
constexpr size_t count_log = 5;
constexpr size_t count = 1 << count_log;
// Pre allocate all the objects
size_t* objects[count];
for (size_t i = 0; i < count; i++)
{
size_t rand = r.next() & ((1 << 28) - 1);
size_t size = (1 << 24) + rand;
// store object
objects[i] = (size_t*)alloc->alloc(size);
// Store allocators size for this object
*objects[i] = Alloc::alloc_size(objects[i]);
check_external_pointer_large(objects[i]);
if (i > 0)
check_external_pointer_large(objects[i - 1]);
}
for (size_t i = 0; i < count; i++)
{
check_external_pointer_large(objects[i]);
}
// Deallocate everything
for (size_t i = 0; i < count; i++)
{
alloc->dealloc(objects[i]);
}
}
void test_alloc_16M()
{
auto* alloc = ThreadAlloc::get();
// sizes >= 16M use large_alloc
const size_t size = 16'000'000;
void* p1 = alloc->alloc(size);
assert(Alloc::alloc_size(Alloc::external_pointer(p1)) >= size);
alloc->dealloc(p1);
}
int main(int argc, char** argv)
{
#ifdef USE_SYSTEMATIC_TESTING
opt::Opt opt(argc, argv);
size_t seed = opt.is<size_t>("--seed", 0);
Virtual::systematic_bump_ptr() += seed << 17;
#else
UNUSED(argc);
UNUSED(argv);
#endif
test_external_pointer_large();
test_alloc_dealloc_64k();
test_random_allocation();
test_calloc();
test_double_alloc();
test_external_pointer();
test_alloc_16M();
return 0;
}

36
src/test/func/rounding/rounding.cc Normal file
View File

@@ -0,0 +1,36 @@
#include <snmalloc.h>
#include <test/opt.h>
#include <test/xoroshiro.h>
#include <unordered_set>
using namespace snmalloc;
// Check for all sizeclass that we correctly round every offset within
// a superslab to the correct value, by comparing with the standard
// unoptimised version using division.
// Also check we correctly determine multiples using optimized check.
int main(int argc, char** argv)
{
UNUSED(argc);
UNUSED(argv);
for (size_t size_class = 0; size_class < NUM_SIZECLASSES; size_class++)
{
size_t rsize = sizeclass_to_size((uint8_t)size_class);
for (size_t offset = 0; offset < SUPERSLAB_SIZE; offset++)
{
size_t rounded = (offset / rsize) * rsize;
bool mod_0 = (offset % rsize) == 0;
size_t opt_rounded = round_by_sizeclass(rsize, offset);
if (rounded != opt_rounded)
abort();
bool opt_mod_0 = is_multiple_of_sizeclass(rsize, offset);
if (opt_mod_0 != mod_0)
abort();
}
}
return 0;
}

249
src/test/histogram.h Normal file
View File

@@ -0,0 +1,249 @@
#pragma once
#ifdef USE_MEASURE
# include "../ds/flaglock.h"
# include <algorithm>
# include <iomanip>
# include <iostream>
# define MEASURE_TIME_MARKERS(id, minbits, maxbits, markers) \
static constexpr const char* const id##_time_markers[] = markers; \
static histogram::Global<histogram::Histogram<uint64_t, minbits, maxbits>> \
id##_time_global(#id, __FILE__, __LINE__, id##_time_markers); \
static thread_local histogram::Histogram<uint64_t, minbits, maxbits> \
id##_time_local(id##_time_global); \
histogram::MeasureTime<histogram::Histogram<uint64_t, minbits, maxbits>> \
id##_time(id##_time_local);
# define MEASURE_TIME(id, minbits, maxbits) \
MEASURE_TIME_MARKERS(id, minbits, maxbits, {nullptr})
# define MARKERS(...) \
{ \
__VA_ARGS__, nullptr \
}
namespace histogram
{
using namespace snmalloc;
template<class H>
class Global;
template<
class V,
size_t LOW_BITS,
size_t HIGH_BITS,
size_t INTERMEDIATE_BITS = LOW_BITS>
class Histogram
{
public:
using This = Histogram<V, LOW_BITS, HIGH_BITS, INTERMEDIATE_BITS>;
friend Global<This>;
static_assert(LOW_BITS < HIGH_BITS, "LOW_BITS must be less than HIGH_BITS");
static constexpr V LOW = (V)((size_t)1 << LOW_BITS);
static constexpr V HIGH = (V)((size_t)1 << HIGH_BITS);
static constexpr size_t BUCKETS =
((HIGH_BITS - LOW_BITS) << INTERMEDIATE_BITS) + 2;
private:
V high = (std::numeric_limits<V>::min)();
size_t overflow;
size_t count[BUCKETS];
Global<This>* global;
public:
Histogram() : global(nullptr) {}
Histogram(Global<This>& g) : global(&g) {}
~Histogram()
{
if (global != nullptr)
global->add(*this);
}
void record(V value)
{
if (value > high)
high = value;
if (value >= HIGH)
{
overflow++;
}
else
{
auto i = get_index(value);
assert(i < BUCKETS);
count[i]++;
}
}
V get_high()
{
return high;
}
size_t get_overflow()
{
return overflow;
}
size_t get_buckets()
{
return BUCKETS;
}
size_t get_count(size_t index)
{
if (index >= BUCKETS)
return 0;
return count[index];
}
static std::pair<V, V> get_range(size_t index)
{
if (index >= BUCKETS)
return std::make_pair(HIGH, HIGH);
if (index == 0)
return std::make_pair(0, get_value(index));
return std::make_pair(get_value(index - 1) + 1, get_value(index));
}
void add(This& that)
{
high = (std::max)(high, that.high);
overflow += that.overflow;
for (size_t i = 0; i < BUCKETS; i++)
count[i] += that.count[i];
}
void print(std::ostream& o)
{
o << "\tHigh: " << high << std::endl
<< "\tOverflow: " << overflow << std::endl;
size_t grand_total = overflow;
for (size_t i = 0; i < BUCKETS; i++)
grand_total += count[i];
size_t old_percentage = 0;
size_t cumulative_total = 0;
for (size_t i = 0; i < BUCKETS; i++)
{
auto r = get_range(i);
cumulative_total += count[i];
o << "\t" << std::setfill(' ') << std::setw(6) << std::get<0>(r) << ".."
<< std::setfill(' ') << std::setw(6) << std::get<1>(r) << ": "
<< std::setfill(' ') << std::setw(10) << count[i];
auto percentage = (cumulative_total * 100 / grand_total);
if (percentage != old_percentage)
{
old_percentage = percentage;
o << std::setfill(' ') << std::setw(20)
<< (cumulative_total * 100 / grand_total) << "%";
}
o << std::endl;
}
}
static size_t get_index(V value)
{
return bits::to_exp_mant<INTERMEDIATE_BITS, LOW_BITS - INTERMEDIATE_BITS>(
value);
}
static V get_value(size_t index)
{
return bits::
from_exp_mant<INTERMEDIATE_BITS, LOW_BITS - INTERMEDIATE_BITS>(index);
}
};
template<class H>
class Global
{
private:
const char* name;
const char* file;
size_t line;
const char* const* markers;
std::atomic_flag lock = ATOMIC_FLAG_INIT;
H aggregate;
public:
Global(
const char* name_,
const char* file_,
size_t line_,
const char* const* markers)
: name(name_), file(file_), line(line_), markers(markers)
{}
~Global()
{
print();
}
void add(H& histogram)
{
FlagLock f(lock);
aggregate.add(histogram);
}
private:
void print()
{
std::cout << name;
if (markers != nullptr)
{
std::cout << ": ";
size_t i = 0;
while (markers[i] != nullptr)
std::cout << markers[i++] << " ";
}
std::cout << std::endl << file << ":" << line << std::endl;
aggregate.print(std::cout);
}
};
template<class H>
class MeasureTime
{
private:
H& histogram;
uint64_t t;
public:
MeasureTime(H& histogram_) : histogram(histogram_)
{
t = bits::benchmark_time_start();
}
~MeasureTime()
{
histogram.record(bits::benchmark_time_end() - t);
}
};
}
#else
# define MEASURE_TIME(id, minbits, maxbits)
# define MEASURE_TIME_MARKERS(id, minbits, maxbits, markers)
#endif

14
src/test/measuretime.h Normal file
View File

@@ -0,0 +1,14 @@
#pragma once
#include <chrono>
#include <iomanip>
#include <iostream>
#define DO_TIME(name, code) \
{ \
auto start__ = std::chrono::high_resolution_clock::now(); \
code auto finish__ = std::chrono::high_resolution_clock::now(); \
auto diff__ = finish__ - start__; \
std::cout << name << ": " << std::setw(12) << diff__.count() << " ns" \
<< std::endl; \
}

91
src/test/opt.h Normal file
View File

@@ -0,0 +1,91 @@
#pragma once
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <type_traits>
namespace opt
{
class Opt
{
private:
int argc;
char** argv;
public:
Opt(int argc, char** argv) : argc(argc), argv(argv) {}
bool has(const char* opt)
{
for (int i = 1; i < argc; i++)
{
if (!strcmp(opt, argv[i]))
return true;
}
return false;
}
template<class T>
T is(const char* opt, T def)
{
size_t len = strlen(opt);
for (int i = 1; i < argc; i++)
{
const char* p = param(opt, len, i);
if (p != nullptr)
{
char* end = nullptr;
T r;
if (std::is_unsigned<T>::value)
r = (T)strtoull(p, &end, 10);
else
r = (T)strtoll(p, &end, 10);
if ((r == 0) && (end == p))
return def;
return r;
}
}
return def;
}
const char* is(const char* opt, const char* def)
{
size_t len = strlen(opt);
for (int i = 1; i < argc; i++)
{
const char* p = param(opt, len, i);
if (p != nullptr)
return p;
}
return def;
}
private:
const char* param(const char* opt, size_t len, int i)
{
if (strncmp(opt, argv[i], len))
return nullptr;
switch (argv[i][len])
{
case '\0':
return (i < (argc - 1)) ? argv[i + 1] : nullptr;
case '=':
return &argv[i][len + 1];
default:
return nullptr;
}
}
};
}

176
src/test/perf/contention/contention.cc Normal file
View File

@@ -0,0 +1,176 @@
#include "test/measuretime.h"
#include "test/opt.h"
#include "test/usage.h"
#include "test/xoroshiro.h"
#include <iomanip>
#include <iostream>
#include <snmalloc.h>
#include <thread>
#include <vector>
using namespace snmalloc;
bool use_malloc = false;
template<void f(size_t id)>
class ParallelTest
{
private:
std::atomic<bool> flag = false;
std::atomic<size_t> ready = 0;
uint64_t start;
uint64_t end;
std::atomic<size_t> complete = 0;
size_t cores;
void run(size_t id)
{
auto prev = ready.fetch_add(1);
if (prev + 1 == cores)
{
start = bits::tick();
flag = true;
}
while (!flag)
bits::pause();
f(id);
prev = complete.fetch_add(1);
if (prev + 1 == cores)
{
end = bits::tick();
}
}
public:
ParallelTest(size_t cores) : cores(cores)
{
std::thread* t = new std::thread[cores];
for (size_t i = 0; i < cores; i++)
{
t[i] = std::thread(&ParallelTest::run, this, i);
}
// Wait for all the threads.
for (size_t i = 0; i < cores; i++)
{
t[i].join();
}
delete[] t;
}
uint64_t time()
{
return end - start;
}
};
std::atomic<size_t*>* contention;
size_t swapsize;
size_t swapcount;
void test_tasks_f(size_t id)
{
Alloc* a = ThreadAlloc::get();
xoroshiro::p128r32 r(id + 5000);
for (size_t n = 0; n < swapcount; n++)
{
size_t size = 16 + (r.next() % 1024);
size_t* res = (size_t*)(use_malloc ? malloc(size) : a->alloc(size));
*res = size;
size_t* out =
contention[n % swapsize].exchange(res, std::memory_order_relaxed);
if (out != nullptr)
{
size = *out;
if (use_malloc)
free(out);
else
a->dealloc(out, size);
}
}
};
void test_tasks(size_t num_tasks, size_t count, size_t size)
{
Alloc* a = ThreadAlloc::get();
contention = new std::atomic<size_t*>[size];
xoroshiro::p128r32 r;
for (size_t n = 0; n < size; n++)
{
size_t alloc_size = 16 + (r.next() % 1024);
size_t* res =
(size_t*)(use_malloc ? malloc(alloc_size) : a->alloc(alloc_size));
*res = alloc_size;
contention[n] = res;
}
swapcount = count;
swapsize = size;
#ifdef USE_SNMALLOC_STATS
Stats s0;
current_alloc_pool()->aggregate_stats(s0);
#endif
{
ParallelTest<test_tasks_f> test(num_tasks);
std::cout << "Task test, " << num_tasks << " threads, " << count
<< " swaps per thread " << test.time() << "ticks" << std::endl;
for (size_t n = 0; n < swapsize; n++)
{
if (contention[n] != nullptr)
{
if (use_malloc)
free(contention[n]);
else
a->dealloc(contention[n], *contention[n]);
}
}
delete[] contention;
}
#ifndef NDEBUG
current_alloc_pool()->debug_check_empty();
#endif
};
int main(int argc, char** argv)
{
opt::Opt opt(argc, argv);
size_t cores = opt.is<size_t>("--cores", 8);
size_t count = opt.is<size_t>("--swapcount", 1 << 20);
size_t size = opt.is<size_t>("--swapsize", 1 << 18);
use_malloc = opt.has("--use_malloc");
std::cout << "Allocator is " << (use_malloc ? "System" : "snmalloc")
<< std::endl;
for (size_t i = cores; i > 0; i >>= 1)
test_tasks(i, count, size);
if (opt.has("--stats"))
{
#ifdef USE_SNMALLOC_STATS
Stats s;
current_alloc_pool()->aggregate_stats(s);
s.print<Alloc>(std::cout);
#endif
usage::print_memory();
}
return 0;
}

78
src/test/perf/external_pointer/externalpointer.cc Normal file
View File

@@ -0,0 +1,78 @@
#include <snmalloc.h>
#include <test/measuretime.h>
#include <test/xoroshiro.h>
#include <unordered_set>
using namespace snmalloc;
constexpr size_t count_log = 20;
constexpr size_t count = 1 << count_log;
// Pre allocate all the objects
size_t* objects[count];
NOINLINE void setup(xoroshiro::p128r64& r, Alloc* alloc)
{
for (size_t i = 0; i < count; i++)
{
size_t rand = (size_t)r.next();
size_t offset = bits::clz(rand);
if (offset > 30)
offset = 30;
size_t size = (rand & 15) << offset;
if (size < 16)
size = 16;
// store object
objects[i] = (size_t*)alloc->alloc(size);
// Store allocators size for this object
*objects[i] = Alloc::alloc_size(objects[i]);
}
}
NOINLINE void teardown(Alloc* alloc)
{
// Deallocate everything
for (size_t i = 0; i < count; i++)
{
alloc->dealloc(objects[i]);
}
current_alloc_pool()->debug_check_empty();
}
void test_external_pointer(xoroshiro::p128r64& r)
{
auto* alloc = ThreadAlloc::get();
setup(r, alloc);
DO_TIME("External pointer queries ", {
for (size_t i = 0; i < 10000000; i++)
{
size_t rand = (size_t)r.next();
size_t oid = rand & (((size_t)1 << count_log) - 1);
size_t* external_ptr = objects[oid];
size_t size = *external_ptr;
size_t offset = (size >> 4) * (rand & 15);
size_t interior_ptr = ((size_t)external_ptr) + offset;
void* calced_external = Alloc::external_pointer((void*)interior_ptr);
if (calced_external != external_ptr)
abort();
}
});
teardown(alloc);
}
int main(int, char**)
{
xoroshiro::p128r64 r;
#if NDEBUG
size_t nn = 30;
#else
size_t nn = 3;
#endif
for (size_t n = 0; n < nn; n++)
test_external_pointer(r);
return 0;
}

83
src/test/perf/singlethread/singlethread.cc Normal file
View File

@@ -0,0 +1,83 @@
#include <snmalloc.h>
#include <test/measuretime.h>
#include <unordered_set>
using namespace snmalloc;
template<ZeroMem zero_mem>
void test_alloc_dealloc(size_t count, size_t size, bool write)
{
auto* alloc = ThreadAlloc::get();
DO_TIME(
"Count: " << std::setw(6) << count << ", Size: " << std::setw(6) << size
<< ", ZeroMem: " << (zero_mem == YesZero) << ", Write: " << write,
{
std::unordered_set<void*> set;
// alloc 1.5x objects
for (size_t i = 0; i < ((count * 3) / 2); i++)
{
void* p = alloc->alloc<zero_mem>(size);
assert(set.find(p) == set.end());
if (write)
*(int*)p = 4;
set.insert(p);
}
// free 0.25x of the objects
for (size_t i = 0; i < (count / 4); i++)
{
auto it = set.begin();
void* p = *it;
alloc->dealloc(p, size);
set.erase(it);
assert(set.find(p) == set.end());
}
// alloc 1x objects
for (size_t i = 0; i < count; i++)
{
void* p = alloc->alloc<zero_mem>(size);
assert(set.find(p) == set.end());
if (write)
*(int*)p = 4;
set.insert(p);
}
// free everything
while (!set.empty())
{
auto it = set.begin();
alloc->dealloc(*it, size);
set.erase(it);
}
});
current_alloc_pool()->debug_check_empty();
}
int main(int, char**)
{
for (size_t size = 16; size <= 128; size <<= 1)
{
test_alloc_dealloc<NoZero>(1 << 15, size, false);
test_alloc_dealloc<NoZero>(1 << 15, size, true);
test_alloc_dealloc<YesZero>(1 << 15, size, false);
test_alloc_dealloc<YesZero>(1 << 15, size, true);
}
for (size_t size = 1 << 12; size <= 1 << 17; size <<= 1)
{
test_alloc_dealloc<NoZero>(1 << 10, size, false);
test_alloc_dealloc<NoZero>(1 << 10, size, true);
test_alloc_dealloc<YesZero>(1 << 10, size, false);
test_alloc_dealloc<YesZero>(1 << 10, size, true);
}
return 0;
}

42
src/test/usage.h Normal file
View File

@@ -0,0 +1,42 @@
#pragma once
#if defined(_WIN32)
# define WIN32_LEAN_AND_MEAN
# define NOMINMAX
# include <windows.h>
// Needs to be included after windows.h
# include <psapi.h>
#endif
#include <iomanip>
#include <iostream>
namespace usage
{
void print_memory()
{
#if defined(_WIN32)
PROCESS_MEMORY_COUNTERS_EX pmc;
if (!GetProcessMemoryInfo(
GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc)))
return;
std::cout << "Memory info:" << std::endl
<< "\tPageFaultCount: " << pmc.PageFaultCount << std::endl
<< "\tPeakWorkingSetSize: " << pmc.PeakWorkingSetSize << std::endl
<< "\tWorkingSetSize: " << pmc.WorkingSetSize << std::endl
<< "\tQuotaPeakPagedPoolUsage: " << pmc.QuotaPeakPagedPoolUsage
<< std::endl
<< "\tQuotaPagedPoolUsage: " << pmc.QuotaPagedPoolUsage
<< std::endl
<< "\tQuotaPeakNonPagedPoolUsage: "
<< pmc.QuotaPeakNonPagedPoolUsage << std::endl
<< "\tQuotaNonPagedPoolUsage: " << pmc.QuotaNonPagedPoolUsage
<< std::endl
<< "\tPagefileUsage: " << pmc.PagefileUsage << std::endl
<< "\tPeakPagefileUsage: " << pmc.PeakPagefileUsage << std::endl
<< "\tPrivateUsage: " << pmc.PrivateUsage << std::endl;
#endif
}
};

71
src/test/xoroshiro.h Normal file
View File

@@ -0,0 +1,71 @@
#pragma once
#include <cstdint>
#include <cstdlib>
namespace xoroshiro
{
namespace detail
{
template<typename STATE, typename RESULT, STATE A, STATE B, STATE C>
class XorOshiro
{
private:
static constexpr unsigned STATE_BITS = 8 * sizeof(STATE);
static constexpr unsigned RESULT_BITS = 8 * sizeof(RESULT);
static_assert(
STATE_BITS >= RESULT_BITS,
"STATE must have at least as many bits as RESULT");
STATE x;
STATE y;
static inline STATE rotl(STATE x, STATE k)
{
return (x << k) | (x >> (STATE_BITS - k));
}
public:
XorOshiro(STATE x_ = 5489, STATE y_ = 0) : x(x_), y(y_)
{
// If both zero, then this does not work
if (x_ == 0 && y_ == 0)
abort();
next();
}
void set_state(STATE x_, STATE y_ = 0)
{
// If both zero, then this does not work
if (x_ == 0 && y_ == 0)
abort();
x = x_;
y = y_;
next();
}
RESULT next()
{
STATE r = x + y;
y ^= x;
x = rotl(x, A) ^ y ^ (y << B);
y = rotl(y, C);
// If both zero, then this does not work
if (x == 0 && y == 0)
abort();
return r >> (STATE_BITS - RESULT_BITS);
}
};
}
using p128r64 = detail::XorOshiro<uint64_t, uint64_t, 55, 14, 36>;
using p128r32 = detail::XorOshiro<uint64_t, uint32_t, 55, 14, 36>;
using p64r32 = detail::XorOshiro<uint32_t, uint32_t, 27, 7, 20>;
using p64r16 = detail::XorOshiro<uint32_t, uint16_t, 27, 7, 20>;
using p32r16 = detail::XorOshiro<uint16_t, uint16_t, 13, 5, 10>;
using p32r8 = detail::XorOshiro<uint16_t, uint8_t, 13, 5, 10>;
using p16r8 = detail::XorOshiro<uint8_t, uint8_t, 4, 7, 3>;
}