Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce header layering (#503)

Browse Source 
See src/snmalloc/README.md for an explanation of the layers.

Some other cleanups on the way:

Fine-grained stats support is now gone.

It's been broken for two years, it depends on iostream (which then
causes linker failures with libstdc++) and it's collecting the wrong
stats for the new design.  After discussion with @mjp41, it's better to
remove it and introduce new stats support later, rather than keep broken
code in the main branch.

Tracing was controlled with a preprocessor macro, now there's also a
CMake option.
This commit is contained in:
David Chisnall
2022-04-06 09:59:33 +01:00
committed by GitHub
parent 65ee6b2a2f
commit f6e9796bbc
132 changed files with 545 additions and 987 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
src/ds/csv.h
View File

@@ -1,55 +0,0 @@
#pragma once
#include <iostream>
#include <string>
namespace snmalloc
{
class CSVStream
{
private:
std::ostream* out;
bool first = true;
public:
class Endl
{};
Endl endl;
CSVStream(std::ostream* o) : out(o) {}
void preprint()
{
if (!first)
{
*out << ", ";
}
else
{
first = false;
}
}
CSVStream& operator<<(const std::string& str)
{
preprint();
*out << str;
return *this;
}
CSVStream& operator<<(uint64_t u)
{
preprint();
*out << u;
return *this;
}
CSVStream& operator<<(Endl)
{
*out << std::endl;
first = true;
return *this;
}
};
} // namespace snmalloc

428
src/mem/allocstats.h
View File

@@ -1,428 +0,0 @@
#pragma once
#include "../ds/bits.h"
#include "../mem/sizeclasstable.h"
#include <cstdint>
#ifdef USE_SNMALLOC_STATS
# include "../ds/csv.h"
# include <cstring>
# include <iostream>
#endif
namespace snmalloc
{
template<size_t N, size_t LARGE_N>
struct AllocStats
{
constexpr AllocStats() = default;
struct CurrentMaxPair
{
size_t current{0};
size_t max{0};
size_t used{0};
void inc()
{
current++;
used++;
if (current > max)
max++;
}
void dec()
{
// Split stats means this is not true.
// TODO reestablish checks, when we sanitise the stats.
// SNMALLOC_ASSERT(current > 0);
current--;
}
bool is_empty()
{
return current == 0;
}
bool is_unused()
{
return max == 0;
}
void add(CurrentMaxPair& that)
{
current += that.current;
max += that.max;
used += that.used;
}
#ifdef USE_SNMALLOC_STATS
void print(CSVStream& csv, size_t multiplier = 1)
{
csv << current * multiplier << max * multiplier << used * multiplier;
}
#endif
};
struct Stats
{
constexpr Stats() = default;
CurrentMaxPair count;
CurrentMaxPair slab_count;
uint64_t time{0};
uint64_t ticks{0};
double online_average{0};
bool is_empty()
{
return count.is_empty();
}
void add(Stats& that)
{
count.add(that.count);
slab_count.add(that.slab_count);
}
void addToRunningAverage()
{
uint64_t now = Aal::tick();
if (slab_count.current != 0)
{
double occupancy = static_cast<double>(count.current) /
static_cast<double>(slab_count.current);
uint64_t duration = now - time;
if (ticks == 0)
online_average = occupancy;
else
online_average +=
((occupancy - online_average) * static_cast<double>(duration)) /
static_cast<double>(ticks);
ticks += duration;
}
time = now;
}
#ifdef USE_SNMALLOC_STATS
void
print(CSVStream& csv, size_t multiplier = 1, size_t slab_multiplier = 1)
{
// Keep in sync with header lower down
count.print(csv, multiplier);
slab_count.print(csv, slab_multiplier);
size_t average =
static_cast<size_t>(online_average * static_cast<double>(multiplier));
csv << average << (slab_multiplier - average) * slab_count.max
<< csv.endl;
}
#endif
};
#ifdef USE_SNMALLOC_STATS
static constexpr size_t BUCKETS_BITS = 4;
static constexpr size_t BUCKETS = 1 << BUCKETS_BITS;
static constexpr size_t TOTAL_BUCKETS =
bits::to_exp_mant_const<BUCKETS_BITS>(
bits::one_at_bit(bits::ADDRESS_BITS - 1));
Stats sizeclass[N];
size_t large_pop_count[LARGE_N] = {0};
size_t large_push_count[LARGE_N] = {0};
size_t remote_freed = 0;
size_t remote_posted = 0;
size_t remote_received = 0;
size_t superslab_push_count = 0;
size_t superslab_pop_count = 0;
size_t superslab_fresh_count = 0;
size_t segment_count = 0;
size_t bucketed_requests[TOTAL_BUCKETS] = {};
#endif
void alloc_request(size_t size)
{
UNUSED(size);
#ifdef USE_SNMALLOC_STATS
auto index = (size == 0) ? 0 : bits::to_exp_mant<BUCKETS_BITS>(size);
SNMALLOC_ASSERT(index < TOTAL_BUCKETS);
bucketed_requests[index]++;
#endif
}
bool is_empty()
{
#ifdef USE_SNMALLOC_STATS
for (size_t i = 0; i < N; i++)
{
if (!sizeclass[i].is_empty())
return false;
}
for (size_t i = 0; i < LARGE_N; i++)
{
if (large_push_count[i] != large_pop_count[i])
return false;
}
return (remote_freed == remote_posted);
#else
return true;
#endif
}
void sizeclass_alloc(smallsizeclass_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
sizeclass[sc].addToRunningAverage();
sizeclass[sc].count.inc();
#endif
}
void sizeclass_dealloc(smallsizeclass_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
sizeclass[sc].addToRunningAverage();
sizeclass[sc].count.dec();
#endif
}
void large_alloc(size_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
SNMALLOC_ASSUME(sc < LARGE_N);
large_pop_count[sc]++;
#endif
}
void sizeclass_alloc_slab(smallsizeclass_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
sizeclass[sc].addToRunningAverage();
sizeclass[sc].slab_count.inc();
#endif
}
void sizeclass_dealloc_slab(smallsizeclass_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
sizeclass[sc].addToRunningAverage();
sizeclass[sc].slab_count.dec();
#endif
}
void large_dealloc(size_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
large_push_count[sc]++;
#endif
}
void segment_create()
{
#ifdef USE_SNMALLOC_STATS
segment_count++;
#endif
}
void superslab_pop()
{
#ifdef USE_SNMALLOC_STATS
superslab_pop_count++;
#endif
}
void superslab_push()
{
#ifdef USE_SNMALLOC_STATS
superslab_push_count++;
#endif
}
void superslab_fresh()
{
#ifdef USE_SNMALLOC_STATS
superslab_fresh_count++;
#endif
}
void remote_free(smallsizeclass_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
remote_freed += sizeclass_to_size(sc);
#endif
}
void remote_post()
{
#ifdef USE_SNMALLOC_STATS
remote_posted = remote_freed;
#endif
}
void remote_receive(smallsizeclass_t sc)
{
UNUSED(sc);
#ifdef USE_SNMALLOC_STATS
remote_received += sizeclass_to_size(sc);
#endif
}
void add(AllocStats<N, LARGE_N>& that)
{
UNUSED(that);
#ifdef USE_SNMALLOC_STATS
for (size_t i = 0; i < N; i++)
sizeclass[i].add(that.sizeclass[i]);
for (size_t i = 0; i < LARGE_N; i++)
{
large_push_count[i] += that.large_push_count[i];
large_pop_count[i] += that.large_pop_count[i];
}
for (size_t i = 0; i < TOTAL_BUCKETS; i++)
bucketed_requests[i] += that.bucketed_requests[i];
remote_freed += that.remote_freed;
remote_posted += that.remote_posted;
remote_received += that.remote_received;
superslab_pop_count += that.superslab_pop_count;
superslab_push_count += that.superslab_push_count;
superslab_fresh_count += that.superslab_fresh_count;
segment_count += that.segment_count;
#endif
}
#ifdef USE_SNMALLOC_STATS
template<class Alloc>
void print(std::ostream& o, uint64_t dumpid = 0, uint64_t allocatorid = 0)
{
UNUSED(o, dumpid, allocatorid);
CSVStream csv(&o);
if (dumpid == 0)
{
// Output headers for initial dump
// Keep in sync with data dump
csv << "GlobalStats"
<< "DumpID"
<< "AllocatorID"
<< "Remote freed"
<< "Remote posted"
<< "Remote received"
<< "Superslab pop"
<< "Superslab push"
<< "Superslab fresh"
<< "Segments" << csv.endl;
csv << "BucketedStats"
<< "DumpID"
<< "AllocatorID"
<< "Size group"
<< "Size"
<< "Current count"
<< "Max count"
<< "Total Allocs"
<< "Current Slab bytes"
<< "Max Slab bytes"
<< "Total slab allocs"
<< "Average Slab Usage"
<< "Average wasted space" << csv.endl;
csv << "LargeBucketedStats"
<< "DumpID"
<< "AllocatorID"
<< "Size group"
<< "Size"
<< "Push count"
<< "Pop count" << csv.endl;
csv << "AllocSizes"
<< "DumpID"
<< "AllocatorID"
<< "ClassID"
<< "Low size"
<< "High size"
<< "Count" << csv.endl;
}
for (smallsizeclass_t i = 0; i < N; i++)
{
if (sizeclass[i].count.is_unused())
continue;
sizeclass[i].addToRunningAverage();
csv << "BucketedStats" << dumpid << allocatorid << i
<< sizeclass_to_size(i);
sizeclass[i].print(csv, sizeclass_to_size(i));
}
// for (uint8_t i = 0; i < LARGE_N; i++)
// {
// if ((large_push_count[i] == 0) && (large_pop_count[i] == 0))
// continue;
// csv << "LargeBucketedStats" << dumpid << allocatorid << (i + N)
// << large_sizeclass_to_size(i) << large_push_count[i]
// << large_pop_count[i] << csv.endl;
// }
size_t low = 0;
size_t high = 0;
for (size_t i = 0; i < TOTAL_BUCKETS; i++)
{
low = high + 1;
high = bits::from_exp_mant<BUCKETS_BITS>(i);
if (bucketed_requests[i] == 0)
continue;
csv << "AllocSizes" << dumpid << allocatorid << i << low << high
<< bucketed_requests[i] << csv.endl;
}
csv << "GlobalStats" << dumpid << allocatorid << remote_freed
<< remote_posted << remote_received << superslab_pop_count
<< superslab_push_count << superslab_fresh_count << segment_count
<< csv.endl;
}
#endif
void start()
{
#ifdef USE_SNMALLOC_STATS
for (size_t i = 0; i < N; i++)
sizeclass[i].time = Aal::tick();
#endif
}
};
} // namespace snmalloc

28
src/override/override.h
View File

@@ -1,28 +0,0 @@
#pragma once
// Core implementation of snmalloc independent of the configuration mode
#include "../snmalloc_core.h"
#ifndef SNMALLOC_PROVIDE_OWN_CONFIG
# include "../backend/globalconfig.h"
// The default configuration for snmalloc is used if alternative not defined
namespace snmalloc
{
using Alloc = snmalloc::LocalAllocator<snmalloc::Globals>;
} // namespace snmalloc
#endif
// User facing API surface, needs to know what `Alloc` is.
#include "../snmalloc_front.h"
#ifndef SNMALLOC_EXPORT
# define SNMALLOC_EXPORT
#endif
#ifdef SNMALLOC_STATIC_LIBRARY_PREFIX
# define __SN_CONCAT(a, b) a##b
# define __SN_EVALUATE(a, b) __SN_CONCAT(a, b)
# define SNMALLOC_NAME_MANGLE(a) \
__SN_EVALUATE(SNMALLOC_STATIC_LIBRARY_PREFIX, a)
#elif !defined(SNMALLOC_NAME_MANGLE)
# define SNMALLOC_NAME_MANGLE(a) a
#endif

22
src/snmalloc.h
View File

@@ -1,22 +0,0 @@
#pragma once
// Core implementation of snmalloc independent of the configuration mode
#include "snmalloc_core.h"
// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own
// definition of `snmalloc::Alloc` and include `snmalloc_front.h` before
// including any files that include `snmalloc.h` and consume the global
// allocation APIs.
#ifndef SNMALLOC_PROVIDE_OWN_CONFIG
// Default implementation of global state
# include "backend/globalconfig.h"
// The default configuration for snmalloc
namespace snmalloc
{
using Alloc = snmalloc::LocalAllocator<snmalloc::Globals>;
}
// User facing API surface, needs to know what `Alloc` is.
# include "snmalloc_front.h"
#endif

40
src/snmalloc/README.md Normal file
View File

@@ -0,0 +1,40 @@
Include hierarchy
-----------------
The `snmalloc/` include path contains all of the snmalloc headers.
These are arranged in a hierarchy such that each of the directories may include ones below, in the following order, starting at the bottom:
- `ds_core/` provides core data structures that depend on the C++ implementation and nothing else.
This directory includes a number of things that abstract over different language extensions (for example, different built-in function names in different compilers).
- `aal/` provides the architecture abstraction layer (AAL).
This layer provides abstractions over CPU-specific intrinsics and defines things such as the virtual address-space size.
There is a single AAL for an snmalloc instantiation.
- `pal/` provides the platform abstraction layer (PAL).
This exposes OS- or environment-specific abstractions into the rest of the code.
An snmalloc instantiation may use more than one PAL, including ones provided by the user.
- `ds/` includes data structures that may depend on platform services or on features specific to the current CPU.
- `mem/` provides the core allocator abstractions.
The code here is templated over a back-end, which defines a particular embedding of snmalloc.
- `backend_helpers/` provides helper classes for use in defining a back end.
This includes data structures such as pagemap implementations (efficient maps from a chunk address to associated metadata) and buddy allocators for managing address-space ranges.
- `backend/` provides some example implementations for snmalloc embeddings that provide a global memory allocator for an address space.
Users may ignore this entirely and use the types in `mem/` with a custom back end to expose an snmalloc instance with specific behaviour.
Layers above this can be used with a custom configuration by defining `SNMALLOC_PROVIDE_OWN_CONFIG` and exporting a type as `snmalloc::Alloc` that defines the type of an `snmalloc::LocalAllocator` template specialisation.
- `global/` provides some front-end components that assume that snmalloc is available in a global configuration.
- `override/` builds on top of `global/` to provide specific implementations with compatibility with external specifications (for example C `malloc`, C++ `operator new`, jemalloc's `*allocx`, or Rust's `std::alloc`).
Each layer until `backend_helpers/` provides a single header with the same name as the directory.
Files in higher layers should depend only on the single-file version.
This allows specific files to be moved to a lower layer if appropriate, without too much code churn.
There is only one exception to this rule: `backend/globalconfig.h`.
This file defines either the default configuration *or* nothing, depending on whether the user has defined `SNMALLOC_PROVIDE_OWN_CONFIG`.
The layers above the back end should include only this file, so that there is a single interception point for externally defined back ends.
External code should include only the following files:
- `snmalloc/snmalloc_core.h` includes everything up to `backend_helpers`.
This provides the building blocks required to assemble an snmalloc instance, but does not assume any global configuration.
- `snmalloc/snmalloc_front.h` assumes a global configuration (either user-provided or the default from `snmalloc/backend/globalconfig.h` and exposes all of the functionality that depends on both.
- `snmalloc/snmalloc.h` is a convenience wrapper that includes both of the above files.
- `snmalloc/override/*.cc` can be compiled as-is or included after `snmalloc/snmalloc_core.h` and a custom global allocator definition to provide specific languages' global memory allocator APIs with a custom snmalloc embedding.

19
src/aal/aal.h → src/snmalloc/aal/aal.h
View File

@@ -1,7 +1,12 @@
/**
* The snmalloc architecture abstraction layer. This defines
* CPU-architecture-specific functionality.
*
* Files in this directory may depend on `ds_core` and each other, but nothing
* else in snmalloc.
*/
#pragma once
#include "../ds/concept.h"
#include "../ds/defines.h"
#include "../ds/ptrwrap.h"
#include "../ds_core/ds_core.h"
#include "aal_concept.h"
#include "aal_consts.h"
@@ -242,10 +247,6 @@ namespace snmalloc
constexpr static bool aal_supports = (AAL::aal_features & F) == F;
} // namespace snmalloc
#if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_32)
# include <intsafe.h>
#endif
#ifdef __POINTER_WIDTH__
# if ((__POINTER_WIDTH__ == 64) && !defined(SNMALLOC_VA_BITS_64)) || \
((__POINTER_WIDTH__ == 32) && !defined(SNMALLOC_VA_BITS_32))
@@ -262,3 +263,7 @@ static_assert(sizeof(size_t) == 4);
#elif defined(SNMALLOC_VA_BITS_64)
static_assert(sizeof(size_t) == 8);
#endif
// Included after the AAL has been defined, depends on the AAL's notion of an
// address
#include "address.h"

0
src/aal/aal_arm.h → src/snmalloc/aal/aal_arm.h
View File

2
src/aal/aal_cheri.h → src/snmalloc/aal/aal_cheri.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include "../ds/defines.h"
#include "../ds_core/ds_core.h"
#include <stddef.h>

30
src/aal/aal_concept.h → src/snmalloc/aal/aal_concept.h
View File

@@ -1,8 +1,7 @@
#pragma once
#ifdef __cpp_concepts
# include "../ds/concept.h"
# include "../ds/ptrwrap.h"
# include "../ds_core/ds_core.h"
# include "aal_consts.h"
# include <cstdint>
@@ -27,9 +26,12 @@ namespace snmalloc
* AALs provide a prefetch operation.
*/
template<typename AAL>
concept ConceptAAL_prefetch = requires(void *ptr)
concept ConceptAAL_prefetch = requires(void* ptr)
{
{ AAL::prefetch(ptr) } noexcept -> ConceptSame<void>;
{
AAL::prefetch(ptr)
}
noexcept->ConceptSame<void>;
};
/**
@@ -38,29 +40,31 @@ namespace snmalloc
template<typename AAL>
concept ConceptAAL_tick = requires()
{
{ AAL::tick() } noexcept -> ConceptSame<uint64_t>;
{
AAL::tick()
}
noexcept->ConceptSame<uint64_t>;
};
template<typename AAL>
concept ConceptAAL_capptr_methods =
requires(capptr::Chunk<void> auth, capptr::AllocFull<void> ret, size_t sz)
requires(capptr::Chunk<void> auth, capptr::AllocFull<void> ret, size_t sz)
{
/**
* Produce a pointer with reduced authority from a more privilged pointer.
* The resulting pointer will have base at auth's address and length of
* exactly sz. auth+sz must not exceed auth's limit.
*/
{ AAL::template capptr_bound<void, capptr::bounds::Chunk>(auth, sz) }
noexcept
-> ConceptSame<capptr::Chunk<void>>;
{
AAL::template capptr_bound<void, capptr::bounds::Chunk>(auth, sz)
}
noexcept->ConceptSame<capptr::Chunk<void>>;
};
template<typename AAL>
concept ConceptAAL =
ConceptAAL_static_members<AAL> &&
ConceptAAL_prefetch<AAL> &&
ConceptAAL_tick<AAL> &&
ConceptAAL_capptr_methods<AAL>;
ConceptAAL_static_members<AAL>&& ConceptAAL_prefetch<AAL>&&
ConceptAAL_tick<AAL>&& ConceptAAL_capptr_methods<AAL>;
} // namespace snmalloc
#endif

0
src/aal/aal_consts.h → src/snmalloc/aal/aal_consts.h
View File

0
src/aal/aal_powerpc.h → src/snmalloc/aal/aal_powerpc.h
View File

0
src/aal/aal_riscv.h → src/snmalloc/aal/aal_riscv.h
View File

0
src/aal/aal_sparc.h → src/snmalloc/aal/aal_sparc.h
View File

0
src/aal/aal_x86.h → src/snmalloc/aal/aal_x86.h
View File

0
src/aal/aal_x86_sgx.h → src/snmalloc/aal/aal_x86_sgx.h
View File

4
src/ds/address.h → src/snmalloc/aal/address.h
View File

@@ -1,7 +1,5 @@
#pragma once
#include "../pal/pal_consts.h"
#include "bits.h"
#include "ptrwrap.h"
#include "../ds_core/ds_core.h"
#include <cstdint>

16
src/backend/backend.h → src/snmalloc/backend/backend.h
View File

@@ -1,19 +1,5 @@
#pragma once
#include "../mem/allocconfig.h"
#include "../mem/metadata.h"
#include "../pal/pal.h"
#include "commitrange.h"
#include "commonconfig.h"
#include "empty_range.h"
#include "globalrange.h"
#include "largebuddyrange.h"
#include "pagemap.h"
#include "pagemapregisterrange.h"
#include "palrange.h"
#include "range_helpers.h"
#include "smallbuddyrange.h"
#include "statsrange.h"
#include "subrange.h"
#include "../backend_helpers/backend_helpers.h"
#if defined(SNMALLOC_CHECK_CLIENT) && !defined(OPEN_ENCLAVE)
/**

3
src/backend/fixedglobalconfig.h → src/snmalloc/backend/fixedglobalconfig.h
View File

@@ -1,9 +1,6 @@
#pragma once
#include "../backend/backend.h"
#include "../mem/corealloc.h"
#include "../mem/pool.h"
#include "commonconfig.h"
namespace snmalloc
{

31
src/backend/globalconfig.h → src/snmalloc/backend/globalconfig.h
View File

@@ -1,19 +1,17 @@
#pragma once
// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own
// definition of `snmalloc::Alloc` before including any files that include
// `snmalloc.h` or consume the global allocation APIs.
#ifndef SNMALLOC_PROVIDE_OWN_CONFIG
#include "../backend/backend.h"
#include "../mem/corealloc.h"
#include "../mem/pool.h"
#include "commonconfig.h"
# include "../backend/backend.h"
#ifdef SNMALLOC_TRACING
# include <iostream>
#endif
namespace snmalloc
{
// Forward reference to thread local cleanup.
void register_clean_up();
#ifdef USE_SNMALLOC_STATS
# ifdef USE_SNMALLOC_STATS
inline static void print_stats()
{
printf("No Stats yet!");
@@ -21,7 +19,7 @@ namespace snmalloc
// current_alloc_pool()->aggregate_stats(s);
// s.print<Alloc>(std::cout);
}
#endif
# endif
/**
* The default configuration for a global snmalloc. This allocates memory
@@ -59,9 +57,9 @@ namespace snmalloc
static void ensure_init()
{
FlagLock lock{initialisation_lock};
#ifdef SNMALLOC_TRACING
# ifdef SNMALLOC_TRACING
message<1024>("Run init_impl");
#endif
# endif
if (initialised)
return;
@@ -74,9 +72,9 @@ namespace snmalloc
// Need to initialise pagemap.
Backend::init();
#ifdef USE_SNMALLOC_STATS
# ifdef USE_SNMALLOC_STATS
atexit(snmalloc::print_stats);
#endif
# endif
initialised = true;
}
@@ -96,3 +94,10 @@ namespace snmalloc
}
};
} // namespace snmalloc
// The default configuration for snmalloc
namespace snmalloc
{
using Alloc = snmalloc::LocalAllocator<snmalloc::Globals>;
} // namespace snmalloc
#endif

14
src/snmalloc/backend_helpers/backend_helpers.h Normal file
View File

@@ -0,0 +1,14 @@
#include "../mem/mem.h"
#include "buddy.h"
#include "commitrange.h"
#include "commonconfig.h"
#include "empty_range.h"
#include "globalrange.h"
#include "largebuddyrange.h"
#include "pagemap.h"
#include "pagemapregisterrange.h"
#include "palrange.h"
#include "range_helpers.h"
#include "smallbuddyrange.h"
#include "statsrange.h"
#include "subrange.h"

6
src/backend/buddy.h → src/snmalloc/backend_helpers/buddy.h
View File

@@ -1,8 +1,6 @@
#pragma once
#include "../ds/address.h"
#include "../ds/bits.h"
#include "../ds/redblacktree.h"
#include "../ds/ds.h"
namespace snmalloc
{
@@ -119,4 +117,4 @@ namespace snmalloc
return bigger;
}
};
} // namespace snmalloc
} // namespace snmalloc

3
src/backend/commitrange.h → src/snmalloc/backend_helpers/commitrange.h
View File

@@ -1,7 +1,6 @@
#pragma once
#include "../ds/ptrwrap.h"
#include "../pal/pal_consts.h"
#include "../pal/pal.h"
namespace snmalloc
{

74
src/backend/commonconfig.h → src/snmalloc/backend_helpers/commonconfig.h
View File

@@ -1,8 +1,6 @@
#pragma once
#include "../backend/backend_concept.h"
#include "../ds/defines.h"
#include "../mem/remoteallocator.h"
#include "../mem/mem.h"
namespace snmalloc
{
@@ -108,75 +106,5 @@ namespace snmalloc
inline static RemoteAllocator unused_remote;
};
/**
* SFINAE helper. Matched only if `T` implements `is_initialised`. Calls
* it if it exists.
*/
template<typename T>
SNMALLOC_FAST_PATH auto call_is_initialised(T*, int)
-> decltype(T::is_initialised())
{
return T::is_initialised();
}
/**
* SFINAE helper. Matched only if `T` does not implement `is_initialised`.
* Unconditionally returns true if invoked.
*/
template<typename T>
SNMALLOC_FAST_PATH auto call_is_initialised(T*, long)
{
return true;
}
namespace detail
{
/**
* SFINAE helper, calls capptr_domesticate in the backend if it exists.
*/
template<
SNMALLOC_CONCEPT(ConceptBackendDomestication) Backend,
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) B>
SNMALLOC_FAST_PATH_INLINE auto
capptr_domesticate(typename Backend::LocalState* ls, CapPtr<T, B> p, int)
-> decltype(Backend::capptr_domesticate(ls, p))
{
return Backend::capptr_domesticate(ls, p);
}
/**
* SFINAE helper. If the back end does not provide special handling for
* domestication then assume all wild pointers can be domesticated.
*/
template<
SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend,
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) B>
SNMALLOC_FAST_PATH_INLINE auto
capptr_domesticate(typename Backend::LocalState*, CapPtr<T, B> p, long)
{
return CapPtr<
T,
typename B::template with_wildness<capptr::dimension::Wildness::Tame>>(
p.unsafe_ptr());
}
} // namespace detail
/**
* Wrapper that calls `Backend::capptr_domesticate` if and only if it is
* implemented. If it is not implemented then this assumes that any wild
* pointer can be domesticated.
*/
template<
SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend,
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) B>
SNMALLOC_FAST_PATH_INLINE auto
capptr_domesticate(typename Backend::LocalState* ls, CapPtr<T, B> p)
{
return detail::capptr_domesticate<Backend>(ls, p, 0);
}
} // namespace snmalloc
#include "../mem/remotecache.h"

2
src/backend/empty_range.h → src/snmalloc/backend_helpers/empty_range.h
View File

@@ -1,5 +1,5 @@
#pragma once
#include "../ds/ptrwrap.h"
#include "../ds_core/ds_core.h"
namespace snmalloc
{

6
src/backend/globalrange.h → src/snmalloc/backend_helpers/globalrange.h
View File

@@ -1,8 +1,6 @@
#pragma once
#include "../ds/defines.h"
#include "../ds/helpers.h"
#include "../ds/ptrwrap.h"
#include "../ds/ds.h"
namespace snmalloc
{
@@ -53,4 +51,4 @@ namespace snmalloc
parent->dealloc_range(base, size);
}
};
} // namespace snmalloc
} // namespace snmalloc

7
src/backend/largebuddyrange.h → src/snmalloc/backend_helpers/largebuddyrange.h
View File

@@ -1,10 +1,7 @@
#pragma once
#include "../ds/address.h"
#include "../ds/bits.h"
#include "../mem/allocconfig.h"
#include "../pal/pal.h"
#include "backend_concept.h"
#include "../ds/ds.h"
#include "../mem/mem.h"
#include "buddy.h"
#include "range_helpers.h"

6
src/backend/pagemap.h → src/snmalloc/backend_helpers/pagemap.h
View File

@@ -1,9 +1,7 @@
#pragma once
#include "../ds/bits.h"
#include "../ds/helpers.h"
#include "../mem/entropy.h"
#include "../pal/pal.h"
#include "../ds/ds.h"
#include "../mem/mem.h"
#include <atomic>
#include <utility>

3
src/backend/pagemapregisterrange.h → src/snmalloc/backend_helpers/pagemapregisterrange.h
View File

@@ -1,6 +1,5 @@
#pragma once
#include "../ds/address.h"
#include "../pal/pal.h"
namespace snmalloc
@@ -42,4 +41,4 @@ namespace snmalloc
return base;
}
};
} // namespace snmalloc
} // namespace snmalloc

3
src/backend/palrange.h → src/snmalloc/backend_helpers/palrange.h
View File

@@ -1,5 +1,4 @@
#pragma once
#include "../ds/address.h"
#include "../pal/pal.h"
namespace snmalloc
@@ -61,4 +60,4 @@ namespace snmalloc
}
}
};
} // namespace snmalloc
} // namespace snmalloc

4
src/backend/range_helpers.h → src/snmalloc/backend_helpers/range_helpers.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include "../ds/ptrwrap.h"
#include "../ds_core/ds_core.h"
namespace snmalloc
{
@@ -34,4 +34,4 @@ namespace snmalloc
length -= align;
}
}
} // namespace snmalloc
} // namespace snmalloc

2
src/backend/smallbuddyrange.h → src/snmalloc/backend_helpers/smallbuddyrange.h
View File

@@ -1,7 +1,5 @@
#pragma once
#include "../ds/address.h"
#include "../mem/allocconfig.h"
#include "../pal/pal.h"
#include "range_helpers.h"

0
src/backend/statsrange.h → src/snmalloc/backend_helpers/statsrange.h
View File

4
src/backend/subrange.h → src/snmalloc/backend_helpers/subrange.h
View File

@@ -1,5 +1,5 @@
#pragma once
#include "../mem/entropy.h"
#include "../mem/mem.h"
namespace snmalloc
{
@@ -54,4 +54,4 @@ namespace snmalloc
return pointer_offset(overblock, offset);
}
};
} // namespace snmalloc
} // namespace snmalloc

3
src/ds/aba.h → src/snmalloc/ds/aba.h
View File

@@ -1,7 +1,6 @@
#pragma once
#include "bits.h"
#include "ptrwrap.h"
#include "../aal/aal.h"
/**
* This file contains an abstraction of ABA protection. This API should be

3
src/mem/allocconfig.h → src/snmalloc/ds/allocconfig.h
View File

@@ -1,8 +1,5 @@
#pragma once
#include "../ds/bits.h"
#include "../pal/pal.h"
namespace snmalloc
{
// 0 intermediate bits results in power of 2 small allocs. 1 intermediate

11
src/snmalloc/ds/ds.h Normal file
View File

@@ -0,0 +1,11 @@
/**
* Data structures used by snmalloc.
*
*/
#pragma once
#include "../pal/pal.h"
#include "aba.h"
#include "allocconfig.h"
#include "flaglock.h"
#include "mpmcstack.h"
#include "singleton.h"

3
src/ds/flaglock.h → src/snmalloc/ds/flaglock.h
View File

@@ -1,8 +1,9 @@
#pragma once
#include "bits.h"
#include "../aal/aal.h"
#include <atomic>
#include <functional>
namespace snmalloc
{

3
src/ds/mpmcstack.h → src/snmalloc/ds/mpmcstack.h
View File

@@ -1,7 +1,8 @@
#pragma once
#include "../ds_core/ds_core.h"
#include "aba.h"
#include "ptrwrap.h"
#include "allocconfig.h"
#if defined(__has_feature)
# if __has_feature(thread_sanitizer)

51
src/snmalloc/ds/singleton.h Normal file
View File

@@ -0,0 +1,51 @@
#pragma once
#include "../ds_core/ds_core.h"
#include "flaglock.h"
#include <array>
#include <atomic>
#include <string_view>
#include <type_traits>
namespace snmalloc
{
/*
* In some use cases we need to run before any of the C++ runtime has been
* initialised. This singleton class is designed to not depend on the
* runtime.
*/
template<class Object, void init(Object*) noexcept>
class Singleton
{
inline static FlagWord flag;
inline static std::atomic<bool> initialised{false};
inline static Object obj;
public:
/**
* If argument is non-null, then it is assigned the value
* true, if this is the first call to get.
* At most one call will be first.
*/
inline SNMALLOC_SLOW_PATH static Object& get(bool* first = nullptr)
{
// If defined should be initially false;
SNMALLOC_ASSERT(first == nullptr || *first == false);
if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire)))
{
FlagLock lock(flag);
if (!initialised)
{
init(&obj);
initialised.store(true, std::memory_order_release);
if (first != nullptr)
*first = true;
}
}
return obj;
}
};
} // namespace snmalloc

52
src/ds/bits.h → src/snmalloc/ds_core/bits.h
View File

@@ -5,13 +5,16 @@
// #define USE_LZCNT
#include "../aal/aal.h"
#include "../pal/pal_consts.h"
#include "defines.h"
#include <atomic>
#include <climits>
#include <cstdint>
#include <type_traits>
#if defined(_MSC_VER)
# include <intsafe.h>
#endif
#ifdef pause
# undef pause
#endif
@@ -30,7 +33,16 @@ namespace snmalloc
namespace bits
{
static constexpr size_t BITS = Aal::bits;
/**
* The number of bits in a `size_t`. Many of the functions in the
* `snmalloc::bits` namespace are defined to operate over `size_t`, mapping
* to the correct compiler builtins irrespective of the size. This size
* does *not* imply the number of bits of virtual address space that are
* actually allowed to be set. `Aal::bits` provides an
* architecture-specific definition of the number of bits of address space
* that exist.
*/
static constexpr size_t BITS = sizeof(size_t) * CHAR_BIT;
/**
* Returns a value of type T that has a single bit set,
@@ -52,7 +64,7 @@ namespace snmalloc
SNMALLOC_ASSERT(x != 0); // Calling with 0 is UB on some implementations
#if defined(_MSC_VER)
# ifdef USE_LZCNT
# ifdef SNMALLOC_VA_BITS_64
# ifdef _WIN64
return __lzcnt64(x);
# else
return __lzcnt((uint32_t)x);
@@ -60,10 +72,10 @@ namespace snmalloc
# else
unsigned long index;
# ifdef SNMALLOC_VA_BITS_64
_BitScanReverse64(&index, x);
# ifdef _WIN64
_BitScanReverse64(&index, static_cast<unsigned __int64>(x));
# else
_BitScanReverse(&index, (unsigned long)x);
_BitScanReverse(&index, static_cast<unsigned long>(x));
# endif
return BITS - index - 1;
@@ -101,10 +113,10 @@ namespace snmalloc
inline size_t rotr(size_t x, size_t n)
{
#if defined(_MSC_VER)
# ifdef SNMALLOC_VA_BITS_64
return _rotr64(x, (int)n);
# ifdef _WIN64
return _rotr64(static_cast<unsigned __int64>(x), static_cast<int>(n));
# else
return _rotr((uint32_t)x, (int)n);
return _rotr(static_cast<unsigned int>(x), static_cast<int>(n));
# endif
#else
return rotr_const(x, n);
@@ -114,10 +126,10 @@ namespace snmalloc
inline size_t rotl(size_t x, size_t n)
{
#if defined(_MSC_VER)
# ifdef SNMALLOC_VA_BITS_64
return _rotl64(x, (int)n);
# ifdef _WIN64
return _rotl64(static_cast<unsigned __int64>(x), static_cast<int>(n));
# else
return _rotl((uint32_t)x, (int)n);
return _rotl(static_cast<unsigned int>(x), static_cast<int>(n));
# endif
#else
return rotl_const(x, n);
@@ -145,11 +157,11 @@ namespace snmalloc
{
SNMALLOC_ASSERT(x != 0); // Calling with 0 is UB on some implementations
#if defined(_MSC_VER)
# ifdef SNMALLOC_VA_BITS_64
return _tzcnt_u64(x);
#if defined(_MSC_VER) && !defined(__clang__)
# ifdef _WIN64
return _tzcnt_u64(static_cast<unsigned __int64>(x));
# else
return _tzcnt_u32((uint32_t)x);
return _tzcnt_u32(static_cast<unsigned int>(x));
# endif
#else
if constexpr (std::is_same_v<unsigned long, std::size_t>)
@@ -191,14 +203,14 @@ namespace snmalloc
overflow = __builtin_mul_overflow(x, y, &prod);
return prod;
#elif defined(_MSC_VER)
# if defined(SNMALLOC_VA_BITS_64)
# ifdef _WIN64
size_t high_prod;
size_t prod = _umul128(x, y, &high_prod);
overflow = high_prod != 0;
return prod;
# else
size_t prod;
overflow = S_OK != UIntMult(x, y, &prod);
UINT prod;
overflow = S_OK != UIntMult(UINT(x), UINT(y), &prod);
return prod;
# endif
#else

0
src/ds/concept.h → src/snmalloc/ds_core/concept.h
View File

0
src/ds/defines.h → src/snmalloc/ds_core/defines.h
View File

16
src/snmalloc/ds_core/ds_core.h Normal file
View File

@@ -0,0 +1,16 @@
#pragma once
/**
* The core definitions for snmalloc. These provide some basic helpers that do
* not depend on anything except for a working C++ implementation.
*
* Files in this directory may not include anything from any other directory in
* snmalloc.
*/
#include "bits.h"
#include "concept.h"
#include "defines.h"
#include "helpers.h"
#include "ptrwrap.h"
#include "redblacktree.h"
#include "seqset.h"

41
src/ds/helpers.h → src/snmalloc/ds_core/helpers.h
View File

@@ -1,53 +1,15 @@
#pragma once
#include "bits.h"
#include "flaglock.h"
#include <array>
#include <atomic>
#include <functional>
#include <string_view>
#include <type_traits>
namespace snmalloc
{
/*
* In some use cases we need to run before any of the C++ runtime has been
* initialised. This singleton class is designed to not depend on the
* runtime.
*/
template<class Object, void init(Object*) noexcept>
class Singleton
{
inline static FlagWord flag;
inline static std::atomic<bool> initialised{false};
inline static Object obj;
public:
/**
* If argument is non-null, then it is assigned the value
* true, if this is the first call to get.
* At most one call will be first.
*/
inline SNMALLOC_SLOW_PATH static Object& get(bool* first = nullptr)
{
// If defined should be initially false;
SNMALLOC_ASSERT(first == nullptr || *first == false);
if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire)))
{
FlagLock lock(flag);
if (!initialised)
{
init(&obj);
initialised.store(true, std::memory_order_release);
if (first != nullptr)
*first = true;
}
}
return obj;
}
};
/**
* Wrapper for wrapping values.
*
@@ -477,4 +439,5 @@ namespace snmalloc
*/
struct Empty
{};
} // namespace snmalloc

4
src/ds/ptrwrap.h → src/snmalloc/ds_core/ptrwrap.h
View File

@@ -1,7 +1,7 @@
#pragma once
#include "../ds/concept.h"
#include "../ds/defines.h"
#include "concept.h"
#include "defines.h"
#include <atomic>

3
src/ds/redblacktree.h → src/snmalloc/ds_core/redblacktree.h
View File

@@ -1,7 +1,4 @@
#pragma once
#include "../pal/pal.h"
#include "concept.h"
#include "defines.h"
#include <array>
#include <cstddef>

4
src/ds/seqset.h → src/snmalloc/ds_core/seqset.h
View File

@@ -1,8 +1,6 @@
#pragma once
#include "address.h"
#include "defines.h"
#include "ptrwrap.h"
#include "../ds_core/ds_core.h"
#include <cstdint>
#include <type_traits>

4
src/mem/bounds_checks.h → src/snmalloc/global/bounds_checks.h
View File

@@ -1,5 +1,5 @@
#pragma once
#include "../snmalloc.h"
#include "threadalloc.h"
namespace snmalloc
{
@@ -107,4 +107,4 @@ namespace snmalloc
}
}
}
} // namespace snmalloc

4
src/snmalloc/global/global.h Normal file
View File

@@ -0,0 +1,4 @@
#include "bounds_checks.h"
#include "memcpy.h"
#include "scopedalloc.h"
#include "threadalloc.h"

5
src/mem/memcpy.h → src/snmalloc/global/memcpy.h
View File

@@ -1,5 +1,6 @@
#pragma once
#include "../mem/bounds_checks.h"
#include "../backend/globalconfig.h"
#include "bounds_checks.h"
namespace snmalloc
{
@@ -328,4 +329,4 @@ namespace snmalloc
Arch::copy(dst, src, len);
return orig_dst;
}
} // namespace
} // namespace snmalloc

1
src/mem/scopedalloc.h → src/snmalloc/global/scopedalloc.h
View File

@@ -1,4 +1,5 @@
#pragma once
#include "../backend/globalconfig.h"
/**
* This header requires that Alloc has been defined.

3
src/mem/threadalloc.h → src/snmalloc/global/threadalloc.h
View File

@@ -1,7 +1,6 @@
#pragma once
#include "../ds/helpers.h"
#include "localalloc.h"
#include "../backend/globalconfig.h"
#if defined(SNMALLOC_EXTERNAL_THREAD_ALLOC)
# define SNMALLOC_THREAD_TEARDOWN_DEFINED

4
src/backend/backend_concept.h → src/snmalloc/mem/backend_concept.h
View File

@@ -1,9 +1,7 @@
#pragma once
#ifdef __cpp_concepts
# include "../ds/address.h"
# include "../ds/concept.h"
# include "../pal/pal_concept.h"
# include "../ds/ds.h"
# include <cstddef>
namespace snmalloc

85
src/snmalloc/mem/backend_wrappers.h Normal file
View File

@@ -0,0 +1,85 @@
#pragma once
/**
* Several of the functions provided by the back end are optional. This file
* contains helpers that are templated on a back end and either call the
* corresponding function or do nothing. This allows the rest of the front end
* to assume that these functions always exist and avoid the need for `if
* constexpr` clauses everywhere. The no-op versions are always inlined and so
* will be optimised away.
*/
#include "../ds_core/ds_core.h"
namespace snmalloc
{
/**
* SFINAE helper. Matched only if `T` implements `is_initialised`. Calls
* it if it exists.
*/
template<typename T>
SNMALLOC_FAST_PATH auto call_is_initialised(T*, int)
-> decltype(T::is_initialised())
{
return T::is_initialised();
}
/**
* SFINAE helper. Matched only if `T` does not implement `is_initialised`.
* Unconditionally returns true if invoked.
*/
template<typename T>
SNMALLOC_FAST_PATH auto call_is_initialised(T*, long)
{
return true;
}
namespace detail
{
/**
* SFINAE helper, calls capptr_domesticate in the backend if it exists.
*/
template<
SNMALLOC_CONCEPT(ConceptBackendDomestication) Backend,
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) B>
SNMALLOC_FAST_PATH_INLINE auto
capptr_domesticate(typename Backend::LocalState* ls, CapPtr<T, B> p, int)
-> decltype(Backend::capptr_domesticate(ls, p))
{
return Backend::capptr_domesticate(ls, p);
}
/**
* SFINAE helper. If the back end does not provide special handling for
* domestication then assume all wild pointers can be domesticated.
*/
template<
SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend,
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) B>
SNMALLOC_FAST_PATH_INLINE auto
capptr_domesticate(typename Backend::LocalState*, CapPtr<T, B> p, long)
{
return CapPtr<
T,
typename B::template with_wildness<capptr::dimension::Wildness::Tame>>(
p.unsafe_ptr());
}
} // namespace detail
/**
* Wrapper that calls `Backend::capptr_domesticate` if and only if it is
* implemented. If it is not implemented then this assumes that any wild
* pointer can be domesticated.
*/
template<
SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend,
typename T,
SNMALLOC_CONCEPT(capptr::ConceptBound) B>
SNMALLOC_FAST_PATH_INLINE auto
capptr_domesticate(typename Backend::LocalState* ls, CapPtr<T, B> p)
{
return detail::capptr_domesticate<Backend>(ls, p, 0);
}
} // namespace snmalloc

3
src/mem/corealloc.h → src/snmalloc/mem/corealloc.h
View File

@@ -1,7 +1,6 @@
#pragma once
#include "../ds/defines.h"
#include "allocconfig.h"
#include "../ds/ds.h"
#include "localcache.h"
#include "metadata.h"
#include "pool.h"

1
src/mem/entropy.h → src/snmalloc/mem/entropy.h
View File

@@ -1,6 +1,5 @@
#pragma once
#include "../ds/address.h"
#include "../pal/pal.h"
#include <cstdint>

0
src/mem/external_alloc.h → src/snmalloc/mem/external_alloc.h
View File

3
src/mem/freelist.h → src/snmalloc/mem/freelist.h
View File

@@ -33,8 +33,7 @@
* and randomly deciding which list to add an element to.
*/
#include "../ds/address.h"
#include "allocconfig.h"
#include "../ds/ds.h"
#include "entropy.h"
#include <cstdint>

45
src/mem/globalalloc.h → src/snmalloc/mem/globalalloc.h
View File

@@ -1,53 +1,10 @@
#pragma once
#include "../ds/helpers.h"
#include "../ds_core/ds_core.h"
#include "localalloc.h"
namespace snmalloc
{
template<SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle>
inline static void aggregate_stats(Stats& stats)
{
static_assert(
SharedStateHandle::Options.CoreAllocIsPoolAllocated,
"Global statistics are available only for pool-allocated configurations");
auto* alloc = AllocPool<SharedStateHandle>::iterate();
while (alloc != nullptr)
{
auto a = alloc->attached_stats();
if (a != nullptr)
stats.add(*a);
stats.add(alloc->stats());
alloc = AllocPool<SharedStateHandle>::iterate(alloc);
}
}
#ifdef USE_SNMALLOC_STATS
template<SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle>
inline static void print_all_stats(std::ostream& o, uint64_t dumpid = 0)
{
static_assert(
SharedStateHandle::Options.CoreAllocIsPoolAllocated,
"Global statistics are available only for pool-allocated configurations");
auto alloc = AllocPool<SharedStateHandle>::iterate();
while (alloc != nullptr)
{
auto stats = alloc->stats();
if (stats != nullptr)
stats->template print<decltype(alloc)>(o, dumpid, alloc->id());
alloc = AllocPool<SharedStateHandle>::iterate(alloc);
}
}
#else
template<SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle>
inline static void print_all_stats(void*& o, uint64_t dumpid = 0)
{
UNUSED(o, dumpid);
}
#endif
template<SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle>
inline static void cleanup_unused()
{

5
src/mem/localalloc.h → src/snmalloc/mem/localalloc.h
View File

@@ -6,7 +6,7 @@
# define ALLOCATOR
#endif
#include "../ds/ptrwrap.h"
#include "../ds/ds.h"
#include "corealloc.h"
#include "freelist.h"
#include "localcache.h"
@@ -18,9 +18,6 @@
# include "external_alloc.h"
#endif
#ifdef SNMALLOC_TRACING
# include <iostream>
#endif
#include <string.h>
#include <utility>
namespace snmalloc

11
src/mem/localcache.h → src/snmalloc/mem/localcache.h
View File

@@ -1,7 +1,6 @@
#pragma once
#include "../ds/ptrwrap.h"
#include "allocstats.h"
#include "../ds/ds.h"
#include "freelist.h"
#include "remotecache.h"
#include "sizeclasstable.h"
@@ -10,8 +9,6 @@
namespace snmalloc
{
using Stats = AllocStats<NUM_SMALL_SIZECLASSES, NUM_LARGE_CLASSES>;
inline static SNMALLOC_FAST_PATH capptr::Alloc<void>
finish_alloc_no_zero(freelist::HeadPtr p, smallsizeclass_t sizeclass)
{
@@ -51,10 +48,6 @@ namespace snmalloc
// This is the entropy for a particular thread.
LocalEntropy entropy;
// TODO: Minimal stats object for just the stats on this datastructure.
// This will be a zero-size structure if stats are not enabled.
Stats stats;
// Pointer to the remote allocator message_queue, used to check
// if a deallocation is local.
RemoteAllocator* remote_allocator;
@@ -111,8 +104,6 @@ namespace snmalloc
{
auto& key = entropy.get_free_list_key();
smallsizeclass_t sizeclass = size_to_sizeclass(size);
stats.alloc_request(size);
stats.sizeclass_alloc(sizeclass);
auto& fl = small_fast_free_lists[sizeclass];
if (SNMALLOC_LIKELY(!fl.empty()))
{

16
src/snmalloc/mem/mem.h Normal file
View File

@@ -0,0 +1,16 @@
#include "backend_concept.h"
#include "backend_wrappers.h"
#include "corealloc.h"
#include "entropy.h"
#include "external_alloc.h"
#include "freelist.h"
#include "globalalloc.h"
#include "localalloc.h"
#include "localcache.h"
#include "metadata.h"
#include "pool.h"
#include "pooled.h"
#include "remoteallocator.h"
#include "remotecache.h"
#include "sizeclasstable.h"
#include "ticker.h"

4
src/mem/metadata.h → src/snmalloc/mem/metadata.h
View File

@@ -1,8 +1,6 @@
#pragma once
#include "../backend/backend_concept.h"
#include "../ds/helpers.h"
#include "../ds/seqset.h"
#include "../ds/ds.h"
#include "freelist.h"
#include "sizeclasstable.h"

4
src/mem/pool.h → src/snmalloc/mem/pool.h
View File

@@ -1,8 +1,6 @@
#pragma once
#include "../ds/flaglock.h"
#include "../ds/mpmcstack.h"
#include "../pal/pal_concept.h"
#include "../ds/ds.h"
#include "pooled.h"
#include <new>

3
src/mem/pooled.h → src/snmalloc/mem/pooled.h
View File

@@ -1,6 +1,7 @@
#pragma once
#include "../ds/bits.h"
#include "../ds/ds.h"
#include "backend_concept.h"
namespace snmalloc
{

8
src/mem/remoteallocator.h → src/snmalloc/mem/remoteallocator.h
View File

@@ -1,9 +1,9 @@
#pragma once
#include "../mem/allocconfig.h"
#include "../mem/freelist.h"
#include "../mem/metadata.h"
#include "../mem/sizeclasstable.h"
#include "../ds/ds.h"
#include "freelist.h"
#include "metadata.h"
#include "sizeclasstable.h"
#include <array>
#include <atomic>

3
src/mem/remotecache.h → src/snmalloc/mem/remotecache.h
View File

@@ -1,6 +1,7 @@
#pragma once
#include "allocconfig.h"
#include "../ds/ds.h"
#include "backend_wrappers.h"
#include "freelist.h"
#include "metadata.h"
#include "remoteallocator.h"

5
src/mem/sizeclasstable.h → src/snmalloc/mem/sizeclasstable.h
View File

@@ -1,9 +1,6 @@
#pragma once
#include "../ds/bits.h"
#include "../ds/defines.h"
#include "../ds/helpers.h"
#include "allocconfig.h"
#include "../ds/ds.h"
/**
* This file contains all the code for transforming transforming sizes to

2
src/mem/ticker.h → src/snmalloc/mem/ticker.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include "../ds/defines.h"
#include "../ds_core/ds_core.h"
#include <cstdint>

0
src/override/jemalloc_compat.cc → src/snmalloc/override/jemalloc_compat.cc
View File

0
src/override/malloc-extensions.cc → src/snmalloc/override/malloc-extensions.cc
View File

0
src/override/malloc-extensions.h → src/snmalloc/override/malloc-extensions.h
View File

0
src/override/malloc.cc → src/snmalloc/override/malloc.cc
View File

2
src/override/memcpy.cc → src/snmalloc/override/memcpy.cc
View File

@@ -1,5 +1,3 @@
#include "mem/memcpy.h"
#include "override.h"
extern "C"

0
src/override/new.cc → src/snmalloc/override/new.cc
View File

15
src/snmalloc/override/override.h Normal file
View File

@@ -0,0 +1,15 @@
#pragma once
#include "../global/global.h"
#ifndef SNMALLOC_EXPORT
# define SNMALLOC_EXPORT
#endif
#ifdef SNMALLOC_STATIC_LIBRARY_PREFIX
# define __SN_CONCAT(a, b) a##b
# define __SN_EVALUATE(a, b) __SN_CONCAT(a, b)
# define SNMALLOC_NAME_MANGLE(a) \
__SN_EVALUATE(SNMALLOC_STATIC_LIBRARY_PREFIX, a)
#elif !defined(SNMALLOC_NAME_MANGLE)
# define SNMALLOC_NAME_MANGLE(a) a
#endif

0
src/override/rust.cc → src/snmalloc/override/rust.cc
View File

16
src/pal/pal.h → src/snmalloc/pal/pal.h
View File

@@ -1,10 +1,22 @@
/**
* The platform abstraction layer. This defines an abstraction that exposes
* services from the operating system or any equivalent environment.
*
* It is possible to have multiple PALs in a single snmalloc instance. For
* example, one may provide the base operating system functionality, another
* may hide some of this or provide its own abstractions for sandboxing or
* isolating heaps.
*
* Files in this directory may depend on the architecture abstraction and core
* layers (`aal` and `ds_core`, respectively) but nothing else in snmalloc.
*/
#pragma once
#include "../ds/concept.h"
#include "../aal/aal.h"
#include "pal_concept.h"
#include "pal_consts.h"
// If simultating OE, then we need the underlying platform
// If simulating OE, then we need the underlying platform
#if defined(OPEN_ENCLAVE)
# include "pal_open_enclave.h"
#endif

0
src/pal/pal_apple.h → src/snmalloc/pal/pal_apple.h
View File

0
src/pal/pal_bsd.h → src/snmalloc/pal/pal_bsd.h
View File

0
src/pal/pal_bsd_aligned.h → src/snmalloc/pal/pal_bsd_aligned.h
View File

93
src/pal/pal_concept.h → src/snmalloc/pal/pal_concept.h
View File

@@ -1,14 +1,13 @@
#pragma once
#ifdef __cpp_concepts
# include "../ds/concept.h"
# include "pal_consts.h"
# include "pal_ds.h"
# include <utility>
namespace snmalloc
{
/*
* These concepts enforce that these are indeed constants that fit in the
* desired types. (This is subtly different from saying that they are the
@@ -41,7 +40,10 @@ namespace snmalloc
template<typename PAL>
concept ConceptPAL_error = requires(const char* const str)
{
{ PAL::error(str) } -> ConceptSame<void>;
{
PAL::error(str)
}
->ConceptSame<void>;
};
/**
@@ -50,25 +52,40 @@ namespace snmalloc
template<typename PAL>
concept ConceptPAL_memops = requires(void* vp, std::size_t sz)
{
{ PAL::notify_not_using(vp, sz) } noexcept -> ConceptSame<void>;
{
PAL::notify_not_using(vp, sz)
}
noexcept->ConceptSame<void>;
{ PAL::template notify_using<NoZero>(vp, sz) } noexcept
-> ConceptSame<void>;
{ PAL::template notify_using<YesZero>(vp, sz) } noexcept
-> ConceptSame<void>;
{
PAL::template notify_using<NoZero>(vp, sz)
}
noexcept->ConceptSame<void>;
{
PAL::template notify_using<YesZero>(vp, sz)
}
noexcept->ConceptSame<void>;
{ PAL::template zero<false>(vp, sz) } noexcept -> ConceptSame<void>;
{ PAL::template zero<true>(vp, sz) } noexcept -> ConceptSame<void>;
{
PAL::template zero<false>(vp, sz)
}
noexcept->ConceptSame<void>;
{
PAL::template zero<true>(vp, sz)
}
noexcept->ConceptSame<void>;
};
/**
* Absent any feature flags, the PAL must support a crude primitive allocator
*/
template<typename PAL>
concept ConceptPAL_reserve =
requires(PAL p, std::size_t sz)
concept ConceptPAL_reserve = requires(PAL p, std::size_t sz)
{
{ PAL::reserve(sz) } noexcept -> ConceptSame<void*>;
{
PAL::reserve(sz)
}
noexcept->ConceptSame<void*>;
};
/**
@@ -77,9 +94,14 @@ namespace snmalloc
template<typename PAL>
concept ConceptPAL_reserve_aligned = requires(std::size_t sz)
{
{ PAL::template reserve_aligned<true>(sz) } noexcept -> ConceptSame<void*>;
{ PAL::template reserve_aligned<false>(sz) } noexcept
-> ConceptSame<void*>;
{
PAL::template reserve_aligned<true>(sz)
}
noexcept->ConceptSame<void*>;
{
PAL::template reserve_aligned<false>(sz)
}
noexcept->ConceptSame<void*>;
};
/**
@@ -88,14 +110,23 @@ namespace snmalloc
template<typename PAL>
concept ConceptPAL_mem_low_notify = requires(PalNotificationObject* pno)
{
{ PAL::expensive_low_memory_check() } -> ConceptSame<bool>;
{ PAL::register_for_low_memory_callback(pno) } -> ConceptSame<void>;
{
PAL::expensive_low_memory_check()
}
->ConceptSame<bool>;
{
PAL::register_for_low_memory_callback(pno)
}
->ConceptSame<void>;
};
template<typename PAL>
concept ConceptPAL_get_entropy64 = requires()
{
{ PAL::get_entropy64() } -> ConceptSame<uint64_t>;
{
PAL::get_entropy64()
}
->ConceptSame<uint64_t>;
};
/**
@@ -105,21 +136,17 @@ namespace snmalloc
* are, naturally, not bound by the corresponding concept.
*/
template<typename PAL>
concept ConceptPAL =
ConceptPAL_static_features<PAL> &&
ConceptPAL_static_sizes<PAL> &&
ConceptPAL_error<PAL> &&
ConceptPAL_memops<PAL> &&
concept ConceptPAL = ConceptPAL_static_features<PAL>&&
ConceptPAL_static_sizes<PAL>&& ConceptPAL_error<PAL>&&
ConceptPAL_memops<PAL> &&
(!pal_supports<Entropy, PAL> ||
ConceptPAL_get_entropy64<PAL>) &&
(!pal_supports<LowMemoryNotification, PAL> ||
ConceptPAL_mem_low_notify<PAL>) &&
(pal_supports<NoAllocation, PAL> ||
(
(!pal_supports<AlignedAllocation, PAL> ||
ConceptPAL_reserve_aligned<PAL>) &&
ConceptPAL_reserve<PAL>)
);
ConceptPAL_get_entropy64<
PAL>)&&(!pal_supports<LowMemoryNotification, PAL> ||
ConceptPAL_mem_low_notify<
PAL>)&&(pal_supports<NoAllocation, PAL> ||
((!pal_supports<AlignedAllocation, PAL> ||
ConceptPAL_reserve_aligned<
PAL>)&&ConceptPAL_reserve<PAL>));
} // namespace snmalloc
#endif

2
src/pal/pal_consts.h → src/snmalloc/pal/pal_consts.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include "../ds/defines.h"
#include "../ds_core/ds_core.h"
#include <atomic>
#include <functional>

0
src/pal/pal_dragonfly.h → src/snmalloc/pal/pal_dragonfly.h
View File

3
src/pal/pal_ds.h → src/snmalloc/pal/pal_ds.h
View File

@@ -1,7 +1,6 @@
#pragma once
#include "../ds/defines.h"
#include "../ds/helpers.h"
#include "../ds_core/ds_core.h"
#include <atomic>
#include <functional>

0
src/pal/pal_freebsd.h → src/snmalloc/pal/pal_freebsd.h
View File

2
src/pal/pal_freebsd_kernel.h → src/snmalloc/pal/pal_freebsd_kernel.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include "../ds/bits.h"
#include "../ds_core/ds_core.h"
#if defined(FreeBSD_KERNEL)
extern "C"

0
src/pal/pal_haiku.h → src/snmalloc/pal/pal_haiku.h
View File

2
src/pal/pal_linux.h → src/snmalloc/pal/pal_linux.h
View File

@@ -1,7 +1,7 @@
#pragma once
#if defined(__linux__)
# include "../ds/bits.h"
# include "../ds_core/ds_core.h"
# include "pal_posix.h"
# include <string.h>

0
src/pal/pal_netbsd.h → src/snmalloc/pal/pal_netbsd.h
View File

0
src/pal/pal_noalloc.h → src/snmalloc/pal/pal_noalloc.h
View File

0
src/pal/pal_open_enclave.h → src/snmalloc/pal/pal_open_enclave.h
View File

0
src/pal/pal_openbsd.h → src/snmalloc/pal/pal_openbsd.h
View File

2
src/pal/pal_plain.h → src/snmalloc/pal/pal_plain.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include "../ds/bits.h"
#include "../ds_core/ds_core.h"
#include "pal_timer_default.h"
namespace snmalloc

5
src/pal/pal_posix.h → src/snmalloc/pal/pal_posix.h
View File

@@ -1,9 +1,6 @@
#pragma once
#ifdef SNMALLOC_TRACING
# include <iostream>
#endif
#include "../ds/address.h"
#include "../aal/aal.h"
#include "pal_timer_default.h"
#if defined(SNMALLOC_BACKTRACE_HEADER)
# include SNMALLOC_BACKTRACE_HEADER

0
src/pal/pal_solaris.h → src/snmalloc/pal/pal_solaris.h
View File

0
src/pal/pal_timer_default.h → src/snmalloc/pal/pal_timer_default.h
View File

3
src/pal/pal_windows.h → src/snmalloc/pal/pal_windows.h
View File

@@ -1,7 +1,6 @@
#pragma once
#include "../ds/address.h"
#include "../ds/bits.h"
#include "../aal/aal.h"
#include "pal_timer_default.h"
#ifdef _WIN32

10
src/snmalloc/snmalloc.h Normal file
View File

@@ -0,0 +1,10 @@
#pragma once
// Core implementation of snmalloc independent of the configuration mode
#include "snmalloc_core.h"
// If the user has defined SNMALLOC_PROVIDE_OWN_CONFIG, this include does
// nothing. Otherwise, it provide a default configuration of snmalloc::Alloc.
#include "backend/globalconfig.h"
// User facing API surface, needs to know what `Alloc` is.
#include "snmalloc_front.h"

3
src/snmalloc/snmalloc_core.h Normal file
View File

@@ -0,0 +1,3 @@
#pragma once
#include "backend_helpers/backend_helpers.h"

Some files were not shown because too many files have changed in this diff Show More