Cheri-research/snmalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #103 from CTSRD-CHERI/for-upstream-cheri-prep-tidy

Browse Source 
CHERI prep work that's just tidying
This commit is contained in:
Matthew Parkinson
2019-11-26 16:42:03 +00:00
committed by GitHub
parent 11923154e8 997ebc5065
commit f70191fb1e
26 changed files with 262 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
CMakeLists.txt
View File

@@ -23,7 +23,7 @@ macro(warnings_high)
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
add_compile_options(-Wsign-conversion)
endif ()
add_compile_options(-Wall -Wextra -Werror)
add_compile_options(-Wall -Wextra -Werror -Wundef)
endif()
endmacro()
@@ -33,6 +33,7 @@ macro(clangformat_targets)
# don't know whether our clang-format file will work with newer versions of the
# tool
set(CLANG_FORMAT_NAMES
clang-format-8
clang-format-7.0
clang-format-6.0
clang-format70
@@ -71,7 +72,12 @@ if(NOT MSVC)
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
target_link_libraries(snmalloc_lib INTERFACE atomic)
endif()
target_compile_options(snmalloc_lib INTERFACE -mcx16)
if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64")
target_compile_options(snmalloc_lib INTERFACE -mcx16)
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
target_compile_options(snmalloc_lib INTERFACE -mcx16)
# XXX elseif ARM?
endif()
else()
set(WIN8COMPAT FALSE CACHE BOOL "Avoid Windows 10 APIs")
if (WIN8COMPAT)
@@ -114,7 +120,13 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY)
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi")
set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG")
else()
add_compile_options(-march=native -fno-exceptions -fno-rtti -g -ftls-model=initial-exec -fomit-frame-pointer)
add_compile_options(-fno-exceptions -fno-rtti -g -ftls-model=initial-exec -fomit-frame-pointer)
if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64")
add_compile_options(-march=native)
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
add_compile_options(-march=native)
# XXX elseif ARM?
endif()
endif()
macro(subdirlist result curdir)
@@ -141,6 +153,10 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY)
if(EXPOSE_EXTERNAL_RESERVE)
target_compile_definitions(${name} PRIVATE -DSNMALLOC_EXPOSE_RESERVE)
endif()
# Ensure that we do not link against C++ stdlib when compiling shims.
set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C)
endmacro()
if(NOT MSVC)

17
src/ds/aba.h
View File

@@ -41,7 +41,7 @@ namespace snmalloc
Independent independent;
};
#else
std::atomic<T*> ptr;
std::atomic<T*> raw;
#endif
public:
@@ -57,13 +57,18 @@ namespace snmalloc
independent.ptr.store(x, std::memory_order_relaxed);
independent.aba.store(0, std::memory_order_relaxed);
#else
ptr.store(x, std::memory_order_relaxed);
raw.store(x, std::memory_order_relaxed);
#endif
}
T* peek()
{
return independent.ptr.load(std::memory_order_relaxed);
return
#ifdef PLATFORM_IS_X86
independent.ptr.load(std::memory_order_relaxed);
#else
raw.load(std::memory_order_relaxed);
#endif
}
Cmp read()
@@ -73,11 +78,11 @@ namespace snmalloc
Cmp{independent.ptr.load(std::memory_order_relaxed),
independent.aba.load(std::memory_order_relaxed)};
#else
ptr.load(std::memory_order_relaxed);
raw.load(std::memory_order_relaxed);
#endif
}
static T* load(Cmp& from)
static T* ptr(Cmp& from)
{
#ifdef PLATFORM_IS_X86
return from.ptr;
@@ -105,7 +110,7 @@ namespace snmalloc
expect, xchg, std::memory_order_relaxed, std::memory_order_relaxed);
# endif
#else
return ptr.compare_exchange_weak(
return raw.compare_exchange_weak(
expect, value, std::memory_order_relaxed, std::memory_order_relaxed);
#endif
}

76
src/ds/address.h
View File

@@ -1,4 +1,7 @@
#pragma once
#include "bits.h"
#include <cassert>
#include <cstdint>
namespace snmalloc
@@ -39,4 +42,77 @@ namespace snmalloc
{
return reinterpret_cast<T*>(address);
}
/**
* Test if a pointer is aligned to a given size, which must be a power of
* two.
*/
template<size_t alignment>
static inline bool is_aligned_block(void* p, size_t size)
{
static_assert(bits::next_pow2_const(alignment) == alignment);
return ((static_cast<size_t>(address_cast(p)) | size) & (alignment - 1)) ==
0;
}
/**
* Align a pointer down to a statically specified granularity, which must be a
* power of two.
*/
template<size_t alignment, typename T = void>
inline T* pointer_align_down(void* p)
{
static_assert(alignment > 0);
static_assert(bits::next_pow2_const(alignment) == alignment);
#if __has_builtin(__builtin_align_down)
return static_cast<T*>(__builtin_align_down(p, alignment));
#else
return pointer_cast<T>(bits::align_down(address_cast(p), alignment));
#endif
}
/**
* Align a pointer up to a statically specified granularity, which must be a
* power of two.
*/
template<size_t alignment, typename T = void>
inline T* pointer_align_up(void* p)
{
static_assert(alignment > 0);
static_assert(bits::next_pow2_const(alignment) == alignment);
#if __has_builtin(__builtin_align_up)
return static_cast<T*>(__builtin_align_up(p, alignment));
#else
return pointer_cast<T>(bits::align_up(address_cast(p), alignment));
#endif
}
/**
* Align a pointer up to a dynamically specified granularity, which must
* be a power of two.
*/
template<typename T = void>
inline T* pointer_align_up(void* p, size_t alignment)
{
assert(alignment > 0);
assert(bits::next_pow2(alignment) == alignment);
#if __has_builtin(__builtin_align_up)
return static_cast<T*>(__builtin_align_up(p, alignment));
#else
return pointer_cast<T>(bits::align_up(address_cast(p), alignment));
#endif
}
/**
* Compute the difference in pointers in units of char. base is
* expected to point to the base of some (sub)allocation into which cursor
* points; would-be negative answers trip an assertion in debug builds.
*/
inline size_t pointer_diff(void* base, void* cursor)
{
assert(cursor >= base);
return static_cast<size_t>(
static_cast<char*>(cursor) - static_cast<char*>(base));
}
} // namespace snmalloc

10
src/ds/bits.h
View File

@@ -6,7 +6,6 @@
// #define USE_LZCNT
#include "../aal/aal.h"
#include "address.h"
#include "defines.h"
#include <atomic>
@@ -240,15 +239,6 @@ namespace snmalloc
return value;
}
template<size_t alignment>
static inline bool is_aligned_block(void* p, size_t size)
{
assert(next_pow2(alignment) == alignment);
return ((static_cast<size_t>(address_cast(p)) | size) &
(alignment - 1)) == 0;
}
/************************************************
*
* Map large range of strictly positive integers

3
src/ds/helpers.h
View File

@@ -7,7 +7,8 @@ namespace snmalloc
{
/*
* In some use cases we need to run before any of the C++ runtime has been
* initialised. This singleton class is design to not depend on the runtime.
* initialised. This singleton class is designed to not depend on the
* runtime.
*/
template<class Object, Object init() noexcept>
class Singleton

6
src/ds/mpmcstack.h
View File

@@ -29,7 +29,7 @@ namespace snmalloc
do
{
T* top = ABAT::load(cmp);
T* top = ABAT::ptr(cmp);
last->next.store(top, std::memory_order_release);
} while (!stack.compare_exchange(cmp, first));
}
@@ -44,7 +44,7 @@ namespace snmalloc
do
{
top = ABAT::load(cmp);
top = ABAT::ptr(cmp);
if (top == nullptr)
break;
@@ -63,7 +63,7 @@ namespace snmalloc
do
{
top = ABAT::load(cmp);
top = ABAT::ptr(cmp);
if (top == nullptr)
break;

72
src/mem/alloc.h
View File

@@ -98,6 +98,9 @@ namespace snmalloc
template<class MP>
friend class AllocPool;
/**
* Allocate memory of a statically known size.
*/
template<
size_t size,
ZeroMem zero_mem = NoZero,
@@ -118,7 +121,6 @@ namespace snmalloc
stats().alloc_request(size);
// Allocate memory of a statically known size.
if constexpr (sizeclass < NUM_SMALL_CLASSES)
{
return small_alloc<zero_mem, allow_reserve>(size);
@@ -137,6 +139,9 @@ namespace snmalloc
#endif
}
/**
* Allocate memory of a dynamically known size.
*/
template<ZeroMem zero_mem = NoZero, AllowReserve allow_reserve = YesReserve>
SNMALLOC_FAST_PATH ALLOCATOR void* alloc(size_t size)
{
@@ -151,7 +156,6 @@ namespace snmalloc
#else
stats().alloc_request(size);
// Allocate memory of a dynamically known size.
// Perform the - 1 on size, so that zero wraps around and ends up on
// slow path.
if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1)))
@@ -186,6 +190,10 @@ namespace snmalloc
#endif
}
/*
* Free memory of a statically known size. Must be called with an
* external pointer.
*/
template<size_t size>
void dealloc(void* p)
{
@@ -198,8 +206,6 @@ namespace snmalloc
handle_message_queue();
// Free memory of a statically known size. Must be called with an
// external pointer.
if (sizeclass < NUM_SMALL_CLASSES)
{
Superslab* super = Superslab::get(p);
@@ -227,6 +233,10 @@ namespace snmalloc
#endif
}
/*
* Free memory of a dynamically known size. Must be called with an
* external pointer.
*/
void dealloc(void* p, size_t size)
{
#ifdef USE_MALLOC
@@ -235,8 +245,6 @@ namespace snmalloc
#else
handle_message_queue();
// Free memory of a dynamically known size. Must be called with an
// external pointer.
sizeclass_t sizeclass = size_to_sizeclass(size);
if (sizeclass < NUM_SMALL_CLASSES)
@@ -266,14 +274,16 @@ namespace snmalloc
#endif
}
/*
* Free memory of an unknown size. Must be called with an external
* pointer.
*/
SNMALLOC_FAST_PATH void dealloc(void* p)
{
#ifdef USE_MALLOC
return free(p);
#else
// Free memory of an unknown size. Must be called with an external
// pointer.
uint8_t size = chunkmap().get(address_cast(p));
Superslab* super = Superslab::get(p);
@@ -311,7 +321,7 @@ namespace snmalloc
RemoteAllocator* target = slab->get_allocator();
// Reading a remote sizeclass won't fail, since the other allocator
// can't reuse the slab, as we have no yet deallocated this pointer.
// can't reuse the slab, as we have not yet deallocated this pointer.
sizeclass_t sizeclass = slab->get_sizeclass();
if (target == public_state())
@@ -353,7 +363,7 @@ namespace snmalloc
Metaslab& meta = super->get_meta(slab);
sizeclass_t sc = meta.sizeclass;
size_t slab_end = static_cast<size_t>(address_cast(slab) + SLAB_SIZE);
void* slab_end = pointer_offset(slab, SLAB_SIZE);
return external_pointer<location>(p, sc, slab_end);
}
@@ -362,8 +372,7 @@ namespace snmalloc
Mediumslab* slab = Mediumslab::get(p);
sizeclass_t sc = slab->get_sizeclass();
size_t slab_end =
static_cast<size_t>(address_cast(slab) + SUPERSLAB_SIZE);
void* slab_end = pointer_offset(slab, SUPERSLAB_SIZE);
return external_pointer<location>(p, sc, slab_end);
}
@@ -442,9 +451,20 @@ namespace snmalloc
private:
using alloc_id_t = typename Remote::alloc_id_t;
/*
* A singly-linked list of Remote objects, supporting append and
* take-all operations. Intended only for the private use of this
* allocator; the Remote objects here will later be taken and pushed
* to the inter-thread message queues.
*/
struct RemoteList
{
/*
* A stub Remote object that will always be the head of this list;
* never taken for further processing.
*/
Remote head;
Remote* last;
RemoteList()
@@ -661,7 +681,7 @@ namespace snmalloc
// All medium size classes are page aligned.
if (i > NUM_SMALL_CLASSES)
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(nullptr, size1));
assert(is_aligned_block<OS_PAGE_SIZE>(nullptr, size1));
}
assert(sc1 == i);
@@ -734,16 +754,23 @@ namespace snmalloc
template<Boundary location>
static uintptr_t
external_pointer(void* p, sizeclass_t sizeclass, size_t end_point)
external_pointer(void* p, sizeclass_t sizeclass, void* end_point)
{
size_t rsize = sizeclass_to_size(sizeclass);
size_t end_point_correction = location == End ?
(end_point - 1) :
(location == OnePastEnd ? end_point : (end_point - rsize));
size_t offset_from_end =
(end_point - 1) - static_cast<size_t>(address_cast(p));
size_t end_to_end = round_by_sizeclass(rsize, offset_from_end);
return end_point_correction - end_to_end;
void* end_point_correction = location == End ?
(static_cast<uint8_t*>(end_point) - 1) :
(location == OnePastEnd ? end_point :
(static_cast<uint8_t*>(end_point) - rsize));
ptrdiff_t offset_from_end =
(static_cast<uint8_t*>(end_point) - 1) - static_cast<uint8_t*>(p);
size_t end_to_end =
round_by_sizeclass(rsize, static_cast<size_t>(offset_from_end));
return address_cast<uint8_t>(
static_cast<uint8_t*>(end_point_correction) - end_to_end);
}
void init_message_queue()
@@ -1166,7 +1193,7 @@ namespace snmalloc
#ifdef CHECK_CLIENT
if (!is_multiple_of_sizeclass(
sizeclass_to_size(sizeclass),
address_cast(slab) + SUPERSLAB_SIZE - address_cast(p)))
pointer_diff(p, pointer_offset(slab, SUPERSLAB_SIZE))))
{
error("Not deallocating start of an object");
}
@@ -1242,6 +1269,7 @@ namespace snmalloc
stats().large_dealloc(large_class);
// Cross-reference largealloc's alloc() decommitted condition.
if ((decommit_strategy != DecommitNone) || (large_class > 0))
large_allocator.memory_provider.notify_not_using(
pointer_offset(p, OS_PAGE_SIZE), rsize - OS_PAGE_SIZE);

2
src/mem/allocconfig.h
View File

@@ -113,7 +113,7 @@ namespace snmalloc
static constexpr size_t MIN_ALLOC_BITS = bits::is64() ? 4 : 3;
static constexpr size_t MIN_ALLOC_SIZE = 1 << MIN_ALLOC_BITS;
// Slabs are 64 kb.
// Slabs are 64 KiB unless constrained to 16 KiB.
static constexpr size_t SLAB_BITS = ADDRESS_SPACE_CONSTRAINED ? 14 : 16;
static constexpr size_t SLAB_SIZE = 1 << SLAB_BITS;
static constexpr size_t SLAB_MASK = ~(SLAB_SIZE - 1);

2
src/mem/chunkmap.h
View File

@@ -211,8 +211,6 @@ namespace snmalloc
ss, static_cast<uint8_t>(64 + i + SUPERSLAB_BITS), run);
ss = ss + SUPERSLAB_SIZE * run;
}
PagemapProvider::pagemap().set(
address_cast(p), static_cast<uint8_t>(size_bits));
}
/**
* Update the pagemap to remove a large allocation, of `size` bytes from

82
src/mem/largealloc.h
View File

@@ -58,7 +58,7 @@ namespace snmalloc
class MemoryProviderStateMixin : public PAL
{
std::atomic_flag lock = ATOMIC_FLAG_INIT;
address_t bump;
void* bump;
size_t remaining;
void new_block()
@@ -71,7 +71,7 @@ namespace snmalloc
PAL::template notify_using<NoZero>(r, OS_PAGE_SIZE);
bump = address_cast(r);
bump = r;
remaining = size;
}
@@ -145,15 +145,21 @@ namespace snmalloc
{
FlagLock f(lock);
auto aligned_bump = bits::align_up(bump, alignment);
if ((aligned_bump - bump) > remaining)
if constexpr (alignment != 0)
{
new_block();
}
else
{
remaining -= aligned_bump - bump;
bump = aligned_bump;
char* aligned_bump = pointer_align_up<alignment, char>(bump);
size_t bump_delta = pointer_diff(bump, aligned_bump);
if (bump_delta > remaining)
{
new_block();
}
else
{
remaining -= bump_delta;
bump = aligned_bump;
}
}
if (remaining < size)
@@ -161,16 +167,17 @@ namespace snmalloc
new_block();
}
p = pointer_cast<void>(bump);
bump += size;
p = bump;
bump = pointer_offset(bump, size);
remaining -= size;
}
auto page_start = bits::align_down(address_cast(p), OS_PAGE_SIZE);
auto page_end = bits::align_up(address_cast(p) + size, OS_PAGE_SIZE);
auto page_start = pointer_align_down<OS_PAGE_SIZE, char>(p);
auto page_end =
pointer_align_up<OS_PAGE_SIZE, char>(pointer_offset(p, size));
PAL::template notify_using<NoZero>(
pointer_cast<void>(page_start), page_end - page_start);
page_start, static_cast<size_t>(page_end - page_start));
return new (p) T(std::forward<Args...>(args)...);
}
@@ -287,7 +294,14 @@ namespace snmalloc
template<AllowReserve allow_reserve>
bool reserve_memory(size_t need, size_t add)
{
if ((address_cast(reserved_start) + need) > address_cast(reserved_end))
assert(reserved_start <= reserved_end);
/*
* Spell this comparison in terms of pointer subtraction like this,
* rather than "reserved_start + need < reserved_end" because the
* sum might not be representable on CHERI.
*/
if (pointer_diff(reserved_start, reserved_end) < need)
{
if constexpr (allow_reserve == YesReserve)
{
@@ -295,8 +309,7 @@ namespace snmalloc
reserved_start =
memory_provider.template reserve<false>(&add, SUPERSLAB_SIZE);
reserved_end = pointer_offset(reserved_start, add);
reserved_start = pointer_cast<void>(
bits::align_up(address_cast(reserved_start), SUPERSLAB_SIZE));
reserved_start = pointer_align_up<SUPERSLAB_SIZE>(reserved_start);
if (add < need)
return false;
@@ -344,30 +357,14 @@ namespace snmalloc
{
stats.superslab_pop();
if constexpr (decommit_strategy == DecommitSuperLazy)
{
if (static_cast<Baseslab*>(p)->get_kind() == Decommitted)
{
// The first page is already in "use" for the stack element,
// this will need zeroing for a YesZero call.
if constexpr (zero_mem == YesZero)
memory_provider.template zero<true>(p, OS_PAGE_SIZE);
// Cross-reference alloc.h's large_dealloc decommitment condition
// and lazy_decommit_if_needed.
bool decommitted =
((decommit_strategy == DecommitSuperLazy) &&
(static_cast<Baseslab*>(p)->get_kind() == Decommitted)) ||
(large_class > 0) || (decommit_strategy != DecommitNone);
// Notify we are using the rest of the allocation.
// Passing zero_mem ensures the PAL provides zeroed pages if
// required.
memory_provider.template notify_using<zero_mem>(
pointer_offset(p, OS_PAGE_SIZE),
bits::align_up(size, OS_PAGE_SIZE) - OS_PAGE_SIZE);
}
else
{
if constexpr (zero_mem == YesZero)
memory_provider.template zero<true>(
p, bits::align_up(size, OS_PAGE_SIZE));
}
}
if ((decommit_strategy != DecommitNone) || (large_class > 0))
if (decommitted)
{
// The first page is already in "use" for the stack element,
// this will need zeroing for a YesZero call.
@@ -375,7 +372,8 @@ namespace snmalloc
memory_provider.template zero<true>(p, OS_PAGE_SIZE);
// Notify we are using the rest of the allocation.
// Passing zero_mem ensures the PAL provides zeroed pages if required.
// Passing zero_mem ensures the PAL provides zeroed pages if
// required.
memory_provider.template notify_using<zero_mem>(
pointer_offset(p, OS_PAGE_SIZE),
bits::align_up(size, OS_PAGE_SIZE) - OS_PAGE_SIZE);

7
src/mem/mediumslab.h
View File

@@ -41,7 +41,7 @@ namespace snmalloc
static Mediumslab* get(void* p)
{
return pointer_cast<Mediumslab>(address_cast(p) & SUPERSLAB_MASK);
return pointer_align_down<SUPERSLAB_SIZE, Mediumslab>(p);
}
void init(RemoteAllocator* alloc, sizeclass_t sc, size_t rsize)
@@ -84,7 +84,7 @@ namespace snmalloc
void* p = pointer_offset(this, (static_cast<size_t>(index) << 8));
free--;
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, OS_PAGE_SIZE));
assert(is_aligned_block<OS_PAGE_SIZE>(p, OS_PAGE_SIZE));
size = bits::align_up(size, OS_PAGE_SIZE);
if constexpr (decommit_strategy == DecommitAll)
@@ -125,8 +125,7 @@ namespace snmalloc
uint16_t pointer_to_index(void* p)
{
// Get the offset from the slab for a memory location.
return static_cast<uint16_t>(
((address_cast(p) - address_cast(this))) >> 8);
return static_cast<uint16_t>(pointer_diff(this, p) >> 8);
}
};
} // namespace snmalloc

5
src/mem/metaslab.h
View File

@@ -15,7 +15,7 @@ namespace snmalloc
Slab* get_slab()
{
return pointer_cast<Slab>(address_cast(this) & SLAB_MASK);
return pointer_align_down<SLAB_SIZE, Slab>(this);
}
};
@@ -224,8 +224,7 @@ namespace snmalloc
{
// Check we are looking at a correctly aligned block
void* start = remove_cache_friendly_offset(curr, sizeclass);
assert(
((address_cast(start) - address_cast(slab) - offset) % size) == 0);
assert(((pointer_diff(slab, start) - offset) % size) == 0);
// Account for free elements in free list
accounted_for += size;

4
src/mem/pagemap.h
View File

@@ -273,9 +273,7 @@ namespace snmalloc
void* page_for_address(uintptr_t p)
{
bool success;
return reinterpret_cast<void*>(
~(OS_PAGE_SIZE - 1) &
reinterpret_cast<uintptr_t>(get_addr<true>(p, success)));
return pointer_align_down<OS_PAGE_SIZE>(get_addr<true>(p, success));
}
T get(uintptr_t p)

5
src/mem/remoteallocator.h
View File

@@ -8,6 +8,11 @@
namespace snmalloc
{
/*
* A region of memory destined for a remote allocator's dealloc() via the
* message passing system. This structure is placed at the beginning of
* the allocation itself when it is queued for sending.
*/
struct Remote
{
using alloc_id_t = size_t;

7
src/mem/slab.h
View File

@@ -16,7 +16,7 @@ namespace snmalloc
uint16_t pointer_to_index(void* p)
{
// Get the offset from the slab for a memory location.
return static_cast<uint16_t>(address_cast(p) - address_cast(this));
return static_cast<uint16_t>(pointer_diff(this, p));
}
public:
@@ -43,8 +43,7 @@ namespace snmalloc
void* head = meta.head;
assert(rsize == sizeclass_to_size(meta.sizeclass));
meta.debug_slab_invariant(this);
assert(sl.get_head() == (SlabLink*)((size_t)this + meta.link));
assert(sl.get_head() == (SlabLink*)pointer_offset(this, meta.link));
assert(!meta.is_full());
void* p = nullptr;
@@ -145,7 +144,7 @@ namespace snmalloc
Metaslab& meta = super->get_meta(this);
return is_multiple_of_sizeclass(
sizeclass_to_size(meta.sizeclass),
address_cast(this) + SLAB_SIZE - address_cast(p));
pointer_diff(p, pointer_offset(this, SLAB_SIZE)));
}
// Returns true, if it deallocation can proceed without changing any status

21
src/mem/superslab.h
View File

@@ -44,9 +44,9 @@ namespace snmalloc
// Used size_t as results in better code in MSVC
size_t slab_to_index(Slab* slab)
{
auto res = ((address_cast(slab) - address_cast(this)) >> SLAB_BITS);
assert(res == (uint8_t)res);
return res;
auto res = (pointer_diff(this, slab) >> SLAB_BITS);
assert(res == static_cast<uint8_t>(res));
return static_cast<uint8_t>(res);
}
public:
@@ -67,7 +67,7 @@ namespace snmalloc
static Superslab* get(void* p)
{
return pointer_cast<Superslab>(address_cast(p) & SUPERSLAB_MASK);
return pointer_align_down<SUPERSLAB_SIZE, Superslab>(p);
}
static bool is_short_sizeclass(sizeclass_t sizeclass)
@@ -82,12 +82,6 @@ namespace snmalloc
if (kind != Super)
{
// If this wasn't previously a Superslab, we need to set up the
// header.
kind = Super;
// Point head at the first non-short slab.
head = 1;
if (kind != Fresh)
{
// If this wasn't previously Fresh, we need to zero some things.
@@ -97,6 +91,13 @@ namespace snmalloc
new (&(meta[i])) Metaslab();
}
}
// If this wasn't previously a Superslab, we need to set up the
// header.
kind = Super;
// Point head at the first non-short slab.
head = 1;
#ifndef NDEBUG
auto curr = head;
for (size_t i = 0; i < SLAB_COUNT - used - 1; i++)

2
src/mem/threadalloc.h
View File

@@ -188,7 +188,7 @@ namespace snmalloc
#ifdef SNMALLOC_USE_THREAD_CLEANUP
/**
* Entry point the allows libc to call into the allocator for per-thread
* Entry point that allows libc to call into the allocator for per-thread
* cleanup.
*/
extern "C" void _malloc_thread_cleanup()

4
src/pal/pal_apple.h
View File

@@ -32,9 +32,9 @@ namespace snmalloc
template<bool page_aligned = false>
void zero(void* p, size_t size)
{
if (page_aligned || bits::is_aligned_block<OS_PAGE_SIZE>(p, size))
if (page_aligned || is_aligned_block<OS_PAGE_SIZE>(p, size))
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
assert(is_aligned_block<OS_PAGE_SIZE>(p, size));
void* r = mmap(
p,
size,

2
src/pal/pal_bsd.h
View File

@@ -33,7 +33,7 @@ namespace snmalloc
*/
void notify_not_using(void* p, size_t size) noexcept
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
assert(is_aligned_block<OS_PAGE_SIZE>(p, size));
madvise(p, size, MADV_FREE);
}
};

4
src/pal/pal_linux.h
View File

@@ -37,9 +37,9 @@ namespace snmalloc
template<bool page_aligned = false>
void zero(void* p, size_t size) noexcept
{
if (page_aligned || bits::is_aligned_block<OS_PAGE_SIZE>(p, size))
if (page_aligned || is_aligned_block<OS_PAGE_SIZE>(p, size))
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
assert(is_aligned_block<OS_PAGE_SIZE>(p, size));
madvise(p, size, MADV_DONTNEED);
}
else

23
src/pal/pal_open_enclave.h
View File

@@ -11,7 +11,7 @@ namespace snmalloc
{
class PALOpenEnclave
{
std::atomic<uintptr_t> oe_base;
std::atomic<void*> oe_base;
public:
/**
@@ -30,27 +30,28 @@ namespace snmalloc
{
if (oe_base == 0)
{
uintptr_t dummy = 0;
oe_base.compare_exchange_strong(dummy, (uintptr_t)__oe_get_heap_base());
void* dummy = NULL;
oe_base.compare_exchange_strong(
dummy, const_cast<void*>(__oe_get_heap_base()));
}
uintptr_t old_base = oe_base;
uintptr_t old_base2 = old_base;
uintptr_t next_base;
auto end = (uintptr_t)__oe_get_heap_end();
void* old_base = oe_base;
void* old_base2 = old_base;
void* next_base;
auto end = __oe_get_heap_end();
do
{
old_base2 = old_base;
auto new_base = bits::align_up(old_base, align);
next_base = new_base + *size;
auto new_base = pointer_align_up(old_base, align);
next_base = pointer_offset(new_base, *size);
if (next_base > end)
error("Out of memory");
} while (oe_base.compare_exchange_strong(old_base, next_base));
*size = next_base - old_base2;
return (void*)old_base;
*size = pointer_diff(old_base2, next_base);
return old_base;
}
template<bool page_aligned = false>

11
src/pal/pal_posix.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include "../ds/bits.h"
#include "../ds/address.h"
#include "../mem/allocconfig.h"
#include <string.h>
@@ -53,7 +53,7 @@ namespace snmalloc
*/
void notify_not_using(void* p, size_t size) noexcept
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
assert(is_aligned_block<OS_PAGE_SIZE>(p, size));
UNUSED(p);
UNUSED(size);
}
@@ -68,8 +68,7 @@ namespace snmalloc
template<ZeroMem zero_mem>
void notify_using(void* p, size_t size) noexcept
{
assert(
bits::is_aligned_block<OS_PAGE_SIZE>(p, size) || (zero_mem == NoZero));
assert(is_aligned_block<OS_PAGE_SIZE>(p, size) || (zero_mem == NoZero));
if constexpr (zero_mem == YesZero)
static_cast<OS*>(this)->template zero<true>(p, size);
@@ -94,9 +93,9 @@ namespace snmalloc
template<bool page_aligned = false>
void zero(void* p, size_t size) noexcept
{
if (page_aligned || bits::is_aligned_block<OS_PAGE_SIZE>(p, size))
if (page_aligned || is_aligned_block<OS_PAGE_SIZE>(p, size))
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
assert(is_aligned_block<OS_PAGE_SIZE>(p, size));
void* r = mmap(
p,
size,

10
src/pal/pal_windows.h
View File

@@ -1,5 +1,6 @@
#pragma once
#include "../ds/address.h"
#include "../ds/bits.h"
#include "../mem/allocconfig.h"
@@ -107,7 +108,7 @@ namespace snmalloc
/// Notify platform that we will not be using these pages
void notify_not_using(void* p, size_t size) noexcept
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
assert(is_aligned_block<OS_PAGE_SIZE>(p, size));
BOOL ok = VirtualFree(p, size, MEM_DECOMMIT);
@@ -119,8 +120,7 @@ namespace snmalloc
template<ZeroMem zero_mem>
void notify_using(void* p, size_t size) noexcept
{
assert(
bits::is_aligned_block<OS_PAGE_SIZE>(p, size) || (zero_mem == NoZero));
assert(is_aligned_block<OS_PAGE_SIZE>(p, size) || (zero_mem == NoZero));
void* r = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE);
@@ -132,9 +132,9 @@ namespace snmalloc
template<bool page_aligned = false>
void zero(void* p, size_t size) noexcept
{
if (page_aligned || bits::is_aligned_block<OS_PAGE_SIZE>(p, size))
if (page_aligned || is_aligned_block<OS_PAGE_SIZE>(p, size))
{
assert(bits::is_aligned_block<OS_PAGE_SIZE>(p, size));
assert(is_aligned_block<OS_PAGE_SIZE>(p, size));
notify_not_using(p, size);
notify_using<YesZero>(p, size);
}

5
src/test/func/malloc/malloc.cc
View File

@@ -21,7 +21,7 @@ void check_result(size_t size, size_t align, void* p, int err, bool null)
if (our_malloc_usable_size(p) < size)
abort();
if (((uintptr_t)p % align) != 0)
if (static_cast<size_t>(reinterpret_cast<uintptr_t>(p) % align) != 0)
abort();
our_free(p);
@@ -106,8 +106,9 @@ int main(int argc, char** argv)
test_posix_memalign(0, 0, EINVAL, true);
test_posix_memalign((size_t)-1, 0, EINVAL, true);
test_posix_memalign(OS_PAGE_SIZE, sizeof(uintptr_t) / 2, EINVAL, true);
for (size_t align = sizeof(size_t); align <= SUPERSLAB_SIZE; align <<= 1)
for (size_t align = sizeof(uintptr_t); align <= SUPERSLAB_SIZE; align <<= 1)
{
for (snmalloc::sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++)
{

2
src/test/func/memory/memory.cc
View File

@@ -173,7 +173,7 @@ void test_external_pointer()
for (size_t offset = 0; offset < size; offset += 17)
{
void* p2 = (void*)((size_t)p1 + offset);
void* p2 = pointer_offset(p1, offset);
void* p3 = Alloc::external_pointer(p2);
void* p4 = Alloc::external_pointer<End>(p2);
UNUSED(p3);

6
src/test/perf/external_pointer/externalpointer.cc
View File

@@ -72,8 +72,8 @@ namespace test
size_t* external_ptr = objects[oid];
size_t size = *external_ptr;
size_t offset = (size >> 4) * (rand & 15);
size_t interior_ptr = ((size_t)external_ptr) + offset;
void* calced_external = Alloc::external_pointer((void*)interior_ptr);
void* interior_ptr = pointer_offset(external_ptr, offset);
void* calced_external = Alloc::external_pointer(interior_ptr);
if (calced_external != external_ptr)
abort();
}
@@ -88,7 +88,7 @@ int main(int, char**)
setup();
xoroshiro::p128r64 r;
#if NDEBUG
#ifdef NDEBUG
size_t nn = 30;
#else
size_t nn = 3;