diff --git a/CMakeLists.txt b/CMakeLists.txt index b90d225..eeab751 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,10 @@ if(USE_MEASURE) target_compile_definitions(snmalloc_lib INTERFACE -DUSE_MEASURE) endif() +if(CACHE_FRIENDLY_OFFSET) + target_compile_definitions(snmalloc_lib INTERFACE -DCACHE_FRIENDLY_OFFSET=${CACHE_FRIENDLY_OFFSET}) +endif() + # To build with just the header library target define SNMALLOC_ONLY_HEADER_LIBRARY # in containing Cmake file. if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) @@ -125,10 +129,6 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))") set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden) - if(CACHE_FRIENDLY_OFFSET) - target_compile_definitions(${name} PRIVATE -DCACHE_FRIENDLY_OFFSET=${CACHE_FRIENDLY_OFFSET}) - endif() - if(EXPOSE_EXTERNAL_PAGEMAP) target_compile_definitions(${name} PRIVATE -DSNMALLOC_EXPOSE_PAGEMAP) endif() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6e09d59..cec0c91 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,6 +41,13 @@ jobs: BuildType: Debug SelfHost: true + Cache Friendly: + CC: clang-6.0 + CXX: clang++-6.0 + BuildType: Debug + SelfHost: false + CMakeArgs: '-DCACHE_FRIENDLY_OFFSET=64' + steps: - script: | sudo add-apt-repository ppa:ubuntu-toolchain-r/test @@ -60,9 +67,9 @@ jobs: displayName: 'Install Build Dependencies' - task: CMake@1 - displayName: 'CMake .. -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"' + displayName: 'CMake .. $(CMakeArgs) -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"' inputs: - cmakeArgs: '.. -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"' + cmakeArgs: '.. $(CMakeArgs) -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" $(CMakeOptions)' - script: | ninja diff --git a/src/mem/alloc.h b/src/mem/alloc.h index e39724d..155f01a 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1072,13 +1072,13 @@ namespace snmalloc assert(sizeclass < NUM_SMALL_CLASSES); auto& fl = small_fast_free_lists[sizeclass]; - auto head = fl.value; + void* head = fl.value; if (likely(head != nullptr)) { - void* p = head; // Read the next slot from the memory that's about to be allocated. - fl.value = Metaslab::follow_next(p); + fl.value = Metaslab::follow_next(head); + void* p = remove_cache_friendly_offset(head, sizeclass); if constexpr (zero_mem == YesZero) { large_allocator.memory_provider.zero(p, size); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 7295324..b422e99 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -204,7 +204,7 @@ namespace snmalloc while (curr != nullptr) { // Check we are looking at a correctly aligned block - void* start = curr; + void* start = remove_cache_friendly_offset(curr, sizeclass); assert( ((address_cast(start) - address_cast(slab) - offset) % size) == 0); diff --git a/src/mem/slab.h b/src/mem/slab.h index 5f21676..f6bb8cf 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -64,7 +64,9 @@ namespace snmalloc meta.add_use(); assert(meta.used == meta.allocated); - p = pointer_offset(this, meta.link); + void* link = pointer_offset(this, meta.link); + p = remove_cache_friendly_offset(link, meta.sizeclass); + meta.set_full(); sl.pop(); p_has_value = true; @@ -113,16 +115,18 @@ namespace snmalloc if (!p_has_value) { - p = pointer_offset(this, meta.head); + void* head = pointer_offset(this, meta.head); // Read the next slot from the memory that's about to be allocated. - void* next = Metaslab::follow_next(p); + void* next = Metaslab::follow_next(head); // Put everything in allocators small_class free list. meta.head = 1; fast_free_list.value = next; // Treat stealing the free list as allocating it all. // Link is not in use, i.e. - 1 is required. meta.used = meta.allocated - 1; + + p = remove_cache_friendly_offset(head, meta.sizeclass); } assert(is_start_of_object(Superslab::get(p), p));