From a5379b24d5b239161abfc9c080c82b687438b35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20Li=C3=A9tar?= Date: Wed, 17 Jul 2019 13:14:59 +0100 Subject: [PATCH 1/4] Add remove_cache_friendly_offset calls in a few places that were missing it. --- CMakeLists.txt | 8 ++++---- azure-pipelines.yml | 11 +++++++++-- src/mem/alloc.h | 6 +++--- src/mem/metaslab.h | 2 +- src/mem/slab.h | 10 +++++++--- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b90d225..eeab751 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,10 @@ if(USE_MEASURE) target_compile_definitions(snmalloc_lib INTERFACE -DUSE_MEASURE) endif() +if(CACHE_FRIENDLY_OFFSET) + target_compile_definitions(snmalloc_lib INTERFACE -DCACHE_FRIENDLY_OFFSET=${CACHE_FRIENDLY_OFFSET}) +endif() + # To build with just the header library target define SNMALLOC_ONLY_HEADER_LIBRARY # in containing Cmake file. if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) @@ -125,10 +129,6 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))") set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden) - if(CACHE_FRIENDLY_OFFSET) - target_compile_definitions(${name} PRIVATE -DCACHE_FRIENDLY_OFFSET=${CACHE_FRIENDLY_OFFSET}) - endif() - if(EXPOSE_EXTERNAL_PAGEMAP) target_compile_definitions(${name} PRIVATE -DSNMALLOC_EXPOSE_PAGEMAP) endif() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6e09d59..cec0c91 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,6 +41,13 @@ jobs: BuildType: Debug SelfHost: true + Cache Friendly: + CC: clang-6.0 + CXX: clang++-6.0 + BuildType: Debug + SelfHost: false + CMakeArgs: '-DCACHE_FRIENDLY_OFFSET=64' + steps: - script: | sudo add-apt-repository ppa:ubuntu-toolchain-r/test @@ -60,9 +67,9 @@ jobs: displayName: 'Install Build Dependencies' - task: CMake@1 - displayName: 'CMake .. -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"' + displayName: 'CMake .. $(CMakeArgs) -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"' inputs: - cmakeArgs: '.. -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"' + cmakeArgs: '.. $(CMakeArgs) -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" $(CMakeOptions)' - script: | ninja diff --git a/src/mem/alloc.h b/src/mem/alloc.h index e39724d..155f01a 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1072,13 +1072,13 @@ namespace snmalloc assert(sizeclass < NUM_SMALL_CLASSES); auto& fl = small_fast_free_lists[sizeclass]; - auto head = fl.value; + void* head = fl.value; if (likely(head != nullptr)) { - void* p = head; // Read the next slot from the memory that's about to be allocated. - fl.value = Metaslab::follow_next(p); + fl.value = Metaslab::follow_next(head); + void* p = remove_cache_friendly_offset(head, sizeclass); if constexpr (zero_mem == YesZero) { large_allocator.memory_provider.zero(p, size); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 7295324..b422e99 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -204,7 +204,7 @@ namespace snmalloc while (curr != nullptr) { // Check we are looking at a correctly aligned block - void* start = curr; + void* start = remove_cache_friendly_offset(curr, sizeclass); assert( ((address_cast(start) - address_cast(slab) - offset) % size) == 0); diff --git a/src/mem/slab.h b/src/mem/slab.h index 5f21676..f6bb8cf 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -64,7 +64,9 @@ namespace snmalloc meta.add_use(); assert(meta.used == meta.allocated); - p = pointer_offset(this, meta.link); + void* link = pointer_offset(this, meta.link); + p = remove_cache_friendly_offset(link, meta.sizeclass); + meta.set_full(); sl.pop(); p_has_value = true; @@ -113,16 +115,18 @@ namespace snmalloc if (!p_has_value) { - p = pointer_offset(this, meta.head); + void* head = pointer_offset(this, meta.head); // Read the next slot from the memory that's about to be allocated. - void* next = Metaslab::follow_next(p); + void* next = Metaslab::follow_next(head); // Put everything in allocators small_class free list. meta.head = 1; fast_free_list.value = next; // Treat stealing the free list as allocating it all. // Link is not in use, i.e. - 1 is required. meta.used = meta.allocated - 1; + + p = remove_cache_friendly_offset(head, meta.sizeclass); } assert(is_start_of_object(Superslab::get(p), p)); From e87d1849268d79139abf4c8d03be62c53b542fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20Li=C3=A9tar?= Date: Wed, 17 Jul 2019 13:51:24 +0100 Subject: [PATCH 2/4] Remove $(CMakeOptions) that was leftover by accident. --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cec0c91..294c4cf 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -69,7 +69,7 @@ jobs: - task: CMake@1 displayName: 'CMake .. $(CMakeArgs) -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"' inputs: - cmakeArgs: '.. $(CMakeArgs) -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" $(CMakeOptions)' + cmakeArgs: '.. $(CMakeArgs) -GNinja -DCMAKE_BUILD_TYPE=$(BuildType) -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"' - script: | ninja From b185337c072430d6d9f62ed4750722aee5b07021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20Li=C3=A9tar?= Date: Wed, 17 Jul 2019 13:54:35 +0100 Subject: [PATCH 3/4] Specify a CMakeArgs for every matrix entry. --- azure-pipelines.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 294c4cf..728baab 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,30 +16,35 @@ jobs: CXX: clang++-6.0 BuildType: Debug SelfHost: false + CMakeArgs: '' Clang-6 Release: CC: clang-6.0 CXX: clang++-6.0 BuildType: Release SelfHost: false + CMakeArgs: '' GCC-7 Debug: CC: gcc-7 CXX: g++-7 BuildType: Debug SelfHost: false + CMakeArgs: '' GCC-7 Release: CC: gcc-7 CXX: g++-7 BuildType: Release SelfHost: false + CMakeArgs: '' Self Host: CC: clang-6.0 CXX: clang++-6.0 BuildType: Debug SelfHost: true + CMakeArgs: '' Cache Friendly: CC: clang-6.0 From 7047cda8ed5bc9837e0b247e0df142ab79273df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20Li=C3=A9tar?= Date: Wed, 17 Jul 2019 14:00:34 +0100 Subject: [PATCH 4/4] Fix warning about shadowing. --- src/mem/slab.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mem/slab.h b/src/mem/slab.h index f6bb8cf..a3fe45e 100644 --- a/src/mem/slab.h +++ b/src/mem/slab.h @@ -115,10 +115,10 @@ namespace snmalloc if (!p_has_value) { - void* head = pointer_offset(this, meta.head); + p = pointer_offset(this, meta.head); // Read the next slot from the memory that's about to be allocated. - void* next = Metaslab::follow_next(head); + void* next = Metaslab::follow_next(p); // Put everything in allocators small_class free list. meta.head = 1; fast_free_list.value = next; @@ -126,7 +126,7 @@ namespace snmalloc // Link is not in use, i.e. - 1 is required. meta.used = meta.allocated - 1; - p = remove_cache_friendly_offset(head, meta.sizeclass); + p = remove_cache_friendly_offset(p, meta.sizeclass); } assert(is_start_of_object(Superslab::get(p), p));