From 0007a53ef9967f9b3b20f052188e9749ebfe5336 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Tue, 3 Aug 2021 17:10:56 +0100 Subject: [PATCH 1/5] Move CI to GitHub Actions A few highlights relative to our existing CI: - Add a FreeBSD 12.2 and 13.0 runner so we have some FreeBSD CI. - Windows builds use msbuild with the Visual-Studio-provided clang toolchain to test clang - The matrix builds describe the axes of the matrix, not all points. - The Arm builds now cross-compile with a native clang and run the tests with qemu, rather than running the compiler, linker, and ctest all with qemu. This also includes a fix for one of the tests that was doing `static_cast(1) << 36`, which is undefined behaviour and was sometimes causing qemu to hang. There is now an assert to catch this in the future. --- .github/workflows/main.yml | 207 ++++++++++++++++++ README.md | 2 +- azure-pipelines.yml | 347 ------------------------------- ci/Toolchain.cmake | 13 ++ ci/linux_arm64 | 7 - ci/linux_armhf | 7 - ci/linux_x64 | 17 -- ci/linux_x86 | 7 - ci/scripts/build.sh | 19 -- ci/scripts/test.sh | 14 -- src/ds/bits.h | 1 + src/test/func/pagemap/pagemap.cc | 5 +- 12 files changed, 225 insertions(+), 421 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 azure-pipelines.yml create mode 100644 ci/Toolchain.cmake delete mode 100644 ci/linux_arm64 delete mode 100644 ci/linux_armhf delete mode 100644 ci/linux_x64 delete mode 100644 ci/linux_x86 delete mode 100755 ci/scripts/build.sh delete mode 100755 ci/scripts/test.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..808b6d0 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,207 @@ +name: snmalloc CI + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master ] + pull_request: + branches: [ master, snmalloc2 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + unixlike: + strategy: + matrix: + # Build each combination of OS and release/debug variants + os: [ "ubuntu-latest", "ubuntu-18.04", "macos-11", "macos-10.15", "freebsd-12.2", "freebsd-13.0" ] + build-type: [ Release, Debug ] + extra-cmake-flags: [ " " ] + # Modify the complete matrix + include: + # Provide the dependency installation for each platform + - os: "ubuntu-18.04" + dependencies: "sudo apt install ninja-build" + - os: "ubuntu-latest" + dependencies: "sudo apt install ninja-build" + - os: "macos-11" + dependencies: "brew update && brew install ninja" + - os: "macos-10.15" + dependencies: "brew update && brew install ninja" + # Skip the tests for the FreeBSD release builds + - os: "freebsd-13.0" + build-type: Release + build-only: yes + - os: "freebsd-12.2" + build-type: Debug + build-only: yes + # Add the self-host build + - os: ubuntu-latest + build-type: Debug + self-host: true + # Add an extra element to the matrix that does a build with clang 12 + # but doesn't run tests. + - os: "freebsd-13.0" + variant: Clang 12 (Build only) + extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++12" + build-only: yes + - os: ubuntu-latest + variant: Clang 10 libstdc++ (Build only) + dependencies: "sudo apt install ninja-build" + extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_CXX_FLAGS=-stdlib=libstdc++" + build-only: yes + # Don't abort runners if a single one fails + fail-fast: false + runs-on: ${{ matrix.os }} + name: ${{ matrix.os }} ${{ matrix.build-type }} ${{ matrix.variant }} + steps: + - uses: actions/checkout@v2 + - name: Install build dependencies + run: ${{ matrix.dependencies }} + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -G Ninja ${{ matrix.extra-cmake-flags }} + # Build with a nice ninja status line + - name: Build + working-directory: ${{github.workspace}}/build + run: NINJA_STATUS="%p [%f:%s/%t] %o/s, %es" ninja + # If the tests are enabled for this job, run them + - name: Test + if: ${{ matrix.build-only != 'yes' }} + working-directory: ${{github.workspace}}/build + run: ctest --output-on-failure -j 4 + - name: Selfhost + if: ${{ matrix.self-host }} + working-directory: ${{github.workspace}}/build + run: | + sudo cp libsnmallocshim.so libsnmallocshim-16mib.so libsnmallocshim-oe.so /usr/local/lib/ + ninja clean + LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja + ninja clean + LD_PRELOAD=/usr/local/lib/libsnmallocshim-16mib.so ninja + ninja clean + LD_PRELOAD=/usr/local/lib/libsnmallocshim-oe.so ninja + + qemu-crossbuild: + strategy: + matrix: + build-type: [ Release, Debug ] + arch: [ arm64, armhf ] + include: + - arch: armhf + system-processor: arm + triple: arm-linux-gnueabihf + rtld: ld-linux-armhf.so.3 + - arch: arm64 + system-processor: aarch64 + triple: aarch64-linux-gnu + rtld: ld-linux-aarch64.so.1 + # Don't abort runners if a single one fails + fail-fast: false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install cross-compile toolchain and QEMU + # Install the dependencies and clang 13. Earlier versions of clang don't + # find the multilib things for this week's Ubuntu filesystem layout. + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 13 + sudo apt install libstdc++-9-dev-${{ matrix.arch }}-cross qemu-user ninja-build lld-13 clang-13 + - name: Configure + run: > + RTLD_NAME=${{ matrix.rtld }} + AARCH=${{ matrix.arch }} + TRIPLE=${{ matrix.triple}} + cmake + -B ${{github.workspace}}/build + -DCMAKE_BUILD_TYPE=${{matrix.build-type}} + -G Ninja + -DSNMALLOC_CI_BUILD=ON + -DSNMALLOC_QEMU_WORKAROUND=ON + -DSNMALLOC_STATIC_LIBRARY=OFF + -DCMAKE_TOOLCHAIN_FILE=ci/Toolchain.cmake + - name: Build + working-directory: ${{github.workspace}}/build + run: NINJA_STATUS="%p [%f:%s/%t] %o/s, %es" ninja + # Run the tests, skipping the -malloc and perf- tests (perf doesn't make + # sense in an emulator and the pass-through malloc is slightly flaky in + # QEMU) + - name: Test + working-directory: ${{github.workspace}}/build + run: ctest --output-on-failure -E '(perf-.*)|(.*-malloc$)' + timeout-minutes: 30 + + windows: + strategy: + matrix: + # Build each combination of OS and release/debug variants + os: [ windows-2016, windows-2019 ] + build-type: [ Release, Debug ] + arch: [ Win32, x64 ] + toolchain: [ "", "-T ClangCL" ] + extra-cmake-flags: [ "" ] + # The ClangCL toolchain was added in Visual Studio 2019, the Windows + # 2016 runners have only VS 2017, so skip them for this configuration + exclude: + - os: windows-2016 + toolchain: "-T ClangCL" + # Add an extra check for the Windows 8 compatible PAL + include: + - os: windows-2016 + build-type: Release + arch: x64 + toolchain: "" + extra-cmake-flags: -DWIN8COMPAT=TRUE + variant: Windows 8 compatible + # Don't abort runners if a single one fails + fail-fast: false + runs-on: ${{ matrix.os }} + name: ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.build-type }} ${{ matrix.toolchain }} ${{ matrix.variant }} + steps: + - uses: actions/checkout@v2 + - name: Configure CMake + run: | + cmake -B ${{github.workspace}}/build -A ${{ matrix.arch }} ${{ matrix.toolchain }} ${{ matrix.extra-cmake-flags }} -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On + # Build with whatever version of msbuild was configured, for a single build configuration. + - name: Build + run: cmake --build ${{github.workspace}}/build -- /m /p:Configuration=${{ matrix.build-type }} + # Run the tests. + - name: Test + working-directory: ${{ github.workspace }}/build + run: ctest -j 2 --interactive-debug-mode 0 --output-on-failure -C ${{ matrix.build-type }} + + + # Job to run clang-format and report errors + format: + runs-on: ubuntu-18.04 + # We don't need to do the build for this job, but we need to configure it to get the clang-format target + steps: + - uses: actions/checkout@v2 + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build + - name: Install clang-tidy + run: sudo apt install clang-tidy-9 + # Run the clang-format check and error if it generates a diff + - name: Run clang-format + working-directory: ${{github.workspace}}/build + run: | + set -eo pipefail + make clangformat + git diff --exit-code + - name: Run clang-tidy + run: | + clang-tidy-9 src/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 + if [ -f tidy.fail ] ; then + cat tidy.fail + exit 1 + fi + + all-checks: + needs: [unixlike, qemu-crossbuild, windows, format] + runs-on: ubuntu-latest + steps: + - name: Dummy step + run: true diff --git a/README.md b/README.md index 6f732b3..61a7a8c 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Comprehensive details about snmalloc's design can be found in the current implementation are [described here](difference.md). Since writing the paper, the performance of snmalloc has improved considerably. -[![Build Status](https://dev.azure.com/snmalloc/snmalloc/_apis/build/status/Microsoft.snmalloc?branchName=master)](https://dev.azure.com/snmalloc/snmalloc/_build/latest?definitionId=1?branchName=master) +[![snmalloc CI](https://github.com/microsoft/snmalloc/actions/workflows/main.yml/badge.svg)](https://github.com/microsoft/snmalloc/actions/workflows/main.yml) # Further documentation diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 6674f31..0000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,347 +0,0 @@ -trigger: -- master - -pr: -- master - -jobs: -- job: - displayName: Linux - pool: - vmImage: 'ubuntu-18.04' - - strategy: - matrix: - 64-bit Clang-7 Debug: - CC: clang-7 - CXX: clang++-7 - BuildType: Debug - SelfHost: false - CMakeArgs: '' - Image: snmallocciteam/build_linux_x64:latest - - 64-bit Clang-7 Release: - CC: clang-7 - CXX: clang++-7 - BuildType: Release - SelfHost: false - CMakeArgs: '' - Image: snmallocciteam/build_linux_x64:latest - - 64-bit Clang-8 Debug: - CC: clang-8 - CXX: clang++-8 - BuildType: Debug - SelfHost: false - CMakeArgs: '' - Image: snmallocciteam/build_linux_x64:latest - - 64-bit Clang-8 Release: - CC: clang-8 - CXX: clang++-8 - BuildType: Release - SelfHost: false - CMakeArgs: '' - Image: snmallocciteam/build_linux_x64:latest - - 64-bit Clang-9 Debug: - CC: clang-9 - CXX: clang++-9 - BuildType: Debug - SelfHost: false - CMakeArgs: '' - Image: snmallocciteam/build_linux_x64:latest - - 64-bit Clang-10 Debug C++20: - CC: clang-10 - CXX: clang++-10 - BuildType: Debug - SelfHost: false - CMakeArgs: '-DSNMALLOC_USE_CXX20=On' - Image: snmallocciteam/build_linux_x64:latest - - 64-bit Clang-9 Release: - CC: clang-9 - CXX: clang++-9 - BuildType: Release - SelfHost: false - CMakeArgs: '' - Image: snmallocciteam/build_linux_x64:latest - - 64-bit GCC-8 Debug: - CC: gcc-8 - CXX: g++-8 - BuildType: Debug - SelfHost: false - CMakeArgs: '' - Image: snmallocciteam/build_linux_x64:latest - - 64-bit GCC-8 Release: - CC: gcc-8 - CXX: g++-8 - BuildType: Release - SelfHost: false - CMakeArgs: '' - Image: snmallocciteam/build_linux_x64:latest - - 64-bit Self Host: - CC: clang-7 - CXX: clang++-7 - BuildType: Debug - SelfHost: true - CMakeArgs: '' - Image: snmallocciteam/build_linux_x64:latest - - 32-bit Clang-9 Debug: - CC: clang-9 - CXX: clang++-9 - BuildType: Debug - SelfHost: false - CMakeArgs: '' - Image: snmallocciteam/build_linux_x86:latest - - 32-bit Clang-9 Release: - CC: clang-9 - CXX: clang++-9 - BuildType: Release - SelfHost: false - CMakeArgs: '' - Image: snmallocciteam/build_linux_x86:latest - - container: $[ variables['Image'] ] - steps: - - script: | - set -eo pipefail - ci/scripts/build.sh - env: - CC: $(CC) - CXX: $(CXX) - BUILD_TYPE: $(BuildType) - CMAKE_ARGS: $(CMakeArgs) - displayName: 'Build' - - - script: | - set -eo pipefail - ci/scripts/test.sh - env: - SELF_HOST: $(SelfHost) - BUILD_TYPE: $(BuildType) - displayName: 'Test' - -- job: - displayName: 'ARM Linux' - pool: - vmImage: 'ubuntu-18.04' - - strategy: - matrix: - 64-bit Clang-9 Debug: - CC: clang-9 - CXX: clang++-9 - BuildType: Debug - SelfHost: false - CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On' - Image: snmallocciteam/build_linux_arm64:latest - - 64-bit Clang-9 Release: - CC: clang-9 - CXX: clang++-9 - BuildType: Release - SelfHost: false - CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On' - Image: snmallocciteam/build_linux_arm64:latest - - 32-bit Clang-9 Debug: - CC: clang-9 - CXX: clang++-9 - BuildType: Debug - SelfHost: false - CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On' - Image: snmallocciteam/build_linux_armhf:latest - - 32-bit Clang-9 Release: - CC: clang-9 - CXX: clang++-9 - BuildType: Release - SelfHost: false - CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On' - Image: snmallocciteam/build_linux_armhf:latest - - steps: - - script: | - sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset - displayName: 'Enable QEmu' - - - script: | - sudo docker run -v $PWD:/src \ - -e CC=$(CC) \ - -e CXX=$(CXX) \ - -e BUILD_TYPE=$(BuildType) \ - -e CMAKE_ARGS=$(CMakeArgs) \ - $(Image) \ - ci/scripts/build.sh - displayName: 'Build' - - - script: | - sudo docker run -v $PWD:/src \ - -e BUILD_TYPE=$(BuildType) \ - -e SELF_HOST=$(SelfHost) \ - $(Image) \ - ci/scripts/test.sh - displayName: 'Test' - -- job: - displayName: Windows - pool: - vmImage: 'vs2017-win2016' - strategy: - matrix: - 64-bit Debug: - BuildType: Debug - CMakeArgs: '-G"Visual Studio 15 2017 Win64"' - JFlag: -j 4 - - 64-bit Release: - BuildType: Release - CMakeArgs: '-G"Visual Studio 15 2017 Win64"' - JFlag: '-j 2' - - 64-bit Release Windows8Compat: - BuildType: Release - CMakeArgs: '-G"Visual Studio 15 2017 Win64" -DWIN8COMPAT=TRUE' - JFlag: '-j 2' - - 32-bit Debug: - BuildType: Debug - CMakeArgs: '-G"Visual Studio 15 2017"' - JFlag: '-j 4' - - 32-bit Release: - BuildType: Release - CMakeArgs: '-G"Visual Studio 15 2017"' - JFlag: '-j 2' - - steps: - - script: | - bash -c "cat /proc/cpuinfo" - bash -c "cat /proc/meminfo" - displayName: 'Machine stats' - - - task: CMake@1 - displayName: 'CMake .. $(CMakeArgs) -DCMAKE_BUILD_TYPE=$(BuildType) -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On' - inputs: - cmakeArgs: '.. $(CMakeArgs) -DCMAKE_BUILD_TYPE=$(BuildType) -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On' - - - task: MSBuild@1 - displayName: 'Build solution build/snmalloc.sln' - inputs: - solution: build/snmalloc.sln - msbuildArguments: '/m /p:Configuration=$(BuildType)' - - - script: 'ctest $(JFlag) --interactive-debug-mode 0 --output-on-failure -C $(BuildType)' - workingDirectory: build - displayName: 'Run Ctest' - -- job: - displayName: WinClang - pool: - vmImage: 'windows-2019' - strategy: - matrix: - 64-bit Debug Clang: - BuildType: Debug - CMakeArgs: '-GNinja -DCMAKE_C_COMPILER="C:/Program Files/LLVM/bin/clang-cl.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/LLVM/bin/clang-cl.exe" -DCMAKE_RC_COMPILER="C:/Program Files/LLVM/bin/llvm-rc"' - JFlag: -j 4 - 64-bit Release Clang: - BuildType: Release - CMakeArgs: '-GNinja -DCMAKE_C_COMPILER="C:/Program Files/LLVM/bin/clang-cl.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/LLVM/bin/clang-cl.exe" -DCMAKE_RC_COMPILER="C:/Program Files/LLVM/bin/llvm-rc"' - JFlag: -j 4 - - steps: - - script: | - bash -c "cat /proc/cpuinfo" - bash -c "cat /proc/meminfo" - displayName: 'Machine stats' - - - script: | - choco install --accept-license -y Ninja llvm - displayName: 'Dependencies' - - - script: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - mkdir build - cd build - cmake .. $(CMakeArgs) -DCMAKE_BUILD_TYPE=$(BuildType) -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On - cmake --build . --config ${BuildType} - displayName: 'build' - - - script: | - set PATH=%PATH%;"C:\Program Files\LLVM\bin\" - func-statistics-1.exe - ctest $(JFlag) --interactive-debug-mode 0 --output-on-failure -C $(BuildType) - - workingDirectory: build - displayName: 'Run Ctest' - -- job: - displayName: macOS - pool: - vmImage: 'macOS-10.15' - strategy: - matrix: - Debug: - BuildType: Debug - Release: - BuildType: Release - - steps: - - task: CMake@1 - displayName: 'CMake .. -DCMAKE_BUILD_TYPE=$(BuildType) -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On' - inputs: - cmakeArgs: '.. -DCMAKE_BUILD_TYPE=$(BuildType) -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On' - - - script: | - set -eo pipefail - make -j 4 - - workingDirectory: build - failOnStderr: true - displayName: 'Compile' - - - script: | - set -eo pipefail - ctest -j 4 --output-on-failure -C $(BuildType) - - workingDirectory: build - failOnStderr: true - displayName: 'Test' - -- job: - displayName: Format - pool: - vmImage: 'ubuntu-18.04' - container: snmallocciteam/build_linux_x64:latest - steps: - - script: | - mkdir build - cd build - cmake -GNinja .. - - env: - CC: /usr/bin/gcc-8 - CXX: /usr/bin/g++-8 - - - script: | - set -eo pipefail - ninja clangformat - git diff --exit-code - - workingDirectory: build - failOnStderr: true - displayName: 'Clang-Format' - - - script: | - set -eo pipefail - clang-tidy-9 src/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 - - displayName: 'Clang-Tidy' diff --git a/ci/Toolchain.cmake b/ci/Toolchain.cmake new file mode 100644 index 0000000..e3e041c --- /dev/null +++ b/ci/Toolchain.cmake @@ -0,0 +1,13 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR $ENV{ARCH}) + +set(triple $ENV{TRIPLE}) + +set(CMAKE_C_COMPILER clang-13) +set(CMAKE_C_COMPILER_TARGET ${triple}) +set(CMAKE_CXX_COMPILER clang++-13) +set(CMAKE_CXX_COMPILER_TARGET ${triple}) +set(CROSS_LINKER_FLAGS "-fuse-ld=lld -Wl,--dynamic-linker=/usr/${triple}/lib/$ENV{RTLD_NAME},-rpath,/usr/${triple}/lib") +set(CMAKE_EXE_LINKER_FLAGS ${CROSS_LINKER_FLAGS}) +set(CMAKE_SHARED_LINKER_FLAGS ${CROSS_LINKER_FLAGS}) +set(CMAKE_MODULE_LINKER_FLAGS ${CROSS_LINKER_FLAGS}) diff --git a/ci/linux_arm64 b/ci/linux_arm64 deleted file mode 100644 index 83bf75f..0000000 --- a/ci/linux_arm64 +++ /dev/null @@ -1,7 +0,0 @@ -FROM multiarch/ubuntu-core:arm64-bionic - -WORKDIR /src - -RUN apt update \ - && apt install --no-install-recommends -y ninja-build clang++-9 cmake \ - && apt -y clean \ No newline at end of file diff --git a/ci/linux_armhf b/ci/linux_armhf deleted file mode 100644 index bb132a6..0000000 --- a/ci/linux_armhf +++ /dev/null @@ -1,7 +0,0 @@ -FROM multiarch/ubuntu-core:armhf-bionic - -WORKDIR /src - -RUN apt update \ - && apt install --no-install-recommends -y ninja-build clang++-9 cmake \ - && apt -y clean \ No newline at end of file diff --git a/ci/linux_x64 b/ci/linux_x64 deleted file mode 100644 index 88552a0..0000000 --- a/ci/linux_x64 +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:20.04 - -RUN apt update \ - && apt install -y software-properties-common \ - && add-apt-repository ppa:ubuntu-toolchain-r/test \ - && apt update \ - && apt install --no-install-recommends -y sudo cmake ninja-build git \ - && apt install --no-install-recommends -y g++-8 \ - && apt install --no-install-recommends -y clang-7 \ - && apt install --no-install-recommends -y clang-8 \ - && apt install --no-install-recommends -y clang-9 \ - && apt install --no-install-recommends -y clang-10 \ - && apt install --no-install-recommends -y clang-format-9 clang-tidy-9 \ - && update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-9 100 \ - && apt remove -y software-properties-common \ - && apt -y autoremove \ - && apt -y clean \ No newline at end of file diff --git a/ci/linux_x86 b/ci/linux_x86 deleted file mode 100644 index 6124ddf..0000000 --- a/ci/linux_x86 +++ /dev/null @@ -1,7 +0,0 @@ -FROM multiarch/ubuntu-core:x86-bionic - -WORKDIR /src - -RUN apt update \ - && apt install --no-install-recommends -y ninja-build clang++-9 cmake \ - && apt -y clean diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh deleted file mode 100755 index 7d49417..0000000 --- a/ci/scripts/build.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -mkdir build -cd build - -if [[ "$CXX" == clang++* ]]; then - CMAKE_CXX_FLAGS+=" -stdlib=libstdc++"; -fi - -cmake \ - $CMAKE_ARGS \ - -GNinja \ - -DSNMALLOC_CI_BUILD=ON \ - -DSNMALLOC_RUST_SUPPORT=ON \ - -DSNMALLOC_QEMU_WORKAROUND=ON \ - -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ - -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" \ - ${SNMALLOC_SRC-..} -ninja \ No newline at end of file diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh deleted file mode 100755 index 02e2e65..0000000 --- a/ci/scripts/test.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -cd build -if [ $SELF_HOST = false ]; then - ctest -j 4 --output-on-failure -C $BUILD_TYPE -else - sudo cp libsnmallocshim.so libsnmallocshim-16mib.so libsnmallocshim-oe.so /usr/local/lib/ - ninja clean - LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja - ninja clean - LD_PRELOAD=/usr/local/lib/libsnmallocshim-16mib.so ninja - ninja clean - LD_PRELOAD=/usr/local/lib/libsnmallocshim-oe.so ninja -fi \ No newline at end of file diff --git a/src/ds/bits.h b/src/ds/bits.h index ec32e3c..777a523 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -48,6 +48,7 @@ namespace snmalloc constexpr T one_at_bit(S shift) { static_assert(std::is_integral_v, "Type must be integral"); + SNMALLOC_ASSERT(sizeof(T) * 8 > static_cast(shift)); return (static_cast(1)) << shift; } diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index ad96170..887e927 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -24,8 +24,9 @@ int main(int argc, char** argv) setup(); + constexpr int bits_to_test = bits::is64() ? 36 : 31; T value = 0; - for (uintptr_t ptr = 0; ptr < bits::one_at_bit(36); + for (uintptr_t ptr = 0; ptr < bits::one_at_bit(bits_to_test); ptr += bits::one_at_bit(GRANULARITY_BITS + 3)) { pagemap_test.set(ptr, value); @@ -38,7 +39,7 @@ int main(int argc, char** argv) std::cout << std::endl; value = 0; - for (uintptr_t ptr = 0; ptr < bits::one_at_bit(36); + for (uintptr_t ptr = 0; ptr < bits::one_at_bit(bits_to_test); ptr += bits::one_at_bit(GRANULARITY_BITS + 3)) { T result = pagemap_test.get(ptr); From a8ccc5d4739700edc03cb1bb7a09b828e82a0716 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Wed, 4 Aug 2021 15:07:33 +0100 Subject: [PATCH 2/5] Fix status badge and LLVM fetching for qemu builds (#362) The status badge should report the most recent status for the master branch, not the most recent actions run. llvm.sh wasn't updated when trunk moved to 14 so was failing to fetch clang-13 (which is now in the llvm-toolchain-focal-13 repo, not the llvm-toolchain-focal repo). Duplicate the correct logic here rather than relying on the external script. --- .github/workflows/main.yml | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 808b6d0..99fe32a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -106,10 +106,10 @@ jobs: # Install the dependencies and clang 13. Earlier versions of clang don't # find the multilib things for this week's Ubuntu filesystem layout. run: | - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 13 - sudo apt install libstdc++-9-dev-${{ matrix.arch }}-cross qemu-user ninja-build lld-13 clang-13 + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" + sudo apt update + sudo apt install libstdc++-9-dev-${{ matrix.arch }}-cross qemu-user ninja-build clang-13 lld-13 - name: Configure run: > RTLD_NAME=${{ matrix.rtld }} diff --git a/README.md b/README.md index 61a7a8c..96c6311 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Comprehensive details about snmalloc's design can be found in the current implementation are [described here](difference.md). Since writing the paper, the performance of snmalloc has improved considerably. -[![snmalloc CI](https://github.com/microsoft/snmalloc/actions/workflows/main.yml/badge.svg)](https://github.com/microsoft/snmalloc/actions/workflows/main.yml) +[![snmalloc CI](https://github.com/microsoft/snmalloc/actions/workflows/main.yml/badge.svg?branch=master)](https://github.com/microsoft/snmalloc/actions/workflows/main.yml) # Further documentation From aec5ac060b1d539fd92ae71191c19c91c354c388 Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo Date: Wed, 4 Aug 2021 18:33:51 +0100 Subject: [PATCH 3/5] Add cross CI for Ubuntu powerpc64el The naming is more than a little confusing. AIUI, in Ubuntu's terminology, the `powerpc` packages are for 32-bit machines, the `ppc64-powerpc` packages are for Big Endian 64-bit machines, and the `ppc64el` packages are for Little Endian 64-bit machines. Everyone seems to have agreed that the long-term answer for 64-bit PowerPC is Little Endian (Debian maintains an unofficial BE port, but Canonical/Ubuntu, RedHat, and SUSE all seem to have standardized on LE). For maximal confusion, the appropriate triple's first component is, however, `powerpc64le`. --- .github/workflows/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 99fe32a..3906c69 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,7 +87,7 @@ jobs: strategy: matrix: build-type: [ Release, Debug ] - arch: [ arm64, armhf ] + arch: [ arm64, armhf, ppc64el ] include: - arch: armhf system-processor: arm @@ -97,6 +97,10 @@ jobs: system-processor: aarch64 triple: aarch64-linux-gnu rtld: ld-linux-aarch64.so.1 + - arch: ppc64el + system-processor: powerpc64le + triple: powerpc64le-linux-gnu + rtld: ld64.so.2 # Don't abort runners if a single one fails fail-fast: false runs-on: ubuntu-latest From 605566974c09e7323ec1fdcb249d20296e8eab43 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Thu, 5 Aug 2021 10:13:59 +0100 Subject: [PATCH 4/5] Force global pagemaps into BSS (#364) Also add a check that the test programs are under about ten megabytes (they're currently around one on platforms that put inline statics full of zeroes into BSS and around 270 on ones that don't). Fixes #339 --- .github/workflows/main.yml | 3 +++ src/ds/defines.h | 8 ++++++++ src/mem/pagemap.h | 1 + 3 files changed, 12 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3906c69..dbb1709 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -66,6 +66,9 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build run: NINJA_STATUS="%p [%f:%s/%t] %o/s, %es" ninja + - name: Test file size of binaries is sane + working-directory: ${{github.workspace}}/build + run: "ls -l func-first_operation-1 ; [ $(ls -l func-first_operation-1 | awk '{ print $5}') -lt 10000000 ]" # If the tests are enabled for this job, run them - name: Test if: ${{ matrix.build-only != 'yes' }} diff --git a/src/ds/defines.h b/src/ds/defines.h index ca2289e..53d362b 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -44,6 +44,14 @@ # endif #endif +#ifdef __APPLE__ +# define SNMALLOC_FORCE_BSS __attribute__((section("__DATA,__bss"))) +#elif defined(__ELF__) +# define SNMALLOC_FORCE_BSS __attribute__((section(".bss"))) +#else +# define SNMALLOC_FORCE_BSS +#endif + #ifndef __has_builtin # define __has_builtin(x) 0 #endif diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h index 5f0972c..25050ec 100644 --- a/src/mem/pagemap.h +++ b/src/mem/pagemap.h @@ -464,6 +464,7 @@ namespace snmalloc * The global pagemap variable. The name of this symbol will include the * type of `T` and `U`. */ + SNMALLOC_FORCE_BSS inline static T global_pagemap; public: From 32187923b8c73b6f7be295ea623f4bc9064315fa Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Thu, 5 Aug 2021 10:21:07 +0100 Subject: [PATCH 5/5] Clean up QEMU CI Use a matrix containing objects rather than a matrix containing keys that we try to use to define arrays of things. Fix a typo in ARCH. Too much Arm made it AARCH. --- .github/workflows/main.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dbb1709..32a3ae7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,23 +90,23 @@ jobs: strategy: matrix: build-type: [ Release, Debug ] - arch: [ arm64, armhf, ppc64el ] - include: - - arch: armhf + arch: + - name: armhf system-processor: arm triple: arm-linux-gnueabihf rtld: ld-linux-armhf.so.3 - - arch: arm64 + - name: arm64 system-processor: aarch64 triple: aarch64-linux-gnu rtld: ld-linux-aarch64.so.1 - - arch: ppc64el + - name: ppc64el system-processor: powerpc64le triple: powerpc64le-linux-gnu rtld: ld64.so.2 # Don't abort runners if a single one fails fail-fast: false runs-on: ubuntu-latest + name: Cross-build for ${{ matrix.arch.triple }} steps: - uses: actions/checkout@v2 - name: Install cross-compile toolchain and QEMU @@ -116,12 +116,12 @@ jobs: wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - sudo add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" sudo apt update - sudo apt install libstdc++-9-dev-${{ matrix.arch }}-cross qemu-user ninja-build clang-13 lld-13 + sudo apt install libstdc++-9-dev-${{ matrix.arch.name }}-cross qemu-user ninja-build clang-13 lld-13 - name: Configure run: > - RTLD_NAME=${{ matrix.rtld }} - AARCH=${{ matrix.arch }} - TRIPLE=${{ matrix.triple}} + RTLD_NAME=${{ matrix.arch.rtld }} + ARCH=${{ matrix.arch.system-processor }} + TRIPLE=${{ matrix.arch.triple}} cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}}