Move CI to GitHub Actions

A few highlights relative to our existing CI:

- Add a FreeBSD 12.2 and 13.0 runner so we have some FreeBSD CI.
- Windows builds use msbuild with the Visual-Studio-provided clang toolchain to test clang
 - The matrix builds describe the axes of the matrix, not all points.
 - The Arm builds now cross-compile with a native clang and run the tests with qemu, rather than running the compiler, linker, and ctest all with qemu.
 
This also includes a fix for one of the tests that was doing `static_cast<unsigned int>(1) << 36`, which is undefined behaviour and was sometimes causing qemu to hang.  There is now an assert to catch this in the future.
This commit is contained in:
David Chisnall
2021-08-03 17:10:56 +01:00
committed by GitHub
parent 704bc423cc
commit 0007a53ef9
12 changed files with 225 additions and 421 deletions

207
.github/workflows/main.yml vendored Normal file
View File

@@ -0,0 +1,207 @@
name: snmalloc CI
# Controls when the workflow will run
on:
# Triggers the workflow on push or pull request events but only for the master branch
push:
branches: [ master ]
pull_request:
branches: [ master, snmalloc2 ]
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
jobs:
unixlike:
strategy:
matrix:
# Build each combination of OS and release/debug variants
os: [ "ubuntu-latest", "ubuntu-18.04", "macos-11", "macos-10.15", "freebsd-12.2", "freebsd-13.0" ]
build-type: [ Release, Debug ]
extra-cmake-flags: [ " " ]
# Modify the complete matrix
include:
# Provide the dependency installation for each platform
- os: "ubuntu-18.04"
dependencies: "sudo apt install ninja-build"
- os: "ubuntu-latest"
dependencies: "sudo apt install ninja-build"
- os: "macos-11"
dependencies: "brew update && brew install ninja"
- os: "macos-10.15"
dependencies: "brew update && brew install ninja"
# Skip the tests for the FreeBSD release builds
- os: "freebsd-13.0"
build-type: Release
build-only: yes
- os: "freebsd-12.2"
build-type: Debug
build-only: yes
# Add the self-host build
- os: ubuntu-latest
build-type: Debug
self-host: true
# Add an extra element to the matrix that does a build with clang 12
# but doesn't run tests.
- os: "freebsd-13.0"
variant: Clang 12 (Build only)
extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++12"
build-only: yes
- os: ubuntu-latest
variant: Clang 10 libstdc++ (Build only)
dependencies: "sudo apt install ninja-build"
extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_CXX_FLAGS=-stdlib=libstdc++"
build-only: yes
# Don't abort runners if a single one fails
fail-fast: false
runs-on: ${{ matrix.os }}
name: ${{ matrix.os }} ${{ matrix.build-type }} ${{ matrix.variant }}
steps:
- uses: actions/checkout@v2
- name: Install build dependencies
run: ${{ matrix.dependencies }}
- name: Configure CMake
run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -G Ninja ${{ matrix.extra-cmake-flags }}
# Build with a nice ninja status line
- name: Build
working-directory: ${{github.workspace}}/build
run: NINJA_STATUS="%p [%f:%s/%t] %o/s, %es" ninja
# If the tests are enabled for this job, run them
- name: Test
if: ${{ matrix.build-only != 'yes' }}
working-directory: ${{github.workspace}}/build
run: ctest --output-on-failure -j 4
- name: Selfhost
if: ${{ matrix.self-host }}
working-directory: ${{github.workspace}}/build
run: |
sudo cp libsnmallocshim.so libsnmallocshim-16mib.so libsnmallocshim-oe.so /usr/local/lib/
ninja clean
LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja
ninja clean
LD_PRELOAD=/usr/local/lib/libsnmallocshim-16mib.so ninja
ninja clean
LD_PRELOAD=/usr/local/lib/libsnmallocshim-oe.so ninja
qemu-crossbuild:
strategy:
matrix:
build-type: [ Release, Debug ]
arch: [ arm64, armhf ]
include:
- arch: armhf
system-processor: arm
triple: arm-linux-gnueabihf
rtld: ld-linux-armhf.so.3
- arch: arm64
system-processor: aarch64
triple: aarch64-linux-gnu
rtld: ld-linux-aarch64.so.1
# Don't abort runners if a single one fails
fail-fast: false
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install cross-compile toolchain and QEMU
# Install the dependencies and clang 13. Earlier versions of clang don't
# find the multilib things for this week's Ubuntu filesystem layout.
run: |
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 13
sudo apt install libstdc++-9-dev-${{ matrix.arch }}-cross qemu-user ninja-build lld-13 clang-13
- name: Configure
run: >
RTLD_NAME=${{ matrix.rtld }}
AARCH=${{ matrix.arch }}
TRIPLE=${{ matrix.triple}}
cmake
-B ${{github.workspace}}/build
-DCMAKE_BUILD_TYPE=${{matrix.build-type}}
-G Ninja
-DSNMALLOC_CI_BUILD=ON
-DSNMALLOC_QEMU_WORKAROUND=ON
-DSNMALLOC_STATIC_LIBRARY=OFF
-DCMAKE_TOOLCHAIN_FILE=ci/Toolchain.cmake
- name: Build
working-directory: ${{github.workspace}}/build
run: NINJA_STATUS="%p [%f:%s/%t] %o/s, %es" ninja
# Run the tests, skipping the -malloc and perf- tests (perf doesn't make
# sense in an emulator and the pass-through malloc is slightly flaky in
# QEMU)
- name: Test
working-directory: ${{github.workspace}}/build
run: ctest --output-on-failure -E '(perf-.*)|(.*-malloc$)'
timeout-minutes: 30
windows:
strategy:
matrix:
# Build each combination of OS and release/debug variants
os: [ windows-2016, windows-2019 ]
build-type: [ Release, Debug ]
arch: [ Win32, x64 ]
toolchain: [ "", "-T ClangCL" ]
extra-cmake-flags: [ "" ]
# The ClangCL toolchain was added in Visual Studio 2019, the Windows
# 2016 runners have only VS 2017, so skip them for this configuration
exclude:
- os: windows-2016
toolchain: "-T ClangCL"
# Add an extra check for the Windows 8 compatible PAL
include:
- os: windows-2016
build-type: Release
arch: x64
toolchain: ""
extra-cmake-flags: -DWIN8COMPAT=TRUE
variant: Windows 8 compatible
# Don't abort runners if a single one fails
fail-fast: false
runs-on: ${{ matrix.os }}
name: ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.build-type }} ${{ matrix.toolchain }} ${{ matrix.variant }}
steps:
- uses: actions/checkout@v2
- name: Configure CMake
run: |
cmake -B ${{github.workspace}}/build -A ${{ matrix.arch }} ${{ matrix.toolchain }} ${{ matrix.extra-cmake-flags }} -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On
# Build with whatever version of msbuild was configured, for a single build configuration.
- name: Build
run: cmake --build ${{github.workspace}}/build -- /m /p:Configuration=${{ matrix.build-type }}
# Run the tests.
- name: Test
working-directory: ${{ github.workspace }}/build
run: ctest -j 2 --interactive-debug-mode 0 --output-on-failure -C ${{ matrix.build-type }}
# Job to run clang-format and report errors
format:
runs-on: ubuntu-18.04
# We don't need to do the build for this job, but we need to configure it to get the clang-format target
steps:
- uses: actions/checkout@v2
- name: Configure CMake
run: cmake -B ${{github.workspace}}/build
- name: Install clang-tidy
run: sudo apt install clang-tidy-9
# Run the clang-format check and error if it generates a diff
- name: Run clang-format
working-directory: ${{github.workspace}}/build
run: |
set -eo pipefail
make clangformat
git diff --exit-code
- name: Run clang-tidy
run: |
clang-tidy-9 src/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0
if [ -f tidy.fail ] ; then
cat tidy.fail
exit 1
fi
all-checks:
needs: [unixlike, qemu-crossbuild, windows, format]
runs-on: ubuntu-latest
steps:
- name: Dummy step
run: true

View File

@@ -34,7 +34,7 @@ Comprehensive details about snmalloc's design can be found in the
current implementation are [described here](difference.md).
Since writing the paper, the performance of snmalloc has improved considerably.
[![Build Status](https://dev.azure.com/snmalloc/snmalloc/_apis/build/status/Microsoft.snmalloc?branchName=master)](https://dev.azure.com/snmalloc/snmalloc/_build/latest?definitionId=1?branchName=master)
[![snmalloc CI](https://github.com/microsoft/snmalloc/actions/workflows/main.yml/badge.svg)](https://github.com/microsoft/snmalloc/actions/workflows/main.yml)
# Further documentation

View File

@@ -1,347 +0,0 @@
trigger:
- master
pr:
- master
jobs:
- job:
displayName: Linux
pool:
vmImage: 'ubuntu-18.04'
strategy:
matrix:
64-bit Clang-7 Debug:
CC: clang-7
CXX: clang++-7
BuildType: Debug
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
64-bit Clang-7 Release:
CC: clang-7
CXX: clang++-7
BuildType: Release
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
64-bit Clang-8 Debug:
CC: clang-8
CXX: clang++-8
BuildType: Debug
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
64-bit Clang-8 Release:
CC: clang-8
CXX: clang++-8
BuildType: Release
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
64-bit Clang-9 Debug:
CC: clang-9
CXX: clang++-9
BuildType: Debug
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
64-bit Clang-10 Debug C++20:
CC: clang-10
CXX: clang++-10
BuildType: Debug
SelfHost: false
CMakeArgs: '-DSNMALLOC_USE_CXX20=On'
Image: snmallocciteam/build_linux_x64:latest
64-bit Clang-9 Release:
CC: clang-9
CXX: clang++-9
BuildType: Release
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
64-bit GCC-8 Debug:
CC: gcc-8
CXX: g++-8
BuildType: Debug
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
64-bit GCC-8 Release:
CC: gcc-8
CXX: g++-8
BuildType: Release
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
64-bit Self Host:
CC: clang-7
CXX: clang++-7
BuildType: Debug
SelfHost: true
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
32-bit Clang-9 Debug:
CC: clang-9
CXX: clang++-9
BuildType: Debug
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x86:latest
32-bit Clang-9 Release:
CC: clang-9
CXX: clang++-9
BuildType: Release
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x86:latest
container: $[ variables['Image'] ]
steps:
- script: |
set -eo pipefail
ci/scripts/build.sh
env:
CC: $(CC)
CXX: $(CXX)
BUILD_TYPE: $(BuildType)
CMAKE_ARGS: $(CMakeArgs)
displayName: 'Build'
- script: |
set -eo pipefail
ci/scripts/test.sh
env:
SELF_HOST: $(SelfHost)
BUILD_TYPE: $(BuildType)
displayName: 'Test'
- job:
displayName: 'ARM Linux'
pool:
vmImage: 'ubuntu-18.04'
strategy:
matrix:
64-bit Clang-9 Debug:
CC: clang-9
CXX: clang++-9
BuildType: Debug
SelfHost: false
CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On'
Image: snmallocciteam/build_linux_arm64:latest
64-bit Clang-9 Release:
CC: clang-9
CXX: clang++-9
BuildType: Release
SelfHost: false
CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On'
Image: snmallocciteam/build_linux_arm64:latest
32-bit Clang-9 Debug:
CC: clang-9
CXX: clang++-9
BuildType: Debug
SelfHost: false
CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On'
Image: snmallocciteam/build_linux_armhf:latest
32-bit Clang-9 Release:
CC: clang-9
CXX: clang++-9
BuildType: Release
SelfHost: false
CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On'
Image: snmallocciteam/build_linux_armhf:latest
steps:
- script: |
sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset
displayName: 'Enable QEmu'
- script: |
sudo docker run -v $PWD:/src \
-e CC=$(CC) \
-e CXX=$(CXX) \
-e BUILD_TYPE=$(BuildType) \
-e CMAKE_ARGS=$(CMakeArgs) \
$(Image) \
ci/scripts/build.sh
displayName: 'Build'
- script: |
sudo docker run -v $PWD:/src \
-e BUILD_TYPE=$(BuildType) \
-e SELF_HOST=$(SelfHost) \
$(Image) \
ci/scripts/test.sh
displayName: 'Test'
- job:
displayName: Windows
pool:
vmImage: 'vs2017-win2016'
strategy:
matrix:
64-bit Debug:
BuildType: Debug
CMakeArgs: '-G"Visual Studio 15 2017 Win64"'
JFlag: -j 4
64-bit Release:
BuildType: Release
CMakeArgs: '-G"Visual Studio 15 2017 Win64"'
JFlag: '-j 2'
64-bit Release Windows8Compat:
BuildType: Release
CMakeArgs: '-G"Visual Studio 15 2017 Win64" -DWIN8COMPAT=TRUE'
JFlag: '-j 2'
32-bit Debug:
BuildType: Debug
CMakeArgs: '-G"Visual Studio 15 2017"'
JFlag: '-j 4'
32-bit Release:
BuildType: Release
CMakeArgs: '-G"Visual Studio 15 2017"'
JFlag: '-j 2'
steps:
- script: |
bash -c "cat /proc/cpuinfo"
bash -c "cat /proc/meminfo"
displayName: 'Machine stats'
- task: CMake@1
displayName: 'CMake .. $(CMakeArgs) -DCMAKE_BUILD_TYPE=$(BuildType) -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On'
inputs:
cmakeArgs: '.. $(CMakeArgs) -DCMAKE_BUILD_TYPE=$(BuildType) -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On'
- task: MSBuild@1
displayName: 'Build solution build/snmalloc.sln'
inputs:
solution: build/snmalloc.sln
msbuildArguments: '/m /p:Configuration=$(BuildType)'
- script: 'ctest $(JFlag) --interactive-debug-mode 0 --output-on-failure -C $(BuildType)'
workingDirectory: build
displayName: 'Run Ctest'
- job:
displayName: WinClang
pool:
vmImage: 'windows-2019'
strategy:
matrix:
64-bit Debug Clang:
BuildType: Debug
CMakeArgs: '-GNinja -DCMAKE_C_COMPILER="C:/Program Files/LLVM/bin/clang-cl.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/LLVM/bin/clang-cl.exe" -DCMAKE_RC_COMPILER="C:/Program Files/LLVM/bin/llvm-rc"'
JFlag: -j 4
64-bit Release Clang:
BuildType: Release
CMakeArgs: '-GNinja -DCMAKE_C_COMPILER="C:/Program Files/LLVM/bin/clang-cl.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/LLVM/bin/clang-cl.exe" -DCMAKE_RC_COMPILER="C:/Program Files/LLVM/bin/llvm-rc"'
JFlag: -j 4
steps:
- script: |
bash -c "cat /proc/cpuinfo"
bash -c "cat /proc/meminfo"
displayName: 'Machine stats'
- script: |
choco install --accept-license -y Ninja llvm
displayName: 'Dependencies'
- script: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
mkdir build
cd build
cmake .. $(CMakeArgs) -DCMAKE_BUILD_TYPE=$(BuildType) -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On
cmake --build . --config ${BuildType}
displayName: 'build'
- script: |
set PATH=%PATH%;"C:\Program Files\LLVM\bin\"
func-statistics-1.exe
ctest $(JFlag) --interactive-debug-mode 0 --output-on-failure -C $(BuildType)
workingDirectory: build
displayName: 'Run Ctest'
- job:
displayName: macOS
pool:
vmImage: 'macOS-10.15'
strategy:
matrix:
Debug:
BuildType: Debug
Release:
BuildType: Release
steps:
- task: CMake@1
displayName: 'CMake .. -DCMAKE_BUILD_TYPE=$(BuildType) -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On'
inputs:
cmakeArgs: '.. -DCMAKE_BUILD_TYPE=$(BuildType) -DSNMALLOC_CI_BUILD=On -DSNMALLOC_RUST_SUPPORT=On'
- script: |
set -eo pipefail
make -j 4
workingDirectory: build
failOnStderr: true
displayName: 'Compile'
- script: |
set -eo pipefail
ctest -j 4 --output-on-failure -C $(BuildType)
workingDirectory: build
failOnStderr: true
displayName: 'Test'
- job:
displayName: Format
pool:
vmImage: 'ubuntu-18.04'
container: snmallocciteam/build_linux_x64:latest
steps:
- script: |
mkdir build
cd build
cmake -GNinja ..
env:
CC: /usr/bin/gcc-8
CXX: /usr/bin/g++-8
- script: |
set -eo pipefail
ninja clangformat
git diff --exit-code
workingDirectory: build
failOnStderr: true
displayName: 'Clang-Format'
- script: |
set -eo pipefail
clang-tidy-9 src/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0
displayName: 'Clang-Tidy'

13
ci/Toolchain.cmake Normal file
View File

@@ -0,0 +1,13 @@
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR $ENV{ARCH})
set(triple $ENV{TRIPLE})
set(CMAKE_C_COMPILER clang-13)
set(CMAKE_C_COMPILER_TARGET ${triple})
set(CMAKE_CXX_COMPILER clang++-13)
set(CMAKE_CXX_COMPILER_TARGET ${triple})
set(CROSS_LINKER_FLAGS "-fuse-ld=lld -Wl,--dynamic-linker=/usr/${triple}/lib/$ENV{RTLD_NAME},-rpath,/usr/${triple}/lib")
set(CMAKE_EXE_LINKER_FLAGS ${CROSS_LINKER_FLAGS})
set(CMAKE_SHARED_LINKER_FLAGS ${CROSS_LINKER_FLAGS})
set(CMAKE_MODULE_LINKER_FLAGS ${CROSS_LINKER_FLAGS})

View File

@@ -1,7 +0,0 @@
FROM multiarch/ubuntu-core:arm64-bionic
WORKDIR /src
RUN apt update \
&& apt install --no-install-recommends -y ninja-build clang++-9 cmake \
&& apt -y clean

View File

@@ -1,7 +0,0 @@
FROM multiarch/ubuntu-core:armhf-bionic
WORKDIR /src
RUN apt update \
&& apt install --no-install-recommends -y ninja-build clang++-9 cmake \
&& apt -y clean

View File

@@ -1,17 +0,0 @@
FROM ubuntu:20.04
RUN apt update \
&& apt install -y software-properties-common \
&& add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt update \
&& apt install --no-install-recommends -y sudo cmake ninja-build git \
&& apt install --no-install-recommends -y g++-8 \
&& apt install --no-install-recommends -y clang-7 \
&& apt install --no-install-recommends -y clang-8 \
&& apt install --no-install-recommends -y clang-9 \
&& apt install --no-install-recommends -y clang-10 \
&& apt install --no-install-recommends -y clang-format-9 clang-tidy-9 \
&& update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-9 100 \
&& apt remove -y software-properties-common \
&& apt -y autoremove \
&& apt -y clean

View File

@@ -1,7 +0,0 @@
FROM multiarch/ubuntu-core:x86-bionic
WORKDIR /src
RUN apt update \
&& apt install --no-install-recommends -y ninja-build clang++-9 cmake \
&& apt -y clean

View File

@@ -1,19 +0,0 @@
#!/bin/bash
mkdir build
cd build
if [[ "$CXX" == clang++* ]]; then
CMAKE_CXX_FLAGS+=" -stdlib=libstdc++";
fi
cmake \
$CMAKE_ARGS \
-GNinja \
-DSNMALLOC_CI_BUILD=ON \
-DSNMALLOC_RUST_SUPPORT=ON \
-DSNMALLOC_QEMU_WORKAROUND=ON \
-DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" \
${SNMALLOC_SRC-..}
ninja

View File

@@ -1,14 +0,0 @@
#!/bin/bash
cd build
if [ $SELF_HOST = false ]; then
ctest -j 4 --output-on-failure -C $BUILD_TYPE
else
sudo cp libsnmallocshim.so libsnmallocshim-16mib.so libsnmallocshim-oe.so /usr/local/lib/
ninja clean
LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja
ninja clean
LD_PRELOAD=/usr/local/lib/libsnmallocshim-16mib.so ninja
ninja clean
LD_PRELOAD=/usr/local/lib/libsnmallocshim-oe.so ninja
fi

View File

@@ -48,6 +48,7 @@ namespace snmalloc
constexpr T one_at_bit(S shift)
{
static_assert(std::is_integral_v<T>, "Type must be integral");
SNMALLOC_ASSERT(sizeof(T) * 8 > static_cast<size_t>(shift));
return (static_cast<T>(1)) << shift;
}

View File

@@ -24,8 +24,9 @@ int main(int argc, char** argv)
setup();
constexpr int bits_to_test = bits::is64() ? 36 : 31;
T value = 0;
for (uintptr_t ptr = 0; ptr < bits::one_at_bit(36);
for (uintptr_t ptr = 0; ptr < bits::one_at_bit(bits_to_test);
ptr += bits::one_at_bit(GRANULARITY_BITS + 3))
{
pagemap_test.set(ptr, value);
@@ -38,7 +39,7 @@ int main(int argc, char** argv)
std::cout << std::endl;
value = 0;
for (uintptr_t ptr = 0; ptr < bits::one_at_bit(36);
for (uintptr_t ptr = 0; ptr < bits::one_at_bit(bits_to_test);
ptr += bits::one_at_bit(GRANULARITY_BITS + 3))
{
T result = pagemap_test.get(ptr);