Benchmarking automatically (#795)

Co-authored-by: Amaury Chamayou <amaury@xargs.fr>

.github/workflows/benchmark.yml

@@ -0,0 +1,72 @@
+name: snmalloc Benchmarking CI
+
+# The following should ensure that the workflow only runs a single set of actions
+# for each PR. But it will not apply this to pushes to the main branch.
+concurrency:
+  group: benchmarking${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Controls when the workflow will run
+on:
+  schedule:
+    - cron: "0 0 * * 1"  # Runs every Monday at midnight UTC
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    runs-on: [self-hosted, 1ES.Pool=snmalloc-perf]
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      # Setup docker buildx
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and run benchmarks in Docker
+        id: docker_build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: benchmark/Dockerfile
+          push: false
+          load: true
+          tags: snmalloc-bench
+          build-args: |
+            benchs=allt
+            repeats=5
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Extracts the benchmark results from the Docker container
+      - name: Extract Benchmark Results
+        run: |
+          docker cp `docker run -d ${{ steps.docker_build.outputs.imageid }}`:/bencher.dev.sn.json .
+
+      # Uploads the benchmark results as an artifact
+      - name: Upload Benchmark Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: bencher.dev.sn.json
+
+      # Upload to graphing service
+      - uses: bencherdev/bencher@main
+      - name: Upload benchmark results to Bencher
+        run: |
+          bencher run \
+          --project snmalloc \
+          --token '${{ secrets.BENCHER_DEV_API_TOKEN }}' \
+          --branch main \
+          --adapter json \
+          --err \
+          --file bencher.dev.sn.json

benchmark/Dockerfile

@@ -0,0 +1,45 @@
+FROM ubuntu:24.04
+
+# Pull mimalloc-bench
+RUN apt-get update && apt-get install -y --no-install-recommends git gpg ca-certificates python3-numpy
+RUN git clone https://github.com/daanx/mimalloc-bench &&\
+    cd mimalloc-bench && \
+    git reset --hard a4ce904286365c7adfba54f0eea3a2df3fc95bd1
+
+WORKDIR /mimalloc-bench
+# Install dependencies
+RUN ./build-bench-env.sh packages
+
+# Tidy up apt cache
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Build benchmarks
+RUN ./build-bench-env.sh bench
+
+RUN ./build-bench-env.sh redis
+
+RUN ./build-bench-env.sh rocksdb \
+  && find /mimalloc-bench/extern/rocksdb-8.1.1 -name "*.o" -delete
+
+RUN ./build-bench-env.sh lean \
+  && find /mimalloc-bench/extern/lean -name "*.o" -delete
+
+RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt
+
+# Build allocator
+RUN mkdir -p /snmalloc
+COPY . /snmalloc
+
+RUN mkdir -p /snmalloc/build
+WORKDIR /snmalloc/build
+RUN cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
+RUN ninja libsnmallocshim.so
+
+# Run benchmarks
+ARG benchs=allt
+ARG repeats=1
+WORKDIR /mimalloc-bench/out/bench
+RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats
+
+WORKDIR /
+RUN python3 /mimalloc-bench/scripts/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv

src/test/func/jemalloc/jemalloc.cc

@@ -1,65 +1,71 @@
-#include <functional>
-#include <limits.h>
-#include <stdio.h>
-#include <test/helpers.h>
-#include <test/setup.h>
+#if defined(SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION)
+int main()
+{
+  return 0;
+}
+#else
+#  include <functional>
+#  include <limits.h>
+#  include <stdio.h>
+#  include <test/helpers.h>
+#  include <test/setup.h>
 
-#define SNMALLOC_NAME_MANGLE(a) our_##a
-#undef SNMALLOC_NO_REALLOCARRAY
-#undef SNMALLOC_NO_REALLOCARR
-#define SNMALLOC_BOOTSTRAP_ALLOCATOR
-#define SNMALLOC_JEMALLOC3_EXPERIMENTAL
-#define SNMALLOC_JEMALLOC_NONSTANDARD
-#include <snmalloc/override/jemalloc_compat.cc>
-#include <snmalloc/override/malloc.cc>
+#  define SNMALLOC_NAME_MANGLE(a) our_##a
+#  undef SNMALLOC_NO_REALLOCARRAY
+#  undef SNMALLOC_NO_REALLOCARR
+#  define SNMALLOC_BOOTSTRAP_ALLOCATOR
+#  define SNMALLOC_JEMALLOC3_EXPERIMENTAL
+#  define SNMALLOC_JEMALLOC_NONSTANDARD
+#  include <snmalloc/override/jemalloc_compat.cc>
+#  include <snmalloc/override/malloc.cc>
 
-#if __has_include(<malloc_np.h>)
-#  include <malloc_np.h>
-#endif
+#  if __has_include(<malloc_np.h>)
+#    include <malloc_np.h>
+#  endif
 
-#ifdef __FreeBSD__
+#  ifdef __FreeBSD__
 /**
  * Enable testing against the versions that we get from libc or elsewhere.
  * Enabled by default on FreeBSD where all of the jemalloc functions are
  * exported from libc.
  */
-#  define TEST_JEMALLOC_MALLOCX
-#endif
+#    define TEST_JEMALLOC_MALLOCX
+#  endif
 
-#define OUR_MALLOCX_LG_ALIGN(la) (static_cast<int>(la))
-#define OUR_MALLOCX_ZERO (one_at_bit<int>(6))
+#  define OUR_MALLOCX_LG_ALIGN(la) (static_cast<int>(la))
+#  define OUR_MALLOCX_ZERO (one_at_bit<int>(6))
 
-#define OUR_ALLOCM_NO_MOVE (one_at_bit<int>(7))
+#  define OUR_ALLOCM_NO_MOVE (one_at_bit<int>(7))
 
-#define OUR_ALLOCM_SUCCESS 0
-#define OUR_ALLOCM_ERR_OOM 1
-#define OUR_ALLOCM_ERR_NOT_MOVED 2
+#  define OUR_ALLOCM_SUCCESS 0
+#  define OUR_ALLOCM_ERR_OOM 1
+#  define OUR_ALLOCM_ERR_NOT_MOVED 2
 
-#ifndef MALLOCX_LG_ALIGN
-#  define MALLOCX_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la)
-#endif
-#ifndef MALLOCX_ZERO
-#  define MALLOCX_ZERO OUR_MALLOCX_ZERO
-#endif
+#  ifndef MALLOCX_LG_ALIGN
+#    define MALLOCX_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la)
+#  endif
+#  ifndef MALLOCX_ZERO
+#    define MALLOCX_ZERO OUR_MALLOCX_ZERO
+#  endif
 
-#ifndef ALLOCM_LG_ALIGN
-#  define ALLOCM_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la)
-#endif
-#ifndef ALLOCM_ZERO
-#  define ALLOCM_ZERO OUR_MALLOCX_ZERO
-#endif
-#ifndef ALLOCM_NO_MOVE
-#  define ALLOCM_NO_MOVE OUR_ALLOCM_NO_MOVE
-#endif
-#ifndef ALLOCM_SUCCESS
-#  define ALLOCM_SUCCESS OUR_ALLOCM_SUCCESS
-#endif
-#ifndef ALLOCM_ERR_OOM
-#  define ALLOCM_ERR_OOM OUR_ALLOCM_ERR_OOM
-#endif
-#ifndef ALLOCM_ERR_NOT_MOVED
-#  define ALLOCM_ERR_NOT_MOVED OUR_ALLOCM_ERR_NOT_MOVED
-#endif
+#  ifndef ALLOCM_LG_ALIGN
+#    define ALLOCM_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la)
+#  endif
+#  ifndef ALLOCM_ZERO
+#    define ALLOCM_ZERO OUR_MALLOCX_ZERO
+#  endif
+#  ifndef ALLOCM_NO_MOVE
+#    define ALLOCM_NO_MOVE OUR_ALLOCM_NO_MOVE
+#  endif
+#  ifndef ALLOCM_SUCCESS
+#    define ALLOCM_SUCCESS OUR_ALLOCM_SUCCESS
+#  endif
+#  ifndef ALLOCM_ERR_OOM
+#    define ALLOCM_ERR_OOM OUR_ALLOCM_ERR_OOM
+#  endif
+#  ifndef ALLOCM_ERR_NOT_MOVED
+#    define ALLOCM_ERR_NOT_MOVED OUR_ALLOCM_ERR_NOT_MOVED
+#  endif
 
 using namespace snmalloc;
 using namespace snmalloc::bits;
@@ -335,21 +341,22 @@ int main()
     our_dallocm,
     our_nallocm>();
 
-#ifndef __PIC__
+#  ifndef __PIC__
   void* bootstrap = __je_bootstrap_malloc(42);
   if (bootstrap == nullptr)
   {
     printf("Failed to allocate from bootstrap malloc\n");
   }
   __je_bootstrap_free(bootstrap);
-#endif
+#  endif
 
   // These tests are for jemalloc compatibility and so should work with
   // jemalloc's implementation of these functions.  If TEST_JEMALLOC is
   // defined then we try
-#ifdef TEST_JEMALLOC_MALLOCX
+#  ifdef TEST_JEMALLOC_MALLOCX
   test_size<mallocx, dallocx, sallocx, nallocx>();
   test_zeroing<mallocx, dallocx, rallocx>();
   test_xallocx<mallocx, dallocx, xallocx>();
-#endif
+#  endif
 }
+#endif // SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION