Merge pull request #7207 from libgit2/ethomson/oidbench

Benchmark the OID code
2026-06-22 06:26:26 +00:00 · 2026-01-26 23:41:43 +00:00
parent 77028ee987 9935da4661
commit b1a826a7ff
90 changed files with 1653 additions and 441 deletions
--- a/.github/workflows/ab-perf.yml
+++ b/.github/workflows/ab-perf.yml
@@ -0,0 +1,229 @@
+# A/B testing with benchmarks to compare a control branch (main) to a
+# candidate branch (a pull request).
+name: A/B Performance Test
+
+on:
+  workflow_dispatch:
+
+env:
+  docker-registry: ghcr.io
+  docker-config-path: ci/docker
+
+jobs:
+  # Run our CI/CD builds.  We build a matrix with the various build targets
+  # and their details.  Then we build either in a docker container (Linux)
+  # or on the actual hosts (macOS, Windows).
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+        # All builds: core platforms
+        - name: "Linux (Noble, GCC, OpenSSL, libssh2)"
+          id: noble-gcc-openssl
+          os: ubuntu-latest
+          container:
+            name: noble
+          env:
+            CC: gcc
+            CMAKE_GENERATOR: Ninja
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+        - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)"
+          id: noble-clang-mbedtls
+          os: ubuntu-latest
+          container:
+            name: noble
+          env:
+            CC: clang
+            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            CMAKE_GENERATOR: Ninja
+        - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)"
+          id: xenial-gcc-openssl
+          os: ubuntu-latest
+          container:
+            name: xenial
+          env:
+            CC: gcc
+            CMAKE_GENERATOR: Ninja
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+        - name: "Linux (Xenial, Clang, mbedTLS, libssh2)"
+          id: xenial-gcc-mbedtls
+          os: ubuntu-latest
+          container:
+            name: xenial
+          env:
+            CC: clang
+            CMAKE_GENERATOR: Ninja
+            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+        - name: "macOS"
+          id: macos
+          os: macos-14
+          setup-script: osx
+          env:
+            CC: clang
+            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            CMAKE_GENERATOR: Ninja
+            PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+        - name: "Windows (amd64, Visual Studio, Schannel)"
+          id: windows-amd64-vs
+          os: windows-2022
+          setup-script: win32
+          build_prefix: RelWithDebInfo
+          env:
+            ARCH: amd64
+            CMAKE_GENERATOR: Visual Studio 17 2022
+            CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
+            BUILD_TEMP: D:\Temp
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+        - name: "Windows (x86, Visual Studio, WinHTTP)"
+          id: windows-x86-vs
+          os: windows-2022
+          setup-script: win32
+          build_prefix: RelWithDebInfo
+          env:
+            ARCH: x86
+            CMAKE_GENERATOR: Visual Studio 17 2022
+            CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
+            BUILD_TEMP: D:\Temp
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+        - name: "Windows (amd64, mingw, WinHTTP)"
+          id: windows-amd64-mingw
+          os: windows-2022
+          setup-script: mingw
+          env:
+            ARCH: amd64
+            CMAKE_GENERATOR: MinGW Makefiles
+            CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            BUILD_TEMP: D:\Temp
+            BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+        - name: "Windows (x86, mingw, Schannel)"
+          id: windows-x86-mingw
+          os: windows-2022
+          setup-script: mingw
+          env:
+            ARCH: x86
+            CMAKE_GENERATOR: MinGW Makefiles
+            CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            BUILD_TEMP: D:\Temp
+            BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+    env: ${{ matrix.platform.env }}
+    runs-on: ${{ matrix.platform.os }}
+    name: "A/B: ${{ matrix.platform.name }}"
+    steps:
+    - name: Check out control
+      uses: actions/checkout@v4
+      with:
+        path: source/control
+        ref: main
+    - name: Check out candidate
+      uses: actions/checkout@v4
+      with:
+        path: source/candidate
+    - name: Set up build environment
+      run: source/candidate/ci/setup-${{ matrix.platform.setup-script }}-build.sh
+      shell: bash
+      if: matrix.platform.setup-script != ''
+    - name: Setup QEMU
+      run: docker run --rm --privileged multiarch/qemu-user-static:register --reset
+      if: matrix.platform.container.qemu == true
+    - name: Set up container
+      uses: ./source/candidate/.github/actions/download-or-build-container
+      with:
+        registry: ${{ env.docker-registry }}
+        config-path: ${{ env.docker-config-path }}
+        container: ${{ matrix.platform.container.name }}
+        github_token: ${{ secrets.github_token }}
+        dockerfile: ${{ matrix.platform.container.dockerfile }}
+      if: matrix.platform.container.name != ''
+    - name: Prepare builds
+      run: |
+        mkdir build
+        mkdir build/control
+        mkdir build/candidate
+    - name: Build control
+      uses: ./source/control/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/control && ../../source/control/ci/build.sh
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+    - name: Build candidate
+      uses: ./source/candidate/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ../../source/candidate/ci/build.sh
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+    - name: Run control benchmarks
+      uses: ./source/control/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/control && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json )
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+    - name: Run candidate benchmarks
+      uses: ./source/candidate/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json )
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+    - name: Organize results
+      run: |
+        mkdir results
+        mv ${BUILD_WORKSPACE:-.}/build/control/results.json ./results/control.json
+        mv ${BUILD_WORKSPACE:-.}/build/candidate/results.json ./results/candidate.json
+    - name: Upload results
+      uses: actions/upload-artifact@v4
+      with:
+        name: results-${{ matrix.platform.id }}
+        path: results
+      if: always()
+
+  # Publish the results
+  publish:
+    name: Publish results
+    needs: [ build ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+    - name: Download test results
+      uses: actions/download-artifact@v4
+      with:
+        path: results
+    - name: Generate markdown
+      run: node ci/compare-benchmarks.js results results.md
+    - name: Update pull request
+      uses: actions/github-script@v7
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const fs = require('fs');
+          const markdown = fs.readFileSync('results.md');
+          github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: markdown
+          });
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -115,7 +115,7 @@ jobs:
        fi

        mkdir benchmark && cd benchmark
-        ../source/tests/benchmarks/benchmark.sh \
+        ../source/benchmarks/cli/benchmark.sh \
            ${SUITE_FLAG} ${DEBUG_FLAG} \
            --admin \
            --baseline-cli "git" --cli "${GIT2_CLI}" --name libgit2 \
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -1,28 +1,21 @@
-# Continuous integration and pull request validation builds for the
-# main and maintenance branches.
-name: CI Build
+# A/B testing with benchmarks to compare a control branch (main) to a
+# candidate branch (a pull request).
+name: A/B Performance Test

 on:
-  push:
-    branches: [ main, maint/* ]
-  pull_request:
-    branches: [ main, maint/* ]
  workflow_dispatch:

 env:
  docker-registry: ghcr.io
  docker-config-path: ci/docker

-permissions:
-  contents: write
-  packages: write
-
 jobs:
  # Run our CI/CD builds.  We build a matrix with the various build targets
  # and their details.  Then we build either in a docker container (Linux)
  # or on the actual hosts (macOS, Windows).
  build:
    strategy:
+      fail-fast: false
      matrix:
        platform:
        # All builds: core platforms
@@ -34,7 +27,8 @@ jobs:
          env:
            CC: gcc
            CMAKE_GENERATOR: Ninja
-            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
        - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)"
          id: noble-clang-mbedtls
          os: ubuntu-latest
@@ -42,167 +36,118 @@ jobs:
            name: noble
          env:
            CC: clang
-            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser
+            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
            CMAKE_GENERATOR: Ninja
-        - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)"
-          id: xenial-gcc-openssl
-          os: ubuntu-latest
-          container:
-            name: xenial
-          env:
-            CC: gcc
-            CMAKE_GENERATOR: Ninja
-            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON
-        - name: "Linux (Xenial, Clang, mbedTLS, libssh2)"
-          id: xenial-gcc-mbedtls
-          os: ubuntu-latest
-          container:
-            name: xenial
-          env:
-            CC: clang
-            CMAKE_GENERATOR: Ninja
-            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2
+#        - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)"
+#          id: xenial-gcc-openssl
+#          os: ubuntu-latest
+#          container:
+#            name: xenial
+#          env:
+#            CC: gcc
+#            CMAKE_GENERATOR: Ninja
+#            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+#            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+#        - name: "Linux (Xenial, Clang, mbedTLS, libssh2)"
+#          id: xenial-gcc-mbedtls
+#          os: ubuntu-latest
+#          container:
+#            name: xenial
+#          env:
+#            CC: clang
+#            CMAKE_GENERATOR: Ninja
+#            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+#            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
        - name: "macOS"
          id: macos
          os: macos-14
          setup-script: osx
          env:
            CC: clang
-            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON
+            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
            CMAKE_GENERATOR: Ninja
            PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
            SKIP_SSH_TESTS: true
            SKIP_NEGOTIATE_TESTS: true
-        - name: "Windows (amd64, Visual Studio, Schannel)"
-          id: windows-amd64-vs
-          os: windows-2022
-          setup-script: win32
-          env:
-            ARCH: amd64
-            CMAKE_GENERATOR: Visual Studio 17 2022
-            CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2
-            BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
-            BUILD_TEMP: D:\Temp
-            SKIP_SSH_TESTS: true
-            SKIP_NEGOTIATE_TESTS: true
-        - name: "Windows (x86, Visual Studio, WinHTTP)"
-          id: windows-x86-vs
-          os: windows-2022
-          setup-script: win32
-          env:
-            ARCH: x86
-            CMAKE_GENERATOR: Visual Studio 17 2022
-            CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2
-            BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
-            BUILD_TEMP: D:\Temp
-            SKIP_SSH_TESTS: true
-            SKIP_NEGOTIATE_TESTS: true
-        - name: "Windows (amd64, mingw, WinHTTP)"
-          id: windows-amd64-mingw
-          os: windows-2022
-          setup-script: mingw
-          env:
-            ARCH: amd64
-            CMAKE_GENERATOR: MinGW Makefiles
-            CMAKE_OPTIONS: -DDEPRECATE_HARD=ON
-            BUILD_TEMP: D:\Temp
-            BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
-            SKIP_SSH_TESTS: true
-            SKIP_NEGOTIATE_TESTS: true
-        - name: "Windows (x86, mingw, Schannel)"
-          id: windows-x86-mingw
-          os: windows-2022
-          setup-script: mingw
-          env:
-            ARCH: x86
-            CMAKE_GENERATOR: MinGW Makefiles
-            CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel
-            BUILD_TEMP: D:\Temp
-            BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
-            SKIP_SSH_TESTS: true
-            SKIP_NEGOTIATE_TESTS: true
-
-        # All builds: sanitizers
-        - name: "Sanitizer (Memory)"
-          id: sanitizer-memory
-          os: ubuntu-latest
-          setup-script: sanitizer
-          container:
-            name: noble
-          env:
-            CC: clang
-            CFLAGS: -fsanitize=memory -fsanitize-memory-track-origins=2 -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer
-            CMAKE_OPTIONS: -DCMAKE_C_EXTENSIONS=ON -DCMAKE_PREFIX_PATH=/usr/local/msan -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
-            CMAKE_GENERATOR: Ninja
-            SKIP_SSH_TESTS: true
-            SKIP_NEGOTIATE_TESTS: true
-            ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
-            UBSAN_OPTIONS: print_stacktrace=1
-        - name: "Sanitizer (Address)"
-          id: sanitizer-address
-          os: ubuntu-latest
-          setup-script: sanitizer
-          container:
-            name: noble
-          env:
-            CC: clang
-            CFLAGS: -fsanitize=address -ggdb -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer
-            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
-            CMAKE_GENERATOR: Ninja
-            SKIP_SSH_TESTS: true
-            SKIP_NEGOTIATE_TESTS: true
-            ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
-            UBSAN_OPTIONS: print_stacktrace=1
-        - name: "Sanitizer (UndefinedBehavior)"
-          id: sanitizer-ub
-          os: ubuntu-latest
-          setup-script: sanitizer
-          container:
-            name: noble
-          env:
-            CC: clang
-            CFLAGS: -fsanitize=undefined,nullability -fno-sanitize-recover=undefined,nullability -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer
-            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
-            CMAKE_GENERATOR: Ninja
-            SKIP_SSH_TESTS: true
-            SKIP_NEGOTIATE_TESTS: true
-            ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
-            UBSAN_OPTIONS: print_stacktrace=1
-        - name: "Sanitizer (Thread)"
-          id: sanitizer-thread
-          os: ubuntu-latest
-          setup-script: sanitizer
-          container:
-            name: noble
-          env:
-            CC: clang
-            CFLAGS: -fsanitize=thread -fno-optimize-sibling-calls -fno-omit-frame-pointer
-            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
-            CMAKE_GENERATOR: Ninja
-            SKIP_SSH_TESTS: true
-            SKIP_NEGOTIATE_TESTS: true
-            ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
-            UBSAN_OPTIONS: print_stacktrace=1
-            TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1
-      fail-fast: false
+#        - name: "Windows (amd64, Visual Studio, Schannel)"
+#          id: windows-amd64-vs
+#          os: windows-2022
+#          setup-script: win32
+#          build_prefix: RelWithDebInfo
+#          env:
+#            ARCH: amd64
+#            CMAKE_GENERATOR: Visual Studio 17 2022
+#            CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+#            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+#            BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
+#            BUILD_TEMP: D:\Temp
+#            SKIP_SSH_TESTS: true
+#            SKIP_NEGOTIATE_TESTS: true
+#        - name: "Windows (x86, Visual Studio, WinHTTP)"
+#          id: windows-x86-vs
+#          os: windows-2022
+#          setup-script: win32
+#          build_prefix: RelWithDebInfo
+#          env:
+#            ARCH: x86
+#            CMAKE_GENERATOR: Visual Studio 17 2022
+#            CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+#            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+#            BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
+#            BUILD_TEMP: D:\Temp
+#            SKIP_SSH_TESTS: true
+#            SKIP_NEGOTIATE_TESTS: true
+#        - name: "Windows (amd64, mingw, WinHTTP)"
+#          id: windows-amd64-mingw
+#          os: windows-2022
+#          setup-script: mingw
+#          env:
+#            ARCH: amd64
+#            CMAKE_GENERATOR: MinGW Makefiles
+#            CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+#            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+#            BUILD_TEMP: D:\Temp
+#            BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
+#            SKIP_SSH_TESTS: true
+#            SKIP_NEGOTIATE_TESTS: true
+#        - name: "Windows (x86, mingw, Schannel)"
+#          id: windows-x86-mingw
+#          os: windows-2022
+#          setup-script: mingw
+#          env:
+#            ARCH: x86
+#            CMAKE_GENERATOR: MinGW Makefiles
+#            CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+#            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+#            BUILD_TEMP: D:\Temp
+#            BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
+#            SKIP_SSH_TESTS: true
+#            SKIP_NEGOTIATE_TESTS: true
    env: ${{ matrix.platform.env }}
    runs-on: ${{ matrix.platform.os }}
-    name: "Build: ${{ matrix.platform.name }}"
+    name: "A/B: ${{ matrix.platform.name }}"
    steps:
-    - name: Check out repository
+    - name: Check out control
      uses: actions/checkout@v4
      with:
-        path: source
-        fetch-depth: 0
+        path: source/control
+# TODO: there are no control benchmarks in main yet
+#        ref: main
+    - name: Check out candidate
+      uses: actions/checkout@v4
+      with:
+        path: source/candidate
    - name: Set up build environment
-      run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh
+      run: source/candidate/ci/setup-${{ matrix.platform.setup-script }}-build.sh
      shell: bash
      if: matrix.platform.setup-script != ''
    - name: Setup QEMU
      run: docker run --rm --privileged multiarch/qemu-user-static:register --reset
      if: matrix.platform.container.qemu == true
    - name: Set up container
-      uses: ./source/.github/actions/download-or-build-container
+      uses: ./source/candidate/.github/actions/download-or-build-container
      with:
        registry: ${{ env.docker-registry }}
        config-path: ${{ env.docker-config-path }}
@@ -210,49 +155,76 @@ jobs:
        github_token: ${{ secrets.github_token }}
        dockerfile: ${{ matrix.platform.container.dockerfile }}
      if: matrix.platform.container.name != ''
-    - name: Prepare build
-      run: mkdir build
-    - name: Build
-      uses: ./source/.github/actions/run-build
-      with:
-        command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/build.sh
-        container: ${{ matrix.platform.container.name }}
-        container-version: ${{ env.docker-registry-container-sha }}
-        shell: ${{ matrix.platform.shell }}
-    - name: Test
-      uses: ./source/.github/actions/run-build
-      with:
-        command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/test.sh
-        container: ${{ matrix.platform.container.name }}
-        container-version: ${{ env.docker-registry-container-sha }}
-        shell: ${{ matrix.platform.shell }}
-    - name: Upload test results
-      uses: actions/upload-artifact@v4
-      if: success() || failure()
-      with:
-        name: test-results-${{ matrix.platform.id }}
-        path: build/results_*.xml
-
-  documentation:
-    name: Validate documentation
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out repository
-      uses: actions/checkout@v4
-    - name: Validate documentation
+    - name: Prepare builds
      run: |
-        (cd script/api-docs && npm install)
-        script/api-docs/api-generator.js --validate-only --strict --deprecate-hard  .
+        mkdir build
+        mkdir build/control
+        mkdir build/candidate
+    - name: Build control
+      uses: ./source/control/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/control && ../../source/control/ci/build.sh
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+    - name: Build candidate
+      uses: ./source/candidate/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ../../source/candidate/ci/build.sh
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+    - name: Run control benchmarks
+      uses: ./source/control/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/control && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json )
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+    - name: Run candidate benchmarks
+      uses: ./source/candidate/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json )
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+    - name: Organize results
+      run: |
+        mkdir results
+        mv ${BUILD_WORKSPACE:-.}/build/control/results.json ./results/control.json
+        mv ${BUILD_WORKSPACE:-.}/build/candidate/results.json ./results/candidate.json
+    - name: Upload results
+      uses: actions/upload-artifact@v4
+      with:
+        name: results-${{ matrix.platform.id }}
+        path: results
+      if: always()

-  test_results:
-    name: Test results
+  # Publish the results
+  publish:
+    name: Publish results
    needs: [ build ]
    if: always()
    runs-on: ubuntu-latest
    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
    - name: Download test results
      uses: actions/download-artifact@v4
-    - name: Generate test summary
-      uses: test-summary/action@v2
      with:
-        paths: 'test-results-*/*.xml'
+        path: results
+    - name: Generate markdown
+      run: node ci/compare-benchmarks.js results results.md
+    - name: Update pull request
+      uses: actions/github-script@v7
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const fs = require('fs');
+          const markdown = fs.readFileSync('results.md');
+          github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: markdown
+          });
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,7 +20,8 @@ option(EXPERIMENTAL_SHA256     "Enable experimental SHA256 support (for R&D/test

 # Optional subsystems
 option(BUILD_SHARED_LIBS       "Build Shared Library (OFF for Static)"                  ON)
-option(BUILD_TESTS             "Build Tests using the Clar suite"                       ON)
+option(BUILD_TESTS             "Build the test suite"                                   ON)
+option(BUILD_BENCHMARKS        "Build the benchmark suite"                             OFF)
 option(BUILD_CLI               "Build the command-line interface"                       ON)
 option(BUILD_EXAMPLES          "Build library usage example apps"                      OFF)
 option(BUILD_FUZZERS           "Build the fuzz targets"                                OFF)
@@ -112,6 +113,10 @@ if(BUILD_TESTS)
 	add_subdirectory(tests)
 endif()

+if(BUILD_BENCHMARKS)
+	add_subdirectory(benchmarks)
+endif()
+
 if(BUILD_EXAMPLES)
 	add_subdirectory(examples)
 endif()
--- a/benchmarks/CMakeLists.txt
+++ b/benchmarks/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(libgit2)
--- a/tests/benchmarks/README.md
+++ b/tests/benchmarks/README.md
--- a/tests/benchmarks/_script/flamegraph/README.md
+++ b/tests/benchmarks/_script/flamegraph/README.md
--- a/tests/benchmarks/_script/flamegraph/aix-perf.pl
+++ b/tests/benchmarks/_script/flamegraph/aix-perf.pl
--- a/tests/benchmarks/_script/flamegraph/difffolded.pl
+++ b/tests/benchmarks/_script/flamegraph/difffolded.pl
--- a/tests/benchmarks/_script/flamegraph/example-dtrace-stacks.txt
+++ b/tests/benchmarks/_script/flamegraph/example-dtrace-stacks.txt
--- a/tests/benchmarks/_script/flamegraph/example-dtrace.svg
+++ b/tests/benchmarks/_script/flamegraph/example-dtrace.svg
--- a/tests/benchmarks/_script/flamegraph/example-perf-stacks.txt.gz
+++ b/tests/benchmarks/_script/flamegraph/example-perf-stacks.txt.gz
--- a/tests/benchmarks/_script/flamegraph/example-perf.svg
+++ b/tests/benchmarks/_script/flamegraph/example-perf.svg
--- a/tests/benchmarks/_script/flamegraph/files.pl
+++ b/tests/benchmarks/_script/flamegraph/files.pl
--- a/tests/benchmarks/_script/flamegraph/flamegraph.pl
+++ b/tests/benchmarks/_script/flamegraph/flamegraph.pl
--- a/tests/benchmarks/_script/flamegraph/jmaps
+++ b/tests/benchmarks/_script/flamegraph/jmaps
--- a/tests/benchmarks/_script/flamegraph/pkgsplit-perf.pl
+++ b/tests/benchmarks/_script/flamegraph/pkgsplit-perf.pl
--- a/tests/benchmarks/_script/flamegraph/range-perf.pl
+++ b/tests/benchmarks/_script/flamegraph/range-perf.pl
--- a/tests/benchmarks/_script/flamegraph/record-test.sh
+++ b/tests/benchmarks/_script/flamegraph/record-test.sh
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-aix.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-aix.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-bpftrace.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-bpftrace.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-chrome-tracing.py
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-chrome-tracing.py
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-elfutils.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-elfutils.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-faulthandler.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-faulthandler.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-gdb.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-gdb.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-go.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-go.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-ibmjava.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-ibmjava.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-instruments.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-instruments.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-java-exceptions.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-java-exceptions.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-jstack.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-jstack.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-ljp.awk
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-ljp.awk
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-perf-sched.awk
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-perf-sched.awk
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-perf.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-perf.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-pmc.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-pmc.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-recursive.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-recursive.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-sample.awk
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-sample.awk
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-stap.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-stap.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-vsprof.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-vsprof.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-vtune-mc.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-vtune-mc.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-vtune.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-vtune.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-wcp.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-wcp.pl
--- a/tests/benchmarks/_script/flamegraph/stackcollapse-xdebug.php
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse-xdebug.php
--- a/tests/benchmarks/_script/flamegraph/stackcollapse.pl
+++ b/tests/benchmarks/_script/flamegraph/stackcollapse.pl
--- a/tests/benchmarks/_script/flamegraph/test.sh
+++ b/tests/benchmarks/_script/flamegraph/test.sh
--- a/tests/benchmarks/benchmark.sh
+++ b/tests/benchmarks/benchmark.sh
--- a/tests/benchmarks/benchmark_helpers.sh
+++ b/tests/benchmarks/benchmark_helpers.sh
@@ -105,7 +105,7 @@ fullpath() {
 }

 resources_dir() {
-	cd "$(dirname "$0")/../resources" && pwd
+	cd "$(dirname "$0")/../../tests/resources" && pwd
 }

 temp_dir() {
--- a/tests/benchmarks/blamegit
+++ b/tests/benchmarks/blamegit
--- a/tests/benchmarks/blamelinux
+++ b/tests/benchmarks/blamelinux
--- a/tests/benchmarks/blamesimple
+++ b/tests/benchmarks/blamesimple
--- a/tests/benchmarks/hash-object__text_100kb
+++ b/tests/benchmarks/hash-object__text_100kb
--- a/tests/benchmarks/hash-object__text_10mb
+++ b/tests/benchmarks/hash-object__text_10mb
--- a/tests/benchmarks/hash-object__text_1kb
+++ b/tests/benchmarks/hash-object__text_1kb
--- a/tests/benchmarks/hash-object__text_nocache_100kb
+++ b/tests/benchmarks/hash-object__text_nocache_100kb
--- a/tests/benchmarks/hash-object__text_nocache_10mb
+++ b/tests/benchmarks/hash-object__text_nocache_10mb
--- a/tests/benchmarks/hash-object__text_nocache_1kb
+++ b/tests/benchmarks/hash-object__text_nocache_1kb
--- a/tests/benchmarks/hash-object__write_text_100kb
+++ b/tests/benchmarks/hash-object__write_text_100kb
--- a/tests/benchmarks/hash-object__write_text_10mb
+++ b/tests/benchmarks/hash-object__write_text_10mb
--- a/tests/benchmarks/hash-object__write_text_1kb
+++ b/tests/benchmarks/hash-object__write_text_1kb
--- a/tests/benchmarks/hash-object__write_text_nocache_100kb
+++ b/tests/benchmarks/hash-object__write_text_nocache_100kb
--- a/tests/benchmarks/hash-object__write_text_nocache_10mb
+++ b/tests/benchmarks/hash-object__write_text_nocache_10mb
--- a/tests/benchmarks/hash-object__write_text_nocache_1kb
+++ b/tests/benchmarks/hash-object__write_text_nocache_1kb
--- a/tests/benchmarks/indexpack250mb
+++ b/tests/benchmarks/indexpack250mb
--- a/benchmarks/libgit2/CMakeLists.txt
+++ b/benchmarks/libgit2/CMakeLists.txt
@@ -0,0 +1,78 @@
+# util: the unit tests for libgit2's utility library
+
+if(NOT "${CMAKE_VERSION}" VERSION_LESS 3.27)
+        cmake_policy(SET CMP0148 OLD)
+endif()
+
+set(Python_ADDITIONAL_VERSIONS 3 2.7)
+find_package(PythonInterp)
+
+if(NOT PYTHONINTERP_FOUND)
+        message(FATAL_ERROR "Could not find a python interpeter, which is needed to build the tests. "
+                             "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests")
+endif()
+
+set(CLAR_PATH "${PROJECT_SOURCE_DIR}/deps/clar")
+set(BENCHMARK_PATH "${CMAKE_CURRENT_SOURCE_DIR}")
+add_definitions(-DCLAR_TMPDIR=\"libgit2_bench\")
+add_definitions(-DCLAR_WIN32_LONGPATHS)
+add_definitions(-DCLAR_HAS_REALPATH)
+add_definitions(-D_FILE_OFFSET_BITS=64)
+
+file(GLOB BENCHMARK_SRC *.c *.h)
+list(SORT BENCHMARK_SRC)
+
+set(CLAR_SRC
+	"${CLAR_PATH}/clar.c"
+	"${CLAR_PATH}/clar.h"
+	"${CLAR_PATH}/clar/fixtures.h"
+	"${CLAR_PATH}/clar/print.h"
+	"${CLAR_PATH}/clar/summary.h"
+	"${CLAR_PATH}/clar/sandbox.h"
+	"${CLAR_PATH}/clar/fs.h")
+
+add_custom_command(
+	OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/clar.suite ${CMAKE_CURRENT_BINARY_DIR}/clar_suite.h
+	COMMAND ${PYTHON_EXECUTABLE} ${CLAR_PATH}/generate.py -p benchmark -o "${CMAKE_CURRENT_BINARY_DIR}" -f .
+	DEPENDS ${BENCHMARK_SRC}
+	WORKING_DIRECTORY ${BENCHMARK_PATH}
+)
+
+set_source_files_properties(
+	${CLAR_PATH}/clar.c
+	PROPERTIES OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clar.suite)
+
+add_executable(libgit2_benchmarks ${CLAR_SRC}
+	${BENCHMARK_SRC}
+	$<TARGET_OBJECTS:util>
+	${LIBGIT2_DEPENDENCY_OBJECTS})
+
+target_link_libraries(libgit2_benchmarks libgit2package ${LIBGIT2_SYSTEM_LIBS})
+if(NOT MSVC_IDE)
+	target_link_libraries(libgit2_benchmarks m)
+endif()
+
+ide_split_sources(libgit2_benchmarks)
+
+target_include_directories(libgit2_benchmarks PRIVATE
+        "${CLAR_PATH}"
+        "${libgit2_BINARY_DIR}/src/util"
+        "${libgit2_BINARY_DIR}/include"
+        "${libgit2_SOURCE_DIR}/src/util"
+        "${libgit2_SOURCE_DIR}/include"
+	"${CMAKE_CURRENT_BINARY_DIR}"
+        "${LIBGIT2_DEPENDENCY_INCLUDES}"
+        "${LIBGIT2_SYSTEM_INCLUDES}")
+
+#
+# Old versions of gcc require us to declare our test functions; don't do
+# this on newer compilers to avoid unnecessary recompilation.
+#
+if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0)
+	target_compile_options(libgit2_benchmarks PRIVATE -include "clar_suite.h")
+endif()
+
+if(MSVC_IDE)
+	set_target_properties(libgit2_benchmarks PROPERTIES COMPILE_FLAGS "/Yuprecompiled.h /FIprecompiled.h")
+	set_source_files_properties("precompiled.c" COMPILE_FLAGS "/Ycprecompiled.h")
+endif()
--- a/benchmarks/libgit2/main.c
+++ b/benchmarks/libgit2/main.c
@@ -0,0 +1,31 @@
+#include <stdio.h>
+
+#include "clar.h"
+#include <git2.h>
+
+#ifdef _WIN32
+int __cdecl main(int argc, char *argv[])
+#else
+int main(int argc, char *argv[])
+#endif
+{
+	int res;
+
+	clar_test_set_mode(CL_TEST_BENCHMARK);
+	clar_test_init(argc, argv);
+
+	res = git_libgit2_init();
+	if (res < 0) {
+		const git_error *err = git_error_last();
+		const char *msg = err ? err->message : "unknown failure";
+		fprintf(stderr, "failed to init libgit2: %s\n", msg);
+		return res;
+	}
+
+	/* Run the test suite */
+	res = clar_test_run();
+
+	clar_test_shutdown();
+
+	return res;
+}
--- a/benchmarks/libgit2/oid.c
+++ b/benchmarks/libgit2/oid.c
@@ -0,0 +1,151 @@
+#include "clar.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include <git2.h>
+
+#define BENCHMARK_OID_COUNT 256
+
+static git_oid sha1_one[BENCHMARK_OID_COUNT];
+static git_oid *sha1_two;
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+static git_oid sha256_one[BENCHMARK_OID_COUNT];
+static git_oid *sha256_two;
+#endif
+
+static void update_data_to_val(git_oid *out, git_oid_t type, uint32_t val)
+{
+	unsigned char data[GIT_OID_MAX_SIZE] = {0};
+	size_t size;
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	size = (type == GIT_OID_SHA256) ? GIT_OID_SHA256_SIZE : GIT_OID_SHA1_SIZE;
+#else
+	size = GIT_OID_SHA1_SIZE;
+
+	((void)(type));
+#endif
+
+	memset(data, 0, GIT_OID_MAX_SIZE);
+
+	data[size - 1] = (unsigned char)(val & 0x000000ff);
+	data[size - 2] = (unsigned char)((val & 0x0000ff00) >> 8);
+	data[size - 3] = (unsigned char)((val & 0x00ff0000) >> 16);
+	data[size - 4] = (unsigned char)((val & 0x00ff0000) >> 24);
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	cl_assert(git_oid_from_raw(out, data, type) == 0);
+#else
+	cl_assert(git_oid_fromraw(out, data) == 0);
+#endif
+}
+
+void benchmark_oid__initialize(void)
+{
+	uint32_t accum = 0;
+	size_t i;
+
+	sha1_two = calloc(BENCHMARK_OID_COUNT, sizeof(git_oid));
+	cl_assert(sha1_two != NULL);
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	sha256_two = calloc(BENCHMARK_OID_COUNT, sizeof(git_oid));
+	cl_assert(sha256_two != NULL);
+#endif
+
+	for (i = 0; i < BENCHMARK_OID_COUNT; i++) {
+		update_data_to_val(&sha1_one[i], GIT_OID_SHA1, accum++);
+		update_data_to_val(&sha1_two[i], GIT_OID_SHA1, accum++);
+	}
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	for (i = 0; i < BENCHMARK_OID_COUNT; i++) {
+		update_data_to_val(&sha256_one[i], GIT_OID_SHA256, accum++);
+		update_data_to_val(&sha256_two[i], GIT_OID_SHA256, accum++);
+	}
+#endif
+}
+
+void benchmark_oid__reset(void)
+{
+}
+
+void benchmark_oid__cleanup(void)
+{
+	free(sha1_two);
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	free(sha256_two);
+#endif
+}
+
+void benchmark_oid__cmp_sha1(void)
+{
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_cmp(&sha1_one[j], &sha1_two[j]);
+}
+
+void benchmark_oid__cmp_sha256(void)
+{
+#ifdef GIT_EXPERIMENTAL_SHA256
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_cmp(&sha256_one[j], &sha256_two[j]);
+#else
+	clar__skip();
+#endif
+}
+
+void benchmark_oid__cpy_sha1(void)
+{
+	git_oid dest;
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_cpy(&dest, &sha1_one[j]);
+}
+
+void benchmark_oid__cpy_sha256(void)
+{
+#ifdef GIT_EXPERIMENTAL_SHA256
+	git_oid dest;
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_cpy(&dest, &sha256_one[j]);
+#else
+	clar__skip();
+#endif
+}
+
+void benchmark_oid__zero_sha1(void)
+{
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_is_zero(&sha1_one[j]);
+}
+
+void benchmark_oid__zero_sha256(void)
+{
+#ifdef GIT_EXPERIMENTAL_SHA256
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_is_zero(&sha256_one[j]);
+#else
+	clar__skip();
+#endif
+}
--- a/benchmarks/libgit2/precompiled.c
+++ b/benchmarks/libgit2/precompiled.c
@@ -0,0 +1 @@
+#include "precompiled.h"
--- a/benchmarks/libgit2/precompiled.h
+++ b/benchmarks/libgit2/precompiled.h
@@ -0,0 +1,2 @@
+#include "git2.h"
+#include "clar.h"
--- a/ci/compare-benchmarks.js
+++ b/ci/compare-benchmarks.js
@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+
+const fs = require('fs');
+const process = require('node:process');
+
+const path = process.argv[2] || '.';
+const output = process.argv[3];
+
+const dirs = fs.readdirSync(path);
+
+tests = [ ];
+results = { };
+
+for (const dir of dirs) {
+    const name = dir.replace(/^results-/, '');
+
+    const control = JSON.parse(fs.readFileSync(`${path}/${dir}/control.json`));
+    const candidate = JSON.parse(fs.readFileSync(`${path}/${dir}/candidate.json`));
+
+    results[name] = { };
+
+    const controlTests = control.tests.map((x) => x.name);
+    const candidateTests = candidate.tests.map((x) => x.name);
+    tests = [...new Set(tests.concat(controlTests).concat(candidateTests))];
+
+    for (const test of tests) {
+        const controlResults = control.tests.filter((x) => x.name === test)[0];
+        const candidateResults = candidate.tests.filter((x) => x.name === test)[0];
+
+        let result;
+
+        if (controlResults.results.status === 'ok' &&
+            candidateResults.results.status === 'ok') {
+            const delta = (candidateResults.results.mean - controlResults.results.mean) / candidateResults.results.mean;
+            const deltaPercent = Math.round(Math.abs(delta) * 1000) / 10;
+            const direction = (delta > 0) ? 'slower' : 'faster';
+            let highlight = '';
+
+            if (delta > 0.1) {
+                highlight = '**';
+            }
+
+            results[name][test] = `${highlight}${deltaPercent}% ${direction}${highlight}`;
+        }
+        else if (controlResults.results.status === 'ok') {
+            results[name][test] = `${candidateResults.results.stauts} in candidate`;
+        }
+        else if (candidateResults.results.status === 'ok') {
+            results[name][test] = `${controlResults.results.stauts} in control`;
+        }
+        else {
+            results[name][test] = controlResults.results.status;
+        }
+    }
+}
+
+let markdown = '| Platform \\ Test |';
+
+for (const test in Object.values(results)[0]) {
+    markdown += ` ${test} |`;
+}
+markdown += '\n';
+
+markdown += `| --- |`;
+for (const test in Object.values(results)[0]) {
+    markdown += ` --- |`;
+}
+markdown += '\n';
+
+for (const name in results) {
+    markdown += `| ${name.replaceAll(/-/g, '&#x2011;')} |`;
+
+    for (const test in results[name]) {
+        markdown += ` ${results[name][test]} |`;
+    }
+
+    markdown += `\n`;
+}
+
+if (output) {
+    fs.writeFileSync(output, markdown);
+}
+else {
+    console.log(markdown);
+}
+
+function p(arr, q) {
+    const array = [...arr].sort();
+    const pos = (array.length - 1) * q;
+    const base = Math.floor(pos);
+    const rest = pos - base;
+
+    if (array[base + 1] !== undefined) {
+        return array[base] + rest * (array[base + 1] - array[base]);
+    }
+    else {
+        return array[base];
+    }
+}
--- a/tests/clar/clar.c
+++ b/tests/clar/clar.c
@@ -100,6 +100,7 @@
 	typedef struct stat STAT_T;
 #endif

+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
 #define MAX(x, y) (((x) > (y)) ? (x) : (y))

 #include "clar.h"
@@ -113,11 +114,11 @@ fixture_path(const char *base, const char *fixture_name);
 #endif

 struct clar_error {
-	const char *file;
-	const char *function;
-	uintmax_t line_number;
-	const char *error_msg;
+	const char *message;
 	char *description;
+	const char *function;
+	const char *file;
+	uintmax_t line_number;

 	struct clar_error *next;
 };
@@ -130,13 +131,22 @@ struct clar_explicit {
 };

 struct clar_report {
-	const char *test;
-	int test_number;
 	const char *suite;
+	const char *test;
+	const char *description;
+	int test_number;
+
+	int runs;

 	enum cl_test_status status;
 	time_t start;
-	double elapsed;
+
+	double *times;
+	double time_min;
+	double time_max;
+	double time_mean;
+	double time_stddev;
+	double time_total;

 	struct clar_error *errors;
 	struct clar_error *last_error;
@@ -150,10 +160,12 @@ struct clar_summary {
 };

 static struct {
+	enum cl_test_mode test_mode;
 	enum cl_test_status test_status;

-	const char *active_test;
 	const char *active_suite;
+	const char *active_test;
+	const char *active_description;

 	int total_skipped;
 	int total_errors;
@@ -162,6 +174,7 @@ static struct {
 	int suites_ran;

 	enum cl_output_format output_format;
+	enum cl_summary_format summary_format;

 	int exit_on_error;
 	int verbosity;
@@ -193,28 +206,32 @@ static struct {

 struct clar_func {
 	const char *name;
+	const char *description;
+	int runs;
 	void (*ptr)(void);
 };

 struct clar_suite {
 	const char *name;
 	struct clar_func initialize;
+	struct clar_func reset;
 	struct clar_func cleanup;
 	const struct clar_func *tests;
 	size_t test_count;
 	int enabled;
 };

-/* From clar_print_*.c */
+/* From print.h */
 static void clar_print_init(int test_count, int suite_count);
 static void clar_print_shutdown(int test_count, int suite_count, int error_count);
 static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error);
-static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status failed);
-static void clar_print_onsuite(const char *suite_name, int suite_index);
+static void clar_print_suite_start(const char *suite_name, int suite_index);
+static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number);
+static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report);
 static void clar_print_onabortv(const char *msg, va_list argp);
 static void clar_print_onabort(const char *msg, ...);

-/* From clar_sandbox.c */
+/* From sandbox.c */
 static void clar_tempdir_init(void);
 static void clar_tempdir_shutdown(void);
 static int clar_sandbox_create(const char *suite_name, const char *test_name);
@@ -224,6 +241,8 @@ static int clar_sandbox_cleanup(void);
 static struct clar_summary *clar_summary_init(const char *filename);
 static int clar_summary_shutdown(struct clar_summary *fp);

+#include "clar/counter.h"
+
 /* Load the declarations for the test suite */
 #include "clar.suite"

@@ -280,70 +299,122 @@ clar_report_all(void)
 	}
 }

-#ifdef WIN32
-# define clar_time DWORD
-
-static void clar_time_now(clar_time *out)
+static void
+compute_times(void)
 {
-	*out = GetTickCount();
-}
+	double total_squares = 0;
+	int i;

-static double clar_time_diff(clar_time *start, clar_time *end)
-{
-	return ((double)*end - (double)*start) / 1000;
-}
-#else
-# include <sys/time.h>
+	_clar.last_report->time_min = _clar.last_report->times[0];
+	_clar.last_report->time_max = _clar.last_report->times[0];
+	_clar.last_report->time_total = _clar.last_report->times[0];

-# define clar_time struct timeval
+	for (i = 1; i < _clar.last_report->runs; i++) {
+		if (_clar.last_report->times[i] < _clar.last_report->time_min)
+			_clar.last_report->time_min = _clar.last_report->times[i];

-static void clar_time_now(clar_time *out)
-{
-	gettimeofday(out, NULL);
-}
+		if (_clar.last_report->times[i] > _clar.last_report->time_max)
+			_clar.last_report->time_max = _clar.last_report->times[i];

-static double clar_time_diff(clar_time *start, clar_time *end)
-{
-	return ((double)end->tv_sec + (double)end->tv_usec / 1.0E6) -
-	       ((double)start->tv_sec + (double)start->tv_usec / 1.0E6);
+		_clar.last_report->time_total += _clar.last_report->times[i];
+	}
+
+	if (_clar.last_report->runs <= 1) {
+		_clar.last_report->time_stddev = 0;
+	} else {
+		_clar.last_report->time_mean = _clar.last_report->time_total / _clar.last_report->runs;
+
+		for (i = 0; i < _clar.last_report->runs; i++) {
+			double dev = (_clar.last_report->times[i] > _clar.last_report->time_mean) ?
+				_clar.last_report->times[i] - _clar.last_report->time_mean :
+				_clar.last_report->time_mean - _clar.last_report->times[i];
+
+			total_squares += (dev * dev);
+		}
+
+		_clar.last_report->time_stddev = sqrt(total_squares / _clar.last_report->runs);
+	}
 }
-#endif

 static void
 clar_run_test(
 	const struct clar_suite *suite,
 	const struct clar_func *test,
 	const struct clar_func *initialize,
+	const struct clar_func *reset,
 	const struct clar_func *cleanup)
 {
-	clar_time start, end;
+	int runs = test->runs;
+	volatile int i = 0;

-	_clar.trampoline_enabled = 1;
+	_clar.last_report->start = time(NULL);
+	_clar.last_report->times = &_clar.last_report->time_mean;

 	CL_TRACE(CL_TRACE__TEST__BEGIN);

 	clar_sandbox_create(suite->name, test->name);

-	_clar.last_report->start = time(NULL);
-	clar_time_now(&start);
+	clar_print_test_start(suite->name, test->name, _clar.tests_ran);
+
+	_clar.trampoline_enabled = 1;

 	if (setjmp(_clar.trampoline) == 0) {
 		if (initialize->ptr != NULL)
 			initialize->ptr();

 		CL_TRACE(CL_TRACE__TEST__RUN_BEGIN);
-		test->ptr();
+
+		do {
+			struct clar_counter start, end;
+			double elapsed;
+
+			if (i > 0 && reset->ptr != NULL) {
+				reset->ptr();
+			} else if (i > 0) {
+				if (_clar.local_cleanup != NULL)
+					_clar.local_cleanup(_clar.local_cleanup_payload);
+				if (cleanup->ptr != NULL)
+					cleanup->ptr();
+				if (initialize->ptr != NULL)
+					initialize->ptr();
+			}
+
+			clar_counter_now(&start);
+			test->ptr();
+			clar_counter_now(&end);
+
+			elapsed = clar_counter_diff(&start, &end);
+
+			/*
+			 * unless the number of runs was explicitly given
+			 * in benchmark mode, use the first run as a sample
+			 * to determine how many runs we should attempt
+			 */
+			if (_clar.test_mode == CL_TEST_BENCHMARK && !runs) {
+				runs = MAX(CLAR_BENCHMARK_RUN_MIN, (int)(CLAR_BENCHMARK_RUN_TIME / elapsed));
+				runs = MIN(CLAR_BENCHMARK_RUN_MAX, runs);
+			}
+
+			if (i == 0 && runs > 1) {
+				_clar.last_report->times = calloc(runs, sizeof(double));
+
+				if (_clar.last_report->times == NULL)
+					clar_abort("Failed to allocate report times.\n");
+			}
+
+			_clar.last_report->runs++;
+			_clar.last_report->times[i] = elapsed;
+		} while(++i < runs);
+
 		CL_TRACE(CL_TRACE__TEST__RUN_END);
 	}

-	clar_time_now(&end);
-
 	_clar.trampoline_enabled = 0;

 	if (_clar.last_report->status == CL_TEST_NOTRUN)
 		_clar.last_report->status = CL_TEST_OK;

-	_clar.last_report->elapsed = clar_time_diff(&start, &end);
+	compute_times();

 	if (_clar.local_cleanup != NULL)
 		_clar.local_cleanup(_clar.local_cleanup_payload);
@@ -363,7 +434,7 @@ clar_run_test(
 	_clar.local_cleanup = NULL;
 	_clar.local_cleanup_payload = NULL;

-	clar_print_ontest(suite->name, test->name, _clar.tests_ran, _clar.last_report->status);
+	clar_print_test_finish(suite->name, test->name, _clar.tests_ran, _clar.last_report);
 }

 static void
@@ -380,10 +451,11 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)
 	if (_clar.exit_on_error && _clar.total_errors)
 		return;

-	clar_print_onsuite(suite->name, ++_clar.suites_ran);
+	clar_print_suite_start(suite->name, ++_clar.suites_ran);

 	_clar.active_suite = suite->name;
 	_clar.active_test = NULL;
+	_clar.active_description = NULL;
 	CL_TRACE(CL_TRACE__SUITE_BEGIN);

 	if (filter) {
@@ -412,11 +484,13 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)
 			continue;

 		_clar.active_test = test[i].name;
+		_clar.active_description = test[i].description;

 		if ((report = calloc(1, sizeof(*report))) == NULL)
 			clar_abort("Failed to allocate report.\n");
 		report->suite = _clar.active_suite;
 		report->test = _clar.active_test;
+		report->description = _clar.active_description;
 		report->test_number = _clar.tests_ran;
 		report->status = CL_TEST_NOTRUN;

@@ -428,13 +502,14 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)

 		_clar.last_report = report;

-		clar_run_test(suite, &test[i], &suite->initialize, &suite->cleanup);
+		clar_run_test(suite, &test[i], &suite->initialize, &suite->reset, &suite->cleanup);

 		if (_clar.exit_on_error && _clar.total_errors)
 			return;
 	}

 	_clar.active_test = NULL;
+	_clar.active_description = NULL;
 	CL_TRACE(CL_TRACE__SUITE_END);
 }

@@ -600,6 +675,14 @@ clar_test_init(int argc, char **argv)
 {
 	const char *summary_env;

+	if (_clar.test_mode == CL_TEST_BENCHMARK) {
+		_clar.output_format = CL_OUTPUT_TIMING;
+		_clar.summary_format = CL_SUMMARY_JSON;
+	} else {
+		_clar.output_format = CL_OUTPUT_CLAP;
+		_clar.summary_format = CL_SUMMARY_JUNIT;
+	}
+
 	if (argc > 1)
 		clar_parse_args(argc, argv);

@@ -622,6 +705,12 @@ clar_test_init(int argc, char **argv)
 	clar_tempdir_init();
 }

+void
+clar_test_set_mode(enum cl_test_mode mode)
+{
+	_clar.test_mode = mode;
+}
+
 int
 clar_test_run(void)
 {
@@ -671,6 +760,9 @@ clar_test_shutdown(void)
 			free(error);
 		}

+		if (report->times != &report->time_mean)
+			free(report->times);
+
 		report_next = report->next;
 		free(report);
 	}
@@ -735,7 +827,7 @@ static void clar__failv(
 	error->file = _clar.invoke_file ? _clar.invoke_file : file;
 	error->function = _clar.invoke_func ? _clar.invoke_func : function;
 	error->line_number = _clar.invoke_line ? _clar.invoke_line : line;
-	error->error_msg = error_msg;
+	error->message = error_msg;

 	if (description != NULL) {
 		va_list args_copy;
@@ -791,14 +883,14 @@ void clar__assert(
 	const char *file,
 	const char *function,
 	size_t line,
-	const char *error_msg,
-	const char *description,
+	const char *error_message,
+	const char *error_description,
 	int should_abort)
 {
 	if (condition)
 		return;

-	clar__fail(file, function, line, error_msg, description, should_abort);
+	clar__fail(file, function, line, error_message, error_description, should_abort);
 }

 void clar__assert_equal(
--- a/tests/clar/clar.h
+++ b/tests/clar/clar.h
@@ -21,6 +21,16 @@
 # define CLAR_MAX_PATH 4096
 #endif

+/*
+ * In benchmark mode, by default, clar will run the test repeatedly for
+ * approximately `CLAR_BENCHMARK_RUN_TIME` seconds, and at least
+ * `CLAR_BENCHMARK_RUN_MIN` iterations.
+ */
+
+#define CLAR_BENCHMARK_RUN_TIME 3.0
+#define CLAR_BENCHMARK_RUN_MIN  10
+#define CLAR_BENCHMARK_RUN_MAX  30000000
+
 #ifndef CLAR_SELFTEST
 # define CLAR_CURRENT_FILE __FILE__
 # define CLAR_CURRENT_LINE __LINE__
@@ -31,6 +41,11 @@
 # define CLAR_CURRENT_FUNC "func"
 #endif

+enum cl_test_mode {
+	CL_TEST_STANDARD,
+	CL_TEST_BENCHMARK,
+};
+
 enum cl_test_status {
 	CL_TEST_OK,
 	CL_TEST_FAILURE,
@@ -41,10 +56,17 @@ enum cl_test_status {
 enum cl_output_format {
 	CL_OUTPUT_CLAP,
 	CL_OUTPUT_TAP,
+	CL_OUTPUT_TIMING,
+};
+
+enum cl_summary_format {
+	CL_SUMMARY_JUNIT,
+	CL_SUMMARY_JSON,
 };

 /** Setup clar environment */
 void clar_test_init(int argc, char *argv[]);
+void clar_test_set_mode(enum cl_test_mode mode);
 int clar_test_run(void);
 void clar_test_shutdown(void);

--- a/deps/clar/clar/counter.h
+++ b/deps/clar/clar/counter.h
@@ -0,0 +1,167 @@
+#define CLAR_COUNTER_TV_DIFF(out_sec, out_usec, start_sec, start_usec, end_sec, end_usec) \
+	if (start_usec > end_usec) { \
+		out_sec = (end_sec - 1) - start_sec; \
+		out_usec = (end_usec + 1000000) - start_usec; \
+	} else { \
+		out_sec = end_sec - start_sec; \
+		out_usec = end_usec - start_usec; \
+	}
+
+#ifdef _WIN32
+
+struct clar_counter {
+	LARGE_INTEGER value;
+};
+
+static void clar_counter_now(struct clar_counter *out)
+{
+	QueryPerformanceCounter(&out->value);
+}
+
+static double clar_counter_diff(
+	struct clar_counter *start,
+	struct clar_counter *end)
+{
+	LARGE_INTEGER freq;
+
+	QueryPerformanceFrequency(&freq);
+	return (double)(end->value.QuadPart - start->value.QuadPart)/(double)freq.QuadPart;
+}
+
+#elif __APPLE__
+
+#include <mach/mach_time.h>
+#include <sys/time.h>
+
+static double clar_counter_scaling_factor = -1;
+
+struct clar_counter {
+	union {
+		uint64_t absolute_time;
+		struct timeval tv;
+	} val;
+};
+
+static void clar_counter_now(struct clar_counter *out)
+{
+	if (clar_counter_scaling_factor == 0) {
+		mach_timebase_info_data_t info;
+
+		clar_counter_scaling_factor =
+			mach_timebase_info(&info) == KERN_SUCCESS ?
+			((double)info.numer / (double)info.denom) / 1.0E6 :
+			-1;
+	}
+
+	/* mach_timebase_info failed; fall back to gettimeofday */
+	if (clar_counter_scaling_factor < 0)
+		gettimeofday(&out->val.tv, NULL);
+	else
+		out->val.absolute_time = mach_absolute_time();
+}
+
+static double clar_counter_diff(
+	struct clar_counter *start,
+	struct clar_counter *end)
+{
+	if (clar_counter_scaling_factor < 0) {
+		time_t sec;
+		suseconds_t usec;
+
+		CLAR_COUNTER_TV_DIFF(sec, usec,
+			start->val.tv.tv_sec, start->val.tv.tv_usec,
+			end->val.tv.tv_sec, end->val.tv.tv_usec);
+
+		return (double)sec + ((double)usec / 1000000.0);
+	} else {
+		return (double)(end->val.absolute_time - start->val.absolute_time) *
+			clar_counter_scaling_factor;
+	}
+}
+
+#elif defined(__amigaos4__)
+
+#include <proto/timer.h>
+
+struct clar_counter {
+	struct TimeVal tv;
+}
+
+static void clar_counter_now(struct clar_counter *out)
+{
+	ITimer->GetUpTime(&out->tv);
+}
+
+static double clar_counter_diff(
+	struct clar_counter *start,
+	struct clar_counter *end)
+{
+	uint32_t sec, usec;
+
+	CLAR_COUNTER_TV_DIFF(sec, usec,
+		start->tv.Seconds, start->tv.Microseconds,
+		end->tv.Seconds, end->tv.Microseconds);
+
+	return (double)sec + ((double)usec / 1000000.0);
+}
+
+#else
+
+#include <sys/time.h>
+
+struct clar_counter {
+	int type;
+	union {
+#ifdef CLOCK_MONOTONIC
+		struct timespec tp;
+#endif
+		struct timeval tv;
+	} val;
+};
+
+static void clar_counter_now(struct clar_counter *out)
+{
+#ifdef CLOCK_MONOTONIC
+	if (clock_gettime(CLOCK_MONOTONIC, &out->val.tp) == 0) {
+		out->type = 0;
+		return;
+	}
+#endif
+
+	/* Fall back to using gettimeofday */
+	out->type = 1;
+	gettimeofday(&out->val.tv, NULL);
+}
+
+static double clar_counter_diff(
+	struct clar_counter *start,
+	struct clar_counter *end)
+{
+	time_t sec;
+	suseconds_t usec;
+
+#ifdef CLOCK_MONOTONIC
+	if (start->type == 0) {
+		time_t sec;
+		long nsec;
+
+		if (start->val.tp.tv_sec > end->val.tp.tv_sec) {
+			sec = (end->val.tp.tv_sec - 1) - start->val.tp.tv_sec;
+			nsec = (end->val.tp.tv_nsec + 1000000000) - start->val.tp.tv_nsec;
+		} else {
+			sec = end->val.tp.tv_sec - start->val.tp.tv_sec;
+			nsec = end->val.tp.tv_nsec - start->val.tp.tv_nsec;
+		}
+
+		return (double)sec + ((double)nsec / 1000000000.0);
+	}
+#endif
+
+	CLAR_COUNTER_TV_DIFF(sec, usec,
+		start->val.tv.tv_sec, start->val.tv.tv_usec,
+		end->val.tv.tv_sec, end->val.tv.tv_usec);
+
+	return (double)sec + ((double)usec / 1000000.0);
+}
+
+#endif
--- a/tests/clar/clar/fixtures.h
+++ b/tests/clar/clar/fixtures.h
--- a/tests/clar/clar/fs.h
+++ b/tests/clar/clar/fs.h
--- a/tests/clar/clar/print.h
+++ b/tests/clar/clar/print.h
@@ -48,7 +48,7 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con
 		error->file,
 		error->line_number);

-	clar_print_indented(error->error_msg, 2);
+	clar_print_indented(error->message, 2);

 	if (error->description != NULL)
 		clar_print_indented(error->description, 2);
@@ -57,9 +57,8 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con
 	fflush(stdout);
 }

-static void clar_print_clap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status)
+static void clar_print_clap_test_start(const char *suite_name, const char *test_name, int test_number)
 {
-	(void)test_name;
 	(void)test_number;

 	if (_clar.verbosity < 0)
@@ -67,15 +66,18 @@ static void clar_print_clap_ontest(const char *suite_name, const char *test_name

 	if (_clar.verbosity > 1) {
 		printf("%s::%s: ", suite_name, test_name);
+		fflush(stdout);
+	}
+}

-		switch (status) {
-		case CL_TEST_OK: printf("ok\n"); break;
-		case CL_TEST_FAILURE: printf("fail\n"); break;
-		case CL_TEST_SKIP: printf("skipped\n"); break;
-		case CL_TEST_NOTRUN: printf("notrun\n"); break;
-		}
-	} else {
-		switch (status) {
+static void clar_print_clap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report)
+{
+	(void)suite_name;
+	(void)test_name;
+	(void)test_number;
+
+	if (_clar.verbosity == 0) {
+		switch (report->status) {
 		case CL_TEST_OK: printf("."); break;
 		case CL_TEST_FAILURE: printf("F"); break;
 		case CL_TEST_SKIP: printf("S"); break;
@@ -83,10 +85,17 @@ static void clar_print_clap_ontest(const char *suite_name, const char *test_name
 		}

 		fflush(stdout);
+	} else if (_clar.verbosity > 1) {
+		switch (report->status) {
+		case CL_TEST_OK: printf("ok\n"); break;
+		case CL_TEST_FAILURE: printf("fail\n"); break;
+		case CL_TEST_SKIP: printf("skipped\n"); break;
+		case CL_TEST_NOTRUN: printf("notrun\n"); break;
+		}
 	}
 }

-static void clar_print_clap_onsuite(const char *suite_name, int suite_index)
+static void clar_print_clap_suite_start(const char *suite_name, int suite_index)
 {
 	if (_clar.verbosity < 0)
 		return;
@@ -127,7 +136,7 @@ static void clar_print_tap_error(int num, const struct clar_report *report, cons

 static void print_escaped(const char *str)
 {
-	const char *c;
+	char *c;

 	while ((c = strchr(str, '\'')) != NULL) {
 		printf("%.*s", (int)(c - str), str);
@@ -138,14 +147,21 @@ static void print_escaped(const char *str)
 	printf("%s", str);
 }

-static void clar_print_tap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status)
+static void clar_print_tap_test_start(const char *suite_name, const char *test_name, int test_number)
+{
+	(void)suite_name;
+	(void)test_name;
+	(void)test_number;
+}
+
+static void clar_print_tap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report)
 {
 	const struct clar_error *error = _clar.last_report->errors;

 	(void)test_name;
 	(void)test_number;

-	switch(status) {
+	switch(report->status) {
 	case CL_TEST_OK:
 		printf("ok %d - %s::%s\n", test_number, suite_name, test_name);
 		break;
@@ -155,7 +171,7 @@ static void clar_print_tap_ontest(const char *suite_name, const char *test_name,
 		if (_clar.verbosity >= 0) {
 			printf("    ---\n");
 			printf("    reason: |\n");
-			clar_print_indented(error->error_msg, 6);
+			clar_print_indented(error->message, 6);

 			if (error->description)
 				clar_print_indented(error->description, 6);
@@ -177,7 +193,7 @@ static void clar_print_tap_ontest(const char *suite_name, const char *test_name,
 	fflush(stdout);
 }

-static void clar_print_tap_onsuite(const char *suite_name, int suite_index)
+static void clar_print_tap_suite_start(const char *suite_name, int suite_index)
 {
 	if (_clar.verbosity < 0)
 		return;
@@ -191,6 +207,114 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg)
 	fflush(stdout);
 }

+/* timings format: useful for benchmarks */
+
+static void clar_print_timing_init(int test_count, int suite_count)
+{
+	(void)test_count;
+	(void)suite_count;
+
+	printf("Started benchmarks (mean time ± stddev / min time … max time):\n\n");
+}
+
+static void clar_print_timing_shutdown(int test_count, int suite_count, int error_count)
+{
+	(void)test_count;
+	(void)suite_count;
+	(void)error_count;
+}
+
+static void clar_print_timing_error(int num, const struct clar_report *report, const struct clar_error *error)
+{
+	(void)num;
+	(void)report;
+	(void)error;
+}
+
+static void clar_print_timing_test_start(const char *suite_name, const char *test_name, int test_number)
+{
+	(void)test_number;
+
+	printf("%s::%s:  ", suite_name, test_name);
+	fflush(stdout);
+}
+
+static void clar_print_timing_time(double t)
+{
+	static const char *units[] = { "sec", "ms", "μs", "ns" };
+	static const int units_len = sizeof(units) / sizeof(units[0]);
+	int unit = 0, exponent = 0, digits;
+
+	while (t < 1.0 && unit < units_len - 1) {
+		t *= 1000.0;
+		unit++;
+	}
+
+	while (t > 0.0 && t < 1.0 && exponent < 10) {
+		t *= 10.0;
+		exponent++;
+	}
+
+	digits = (t < 10.0) ? 3 : ((t < 100.0) ? 2 : 1);
+
+	printf("%.*f", digits, t);
+
+	if (exponent > 0)
+		printf("e-%d", exponent);
+
+	printf(" %s", units[unit]);
+}
+
+static void clar_print_timing_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report)
+{
+	const struct clar_error *error = _clar.last_report->errors;
+
+	(void)suite_name;
+	(void)test_name;
+	(void)test_number;
+
+	switch(report->status) {
+	case CL_TEST_OK:
+		clar_print_timing_time(report->time_mean);
+
+		if (report->runs > 1) {
+			printf(" ± ");
+			clar_print_timing_time(report->time_stddev);
+
+			printf(" / range: ");
+			clar_print_timing_time(report->time_min);
+			printf(" … ");
+			clar_print_timing_time(report->time_max);
+			printf("  (%d runs)", report->runs);
+		}
+
+		printf("\n");
+		break;
+	case CL_TEST_FAILURE:
+		printf("failed: %s\n", error->message);
+		break;
+	case CL_TEST_SKIP:
+	case CL_TEST_NOTRUN:
+		printf("skipped\n");
+		break;
+	}
+
+	fflush(stdout);
+}
+
+static void clar_print_timing_suite_start(const char *suite_name, int suite_index)
+{
+	if (_clar.verbosity == 1)
+		printf("\n%s", suite_name);
+
+	(void)suite_index;
+}
+
+static void clar_print_timing_onabort(const char *fmt, va_list arg)
+{
+	vfprintf(stderr, fmt, arg);
+}
+
 /* indirection between protocol output selection */

 #define PRINT(FN, ...) do { \
@@ -201,6 +325,9 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg)
 			case CL_OUTPUT_TAP: \
 				clar_print_tap_##FN (__VA_ARGS__); \
 				break; \
+			case CL_OUTPUT_TIMING: \
+				clar_print_timing_##FN (__VA_ARGS__); \
+				break; \
 			default: \
 				abort(); \
 		} \
@@ -221,14 +348,19 @@ static void clar_print_error(int num, const struct clar_report *report, const st
 	PRINT(error, num, report, error);
 }

-static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status)
+static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number)
 {
-	PRINT(ontest, suite_name, test_name, test_number, status);
+	PRINT(test_start, suite_name, test_name, test_number);
 }

-static void clar_print_onsuite(const char *suite_name, int suite_index)
+static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report)
 {
-	PRINT(onsuite, suite_name, suite_index);
+	PRINT(test_finish, suite_name, test_name, test_number, report);
+}
+
+static void clar_print_suite_start(const char *suite_name, int suite_index)
+{
+	PRINT(suite_start, suite_name, suite_index);
 }

 static void clar_print_onabortv(const char *msg, va_list argp)
--- a/tests/clar/clar/sandbox.h
+++ b/tests/clar/clar/sandbox.h
--- a/deps/clar/clar/summary.h
+++ b/deps/clar/clar/summary.h
@@ -0,0 +1,311 @@
+#include <stdio.h>
+#include <time.h>
+
+static int clar_summary_time_digits(double t)
+{
+	int digits = 3;
+
+	if (t >= 100.0)
+		return 1;
+	else if (t >= 10.0)
+		return 2;
+
+	while (t > 0.0 && t < 1.0 && digits < 10) {
+		t *= 10.0;
+		digits++;
+	}
+
+	return digits;
+}
+
+static int clar_summary_junit_close_tag(
+    struct clar_summary *summary, const char *tag, int indent)
+{
+	const char *indt;
+
+	if (indent == 0) indt = "";
+	else if (indent == 1) indt = "\t";
+	else indt = "\t\t";
+
+	return fprintf(summary->fp, "%s</%s>\n", indt, tag);
+}
+
+static int clar_summary_junit_testsuites(struct clar_summary *summary)
+{
+	return fprintf(summary->fp, "<testsuites>\n");
+}
+
+static int clar_summary_junit_testsuite(struct clar_summary *summary,
+    int idn, const char *name, time_t timestamp,
+    int test_count, int fail_count, int error_count)
+{
+	struct tm tm;
+	char iso_dt[20];
+
+	localtime_r(&timestamp, &tm);
+	if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", &tm) == 0)
+		return -1;
+
+	return fprintf(summary->fp, "\t<testsuite"
+		       " id=\"%d\""
+		       " name=\"%s\""
+		       " hostname=\"localhost\""
+		       " timestamp=\"%s\""
+		       " tests=\"%d\""
+		       " failures=\"%d\""
+		       " errors=\"%d\">\n",
+		       idn, name, iso_dt, test_count, fail_count, error_count);
+}
+
+static int clar_summary_junit_testcase(struct clar_summary *summary,
+    const char *name, const char *classname, double elapsed)
+{
+	return fprintf(summary->fp,
+	    "\t\t<testcase name=\"%s\" classname=\"%s\" time=\"%.*f\">\n",
+		name, classname, clar_summary_time_digits(elapsed), elapsed);
+}
+
+static int clar_summary_junit_failure(struct clar_summary *summary,
+    const char *type, const char *message, const char *desc)
+{
+	return fprintf(summary->fp,
+	    "\t\t\t<failure type=\"%s\"><![CDATA[%s\n%s]]></failure>\n",
+	    type, message, desc);
+}
+
+static int clar_summary_junit_skipped(struct clar_summary *summary)
+{
+	return fprintf(summary->fp, "\t\t\t<skipped />\n");
+}
+
+static struct clar_summary *clar_summary_junit_init(const char *filename)
+{
+	struct clar_summary *summary;
+	FILE *fp;
+
+	if ((fp = fopen(filename, "w")) == NULL) {
+		perror("fopen");
+		return NULL;
+	}
+
+	if ((summary = malloc(sizeof(struct clar_summary))) == NULL) {
+		perror("malloc");
+		fclose(fp);
+		return NULL;
+	}
+
+	summary->filename = filename;
+	summary->fp = fp;
+
+	return summary;
+}
+
+static int clar_summary_junit_shutdown(struct clar_summary *summary)
+{
+	struct clar_report *report;
+	const char *last_suite = NULL;
+
+	if (clar_summary_junit_testsuites(summary) < 0)
+		goto on_error;
+
+	report = _clar.reports;
+	while (report != NULL) {
+		struct clar_error *error = report->errors;
+
+		if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) {
+			if (clar_summary_junit_testsuite(summary, 0, report->suite,
+			    report->start, _clar.tests_ran, _clar.total_errors, 0) < 0)
+				goto on_error;
+		}
+
+		last_suite = report->suite;
+
+		clar_summary_junit_testcase(summary, report->test, report->suite, report->time_total);
+
+		while (error != NULL) {
+			if (clar_summary_junit_failure(summary, "assert",
+			    error->message, error->description) < 0)
+				goto on_error;
+
+			error = error->next;
+		}
+
+		if (report->status == CL_TEST_SKIP)
+			clar_summary_junit_skipped(summary);
+
+		if (clar_summary_junit_close_tag(summary, "testcase", 2) < 0)
+			goto on_error;
+
+		report = report->next;
+
+		if (!report || strcmp(last_suite, report->suite) != 0) {
+			if (clar_summary_junit_close_tag(summary, "testsuite", 1) < 0)
+				goto on_error;
+		}
+	}
+
+	if (clar_summary_junit_close_tag(summary, "testsuites", 0) < 0 ||
+	    fclose(summary->fp) != 0)
+		goto on_error;
+
+	printf("written summary file to %s\n", summary->filename);
+
+	free(summary);
+	return 0;
+
+on_error:
+	fclose(summary->fp);
+	free(summary);
+	return -1;
+}
+
+static struct clar_summary *clar_summary_json_init(const char *filename)
+{
+	struct clar_summary *summary;
+	FILE *fp;
+
+	if ((fp = fopen(filename, "w")) == NULL) {
+		perror("fopen");
+		return NULL;
+	}
+
+	if ((summary = malloc(sizeof(struct clar_summary))) == NULL) {
+		perror("malloc");
+		fclose(fp);
+		return NULL;
+	}
+
+	summary->filename = filename;
+	summary->fp = fp;
+
+	return summary;
+}
+
+static int clar_summary_json_shutdown(struct clar_summary *summary)
+{
+	struct clar_report *report;
+	int i;
+
+	fprintf(summary->fp, "{\n");
+	fprintf(summary->fp, "  \"tests\": [\n");
+
+	report = _clar.reports;
+	while (report != NULL) {
+		struct clar_error *error = report->errors;
+
+		if (report != _clar.reports)
+			fprintf(summary->fp, ",\n");
+
+		fprintf(summary->fp, "    {\n");
+		fprintf(summary->fp, "      \"name\": \"%s::%s\",\n", report->suite, report->test);
+
+		if (report->description)
+			fprintf(summary->fp, "      \"description\": \"%s\",\n", report->description);
+
+		fprintf(summary->fp, "      \"results\": {\n");
+
+		fprintf(summary->fp, "        \"status\": ");
+		if (report->status == CL_TEST_OK)
+			fprintf(summary->fp, "\"ok\",\n");
+		else if (report->status == CL_TEST_FAILURE)
+			fprintf(summary->fp, "\"failed\",\n");
+		else if (report->status == CL_TEST_SKIP)
+			fprintf(summary->fp, "\"skipped\"\n");
+		else
+			clar_abort("unknown test status %d", report->status);
+
+		if (report->status == CL_TEST_OK) {
+			fprintf(summary->fp, "        \"mean\": %.*f,\n",
+				clar_summary_time_digits(report->time_mean), report->time_mean);
+			fprintf(summary->fp, "        \"stddev\": %.*f,\n",
+				clar_summary_time_digits(report->time_stddev), report->time_stddev);
+			fprintf(summary->fp, "        \"min\": %.*f,\n",
+				clar_summary_time_digits(report->time_min), report->time_min);
+			fprintf(summary->fp, "        \"max\": %.*f,\n",
+				clar_summary_time_digits(report->time_max), report->time_max);
+			fprintf(summary->fp, "        \"times\": [\n");
+
+			for (i = 0; i < report->runs; i++) {
+				if (i > 0)
+					fprintf(summary->fp, ",\n");
+
+				fprintf(summary->fp, "          %.*f",
+					clar_summary_time_digits(report->times[i]), report->times[i]);
+			}
+
+			fprintf(summary->fp, "\n        ]\n");
+		}
+
+		if (report->status == CL_TEST_FAILURE) {
+			fprintf(summary->fp, "        \"errors\": [\n");
+
+			while (error != NULL) {
+				if (error != report->errors)
+					fprintf(summary->fp, ",\n");
+
+				fprintf(summary->fp, "          {\n");
+				fprintf(summary->fp, "            \"message\": \"%s\",\n", error->message);
+
+				if (error->description)
+					fprintf(summary->fp, "            \"description\": \"%s\",\n", error->description);
+
+				fprintf(summary->fp, "            \"function\": \"%s\",\n", error->function);
+				fprintf(summary->fp, "            \"file\": \"%s\",\n", error->file);
+				fprintf(summary->fp, "            \"line\": %" PRIuMAX "\n", error->line_number);
+				fprintf(summary->fp, "          }");
+
+				error = error->next;
+			}
+
+			fprintf(summary->fp, "\n");
+			fprintf(summary->fp, "        ]\n");
+		}
+
+		fprintf(summary->fp, "      }\n");
+		fprintf(summary->fp, "    }");
+
+		report = report->next;
+	}
+
+	fprintf(summary->fp, "\n");
+	fprintf(summary->fp, "  ]\n");
+	fprintf(summary->fp, "}\n");
+
+	if (fclose(summary->fp) != 0)
+		goto on_error;
+
+	printf("written summary file to %s\n", summary->filename);
+
+	free(summary);
+	return 0;
+
+on_error:
+	fclose(summary->fp);
+	free(summary);
+	return -1;
+}
+
+/* indirection between protocol output selection */
+
+#define SUMMARY(FN, ...) do { \
+		switch (_clar.summary_format) { \
+			case CL_SUMMARY_JUNIT: \
+				return clar_summary_junit_##FN (__VA_ARGS__); \
+				break; \
+			case CL_SUMMARY_JSON: \
+				return clar_summary_json_##FN (__VA_ARGS__); \
+				break; \
+			default: \
+				abort(); \
+		} \
+	} while(0)
+
+struct clar_summary *clar_summary_init(const char *filename)
+{
+	SUMMARY(init, filename);
+}
+
+int clar_summary_shutdown(struct clar_summary *summary)
+{
+	SUMMARY(shutdown, summary);
+}
--- a/tests/clar/clar_libgit2.c
+++ b/tests/clar/clar_libgit2.c
--- a/tests/clar/clar_libgit2.h
+++ b/tests/clar/clar_libgit2.h
--- a/tests/clar/clar_libgit2_alloc.c
+++ b/tests/clar/clar_libgit2_alloc.c
--- a/tests/clar/clar_libgit2_alloc.h
+++ b/tests/clar/clar_libgit2_alloc.h
--- a/tests/clar/clar_libgit2_timer.c
+++ b/tests/clar/clar_libgit2_timer.c
--- a/tests/clar/clar_libgit2_timer.h
+++ b/tests/clar/clar_libgit2_timer.h
--- a/tests/clar/clar_libgit2_trace.c
+++ b/tests/clar/clar_libgit2_trace.c
--- a/tests/clar/clar_libgit2_trace.h
+++ b/tests/clar/clar_libgit2_trace.h
--- a/tests/clar/generate.py
+++ b/tests/clar/generate.py
@@ -17,8 +17,13 @@ class Module(object):

        def _render_callback(self, cb):
            if not cb:
-                return '    { NULL, NULL }'
-            return '    { "%s", &%s }' % (cb['short_name'], cb['symbol'])
+                return '    { NULL, NULL, 0, NULL }'
+
+            return '    { "%s", %s, %d, &%s }' % \
+                (cb['short_name'], \
+                 '"' + cb['description'] + '"' if cb['description'] != None else "NULL", \
+                 cb['runs'], \
+                 cb['symbol'])

    class DeclarationTemplate(Template):
        def render(self):
@@ -27,6 +32,9 @@ class Module(object):
            for initializer in self.module.initializers:
                out += "extern %s;\n" % initializer['declaration']

+            if self.module.reset:
+                out += "extern %s;\n" % self.module.reset['declaration']
+
            if self.module.cleanup:
                out += "extern %s;\n" % self.module.cleanup['declaration']

@@ -34,7 +42,7 @@ class Module(object):

    class CallbacksTemplate(Template):
        def render(self):
-            out = "static const struct clar_func _clar_cb_%s[] = {\n" % self.module.name
+            out = "static const struct %s_func _%s_cb_%s[] = {\n" % (self.module.app_name, self.module.app_name, self.module.name)
            out += ",\n".join(self._render_callback(cb) for cb in self.module.callbacks)
            out += "\n};\n"
            return out
@@ -58,14 +66,16 @@ class Module(object):
    {
        "${clean_name}",
    ${initialize},
+    ${reset},
    ${cleanup},
        ${cb_ptr}, ${cb_count}, ${enabled}
    }"""
                ).substitute(
                    clean_name = name,
                    initialize = self._render_callback(initializer),
+                    reset = self._render_callback(self.module.reset),
                    cleanup = self._render_callback(self.module.cleanup),
-                    cb_ptr = "_clar_cb_%s" % self.module.name,
+                    cb_ptr = "_%s_cb_%s" % (self.module.app_name, self.module.name),
                    cb_count = len(self.module.callbacks),
                    enabled = int(self.module.enabled)
                )
@@ -73,8 +83,10 @@ class Module(object):

            return ','.join(templates)

-    def __init__(self, name):
+    def __init__(self, name, app_name, prefix):
        self.name = name
+        self.app_name = app_name
+        self.prefix = prefix

        self.mtime = None
        self.enabled = True
@@ -85,7 +97,7 @@ class Module(object):

    def _skip_comments(self, text):
        SKIP_COMMENTS_REGEX = re.compile(
-            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
+            r'//.*?$|/\*(?!\s*\[clar\]:).*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
            re.DOTALL | re.MULTILINE)

        def _replacer(match):
@@ -95,24 +107,63 @@ class Module(object):
        return re.sub(SKIP_COMMENTS_REGEX, _replacer, text)

    def parse(self, contents):
-        TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\s*\(\s*void\s*\))\s*\{"
+        TEST_FUNC_REGEX = r"^(void\s+(%s_%s__(\w+))\s*\(\s*void\s*\))(?:\s*/\*\s*\[clar\]:\s*(.*?)\s*\*/)?\s*\{"

        contents = self._skip_comments(contents)
-        regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE)
+        regex = re.compile(TEST_FUNC_REGEX % (self.prefix, self.name), re.MULTILINE)

        self.callbacks = []
        self.initializers = []
+        self.reset = None
        self.cleanup = None

-        for (declaration, symbol, short_name) in regex.findall(contents):
+        for (declaration, symbol, short_name, options) in regex.findall(contents):
+            runs = 0
+            description = None
+
+            while options != '':
+                match = re.search(r'^([a-zA-Z0-9]+)=(\"[^"]*\"|[a-zA-Z0-9_\-]+|\d+)(?:,\s*|\Z)(.*)', options)
+
+                if match == None:
+                    print("Invalid options: '%s' for '%s'" % (options, symbol))
+                    sys.exit(1)
+
+                key = match.group(1)
+                value = match.group(2)
+                options = match.group(3)
+
+                match = re.search(r'^\"(.*)\"$', value)
+                if match != None:
+                    value = match.group(1)
+
+                match = re.search(r'([^a-zA-Z0-9 _\-,\.])', value)
+                if match != None:
+                    print("Invalid character '%s' in %s for '%s'" % (match.group(1), key, symbol))
+                    sys.exit(1)
+
+                if key == "description":
+                    description = value
+                elif key == "runs":
+                    if not value.isnumeric():
+                        print("Invalid option: '%s' in runs for '%s'" % (option, symbol))
+                        sys.exit(1)
+                    runs = int(value)
+                else:
+                    print("Invalid option: '%s' for '%s'" % (key, symbol))
+                    sys.exit(1)
+
            data = {
                "short_name" : short_name,
                "declaration" : declaration,
-                "symbol" : symbol
+                "symbol" : symbol,
+                "description" : description,
+                "runs" : runs
            }

            if short_name.startswith('initialize'):
                self.initializers.append(data)
+            elif short_name == 'reset':
+                self.reset = data
            elif short_name == 'cleanup':
                self.cleanup = data
            else:
@@ -179,8 +230,8 @@ class TestSuite(object):

        return modules

-    def load_cache(self):
-        path = os.path.join(self.output, '.clarcache')
+    def load_cache(self, app_name):
+        path = os.path.join(self.output, ".%scache" % app_name)
        cache = {}

        try:
@@ -192,18 +243,18 @@ class TestSuite(object):

        return cache

-    def save_cache(self):
-        path = os.path.join(self.output, '.clarcache')
+    def save_cache(self, app_name):
+        path = os.path.join(self.output, ".%scache" % app_name)
        with open(path, 'wb') as cache:
            pickle.dump(self.modules, cache)

-    def load(self, force = False):
+    def load(self, app_name, prefix, force = False):
        module_data = self.find_modules()
-        self.modules = {} if force else self.load_cache()
+        self.modules = {} if force else self.load_cache(app_name)

        for path, name in module_data:
            if name not in self.modules:
-                self.modules[name] = Module(name)
+                self.modules[name] = Module(name, app_name, prefix)

            if not self.modules[name].refresh(path):
                del self.modules[name]
@@ -222,15 +273,15 @@ class TestSuite(object):
    def callback_count(self):
        return sum(len(module.callbacks) for module in self.modules.values())

-    def write(self):
+    def write(self, name):
        if not os.path.exists(self.output):
            os.makedirs(self.output)

-        wrote_suite = self.write_suite()
-        wrote_header = self.write_header()
+        wrote_suite = self.write_suite(name)
+        wrote_header = self.write_header(name)

        if wrote_suite or wrote_header:
-            self.save_cache()
+            self.save_cache(name)
            return True

        return False
@@ -257,8 +308,8 @@ class TestSuite(object):

        return True

-    def write_suite(self):
-        suite_fn = os.path.join(self.output, 'clar.suite')
+    def write_suite(self, name):
+        suite_fn = os.path.join(self.output, '%s.suite' % name)

        with io.StringIO() as suite_file:
            modules = sorted(self.modules.values(), key=lambda module: module.name)
@@ -271,25 +322,26 @@ class TestSuite(object):
                t = Module.CallbacksTemplate(module)
                suite_file.write(t.render())

-            suites = "static struct clar_suite _clar_suites[] = {" + ','.join(
+            suites = "static struct %s_suite _%s_suites[] = {" % (name, name)
+            suites += ','.join(
                Module.InfoTemplate(module).render() for module in modules
            ) + "\n};\n"

            suite_file.write(suites)

-            suite_file.write(u"static const size_t _clar_suite_count = %d;\n" % self.suite_count())
-            suite_file.write(u"static const size_t _clar_callback_count = %d;\n" % self.callback_count())
+            suite_file.write(u"static const size_t _%s_suite_count = %d;\n" % (name, self.suite_count()))
+            suite_file.write(u"static const size_t _%s_callback_count = %d;\n" % (name, self.callback_count()))

            return self.write_output(suite_fn, suite_file.getvalue())

        return False

-    def write_header(self):
-        header_fn = os.path.join(self.output, 'clar_suite.h')
+    def write_header(self, name):
+        header_fn = os.path.join(self.output, '%s_suite.h' % name)

        with io.StringIO() as header_file:
-            header_file.write(u"#ifndef _____clar_suite_h_____\n")
-            header_file.write(u"#define _____clar_suite_h_____\n")
+            header_file.write(u"#ifndef _____%s_suite_h_____\n" % name)
+            header_file.write(u"#define _____%s_suite_h_____\n" % name)

            modules = sorted(self.modules.values(), key=lambda module: module.name)

@@ -310,6 +362,8 @@ if __name__ == '__main__':
    parser.add_option('-f', '--force', action="store_true", dest='force', default=False)
    parser.add_option('-x', '--exclude', dest='excluded', action='append', default=[])
    parser.add_option('-o', '--output', dest='output')
+    parser.add_option('-n', '--name', dest='name', default='clar')
+    parser.add_option('-p', '--prefix', dest='prefix', default='test')

    options, args = parser.parse_args()
    if len(args) > 1:
@@ -323,7 +377,8 @@ if __name__ == '__main__':
    output = options.output or path

    suite = TestSuite(path, output)
-    suite.load(options.force)
+    suite.load(options.name, options.prefix, options.force)
    suite.disable(options.excluded)
-    if suite.write():
-        print("Written `clar.suite`, `clar_suite.h` (%d tests in %d suites)" % (suite.callback_count(), suite.suite_count()))
+
+    if suite.write(options.name):
+        print("Written `%s.suite`, `%s_suite.h` (%d tests in %d suites)" % (options.name, options.name, suite.callback_count(), suite.suite_count()))
--- a/tests/clar/main.c
+++ b/tests/clar/main.c
--- a/tests/README.md
+++ b/tests/README.md
@@ -2,10 +2,6 @@

 These are the unit and integration tests for the libgit2 projects.

-* `benchmarks`
-  These are benchmark tests that excercise the CLI.
-* `clar`  
-  This is [clar](https://github.com/clar-test/clar) the common test framework.
 * `headertest`  
  This is a simple project that ensures that our public headers are
  compatible with extremely strict compilation options.
--- a/tests/clar/clar/summary.h
+++ b/tests/clar/clar/summary.h
@@ -1,140 +0,0 @@
-
-#include <stdio.h>
-#include <time.h>
-
-static int clar_summary_close_tag(
-    struct clar_summary *summary, const char *tag, int indent)
-{
-	const char *indt;
-
-	if (indent == 0) indt = "";
-	else if (indent == 1) indt = "\t";
-	else indt = "\t\t";
-
-	return fprintf(summary->fp, "%s</%s>\n", indt, tag);
-}
-
-static int clar_summary_testsuites(struct clar_summary *summary)
-{
-	return fprintf(summary->fp, "<testsuites>\n");
-}
-
-static int clar_summary_testsuite(struct clar_summary *summary,
-    int idn, const char *name, time_t timestamp,
-    int test_count, int fail_count, int error_count)
-{
-	struct tm tm;
-	char iso_dt[20];
-
-	localtime_r(&timestamp, &tm);
-	if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", &tm) == 0)
-		return -1;
-
-	return fprintf(summary->fp, "\t<testsuite"
-		       " id=\"%d\""
-		       " name=\"%s\""
-		       " hostname=\"localhost\""
-		       " timestamp=\"%s\""
-		       " tests=\"%d\""
-		       " failures=\"%d\""
-		       " errors=\"%d\">\n",
-		       idn, name, iso_dt, test_count, fail_count, error_count);
-}
-
-static int clar_summary_testcase(struct clar_summary *summary,
-    const char *name, const char *classname, double elapsed)
-{
-	return fprintf(summary->fp,
-	    "\t\t<testcase name=\"%s\" classname=\"%s\" time=\"%.2f\">\n",
-		name, classname, elapsed);
-}
-
-static int clar_summary_failure(struct clar_summary *summary,
-    const char *type, const char *message, const char *desc)
-{
-	return fprintf(summary->fp,
-	    "\t\t\t<failure type=\"%s\"><![CDATA[%s\n%s]]></failure>\n",
-	    type, message, desc);
-}
-
-static int clar_summary_skipped(struct clar_summary *summary)
-{
-	return fprintf(summary->fp, "\t\t\t<skipped />\n");
-}
-
-struct clar_summary *clar_summary_init(const char *filename)
-{
-	struct clar_summary *summary;
-	FILE *fp;
-
-	if ((fp = fopen(filename, "w")) == NULL)
-		clar_abort("Failed to open the summary file '%s': %s.\n",
-			   filename, strerror(errno));
-
-	if ((summary = malloc(sizeof(struct clar_summary))) == NULL)
-		clar_abort("Failed to allocate summary.\n");
-
-	summary->filename = filename;
-	summary->fp = fp;
-
-	return summary;
-}
-
-int clar_summary_shutdown(struct clar_summary *summary)
-{
-	struct clar_report *report;
-	const char *last_suite = NULL;
-
-	if (clar_summary_testsuites(summary) < 0)
-		goto on_error;
-
-	report = _clar.reports;
-	while (report != NULL) {
-		struct clar_error *error = report->errors;
-
-		if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) {
-			if (clar_summary_testsuite(summary, 0, report->suite,
-			    report->start, _clar.tests_ran, _clar.total_errors, 0) < 0)
-				goto on_error;
-		}
-
-		last_suite = report->suite;
-
-		clar_summary_testcase(summary, report->test, report->suite, report->elapsed);
-
-		while (error != NULL) {
-			if (clar_summary_failure(summary, "assert",
-			    error->error_msg, error->description) < 0)
-				goto on_error;
-
-			error = error->next;
-		}
-
-		if (report->status == CL_TEST_SKIP)
-			clar_summary_skipped(summary);
-
-		if (clar_summary_close_tag(summary, "testcase", 2) < 0)
-			goto on_error;
-
-		report = report->next;
-
-		if (!report || strcmp(last_suite, report->suite) != 0) {
-			if (clar_summary_close_tag(summary, "testsuite", 1) < 0)
-				goto on_error;
-		}
-	}
-
-	if (clar_summary_close_tag(summary, "testsuites", 0) < 0 ||
-	    fclose(summary->fp) != 0)
-		goto on_error;
-
-	printf("written summary file to %s\n", summary->filename);
-
-	free(summary);
-	return 0;
-
-on_error:
-	fclose(summary->fp);
-	free(summary);
-	return -1;
-}
--- a/tests/libgit2/CMakeLists.txt
+++ b/tests/libgit2/CMakeLists.txt
@@ -10,9 +10,9 @@ find_package(PythonInterp)
 if(NOT PYTHONINTERP_FOUND)
 	message(FATAL_ERROR "Could not find a python interpreter, which is needed to build the tests. "
 	                     "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests")
-ENDIF()
+endif()

-set(CLAR_PATH "${PROJECT_SOURCE_DIR}/tests/clar")
+set(CLAR_PATH "${PROJECT_SOURCE_DIR}/deps/clar")
 set(CLAR_FIXTURES "${PROJECT_SOURCE_DIR}/tests/resources/")
 set(TEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}")
 add_definitions(-DCLAR_FIXTURE_PATH=\"${CLAR_FIXTURES}\")
@@ -48,7 +48,11 @@ add_executable(libgit2_tests ${SRC_CLAR} ${SRC_TEST} ${LIBGIT2_OBJECTS})
 set_target_properties(libgit2_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
 target_include_directories(libgit2_tests PRIVATE ${TEST_INCLUDES} ${LIBGIT2_INCLUDES} ${LIBGIT2_DEPENDENCY_INCLUDES})
 target_include_directories(libgit2_tests SYSTEM PRIVATE ${LIBGIT2_SYSTEM_INCLUDES})
+
 target_link_libraries(libgit2_tests ${LIBGIT2_SYSTEM_LIBS})
+if(NOT MSVC_IDE)
+	target_link_libraries(libgit2_tests m)
+endif()

 ide_split_sources(libgit2_tests)

--- a/tests/util/CMakeLists.txt
+++ b/tests/util/CMakeLists.txt
@@ -10,9 +10,9 @@ find_package(PythonInterp)
 if(NOT PYTHONINTERP_FOUND)
 	message(FATAL_ERROR "Could not find a python interpeter, which is needed to build the tests. "
 	                     "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests")
-ENDIF()
+endif()

-set(CLAR_PATH "${libgit2_SOURCE_DIR}/tests/clar")
+set(CLAR_PATH "${libgit2_SOURCE_DIR}/deps/clar")
 set(CLAR_FIXTURES "${libgit2_SOURCE_DIR}/tests/resources/")
 set(TEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}")
 add_definitions(-DCLAR_FIXTURE_PATH=\"${CLAR_FIXTURES}\")
@@ -48,7 +48,11 @@ set_target_properties(util_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${libgit2_B

 target_include_directories(util_tests PRIVATE ${TEST_INCLUDES} ${LIBGIT2_INCLUDES} ${LIBGIT2_DEPENDENCY_INCLUDES})
 target_include_directories(util_tests SYSTEM PRIVATE ${LIBGIT2_SYSTEM_INCLUDES})
+
 target_link_libraries(util_tests ${LIBGIT2_SYSTEM_LIBS})
+if(NOT MSVC_IDE)
+	target_link_libraries(util_tests m)
+endif()

 ide_split_sources(util_tests)