From 9935da4661bb7cf06ca3503a8f9dd08838b2543f Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 19 Jan 2026 13:09:36 +0000 Subject: [PATCH] ci: optionally run benchmarks --- .github/workflows/ab-perf.yml | 229 ++++++++++++++++++++++ .github/workflows/main.yml | 346 ++++++++++++++++------------------ 2 files changed, 388 insertions(+), 187 deletions(-) create mode 100644 .github/workflows/ab-perf.yml diff --git a/.github/workflows/ab-perf.yml b/.github/workflows/ab-perf.yml new file mode 100644 index 000000000..72a3eac3c --- /dev/null +++ b/.github/workflows/ab-perf.yml @@ -0,0 +1,229 @@ +# A/B testing with benchmarks to compare a control branch (main) to a +# candidate branch (a pull request). +name: A/B Performance Test + +on: + workflow_dispatch: + +env: + docker-registry: ghcr.io + docker-config-path: ci/docker + +jobs: + # Run our CI/CD builds. We build a matrix with the various build targets + # and their details. Then we build either in a docker container (Linux) + # or on the actual hosts (macOS, Windows). + build: + strategy: + fail-fast: false + matrix: + platform: + # All builds: core platforms + - name: "Linux (Noble, GCC, OpenSSL, libssh2)" + id: noble-gcc-openssl + os: ubuntu-latest + container: + name: noble + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)" + id: noble-clang-mbedtls + os: ubuntu-latest + container: + name: noble + env: + CC: clang + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + CMAKE_GENERATOR: Ninja + - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" + id: xenial-gcc-openssl + os: ubuntu-latest + container: + name: xenial + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" + id: xenial-gcc-mbedtls + os: ubuntu-latest + container: + name: xenial + env: + CC: clang + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "macOS" + id: macos + os: macos-14 + setup-script: osx + env: + CC: clang + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + CMAKE_GENERATOR: Ninja + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, Visual Studio, Schannel)" + id: windows-amd64-vs + os: windows-2022 + setup-script: win32 + build_prefix: RelWithDebInfo + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, Visual Studio, WinHTTP)" + id: windows-x86-vs + os: windows-2022 + setup-script: win32 + build_prefix: RelWithDebInfo + env: + ARCH: x86 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, mingw, WinHTTP)" + id: windows-amd64-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: amd64 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, mingw, Schannel)" + id: windows-x86-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: x86 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + env: ${{ matrix.platform.env }} + runs-on: ${{ matrix.platform.os }} + name: "A/B: ${{ matrix.platform.name }}" + steps: + - name: Check out control + uses: actions/checkout@v4 + with: + path: source/control + ref: main + - name: Check out candidate + uses: actions/checkout@v4 + with: + path: source/candidate + - name: Set up build environment + run: source/candidate/ci/setup-${{ matrix.platform.setup-script }}-build.sh + shell: bash + if: matrix.platform.setup-script != '' + - name: Setup QEMU + run: docker run --rm --privileged multiarch/qemu-user-static:register --reset + if: matrix.platform.container.qemu == true + - name: Set up container + uses: ./source/candidate/.github/actions/download-or-build-container + with: + registry: ${{ env.docker-registry }} + config-path: ${{ env.docker-config-path }} + container: ${{ matrix.platform.container.name }} + github_token: ${{ secrets.github_token }} + dockerfile: ${{ matrix.platform.container.dockerfile }} + if: matrix.platform.container.name != '' + - name: Prepare builds + run: | + mkdir build + mkdir build/control + mkdir build/candidate + - name: Build control + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ../../source/control/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Build candidate + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ../../source/candidate/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run control benchmarks + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run candidate benchmarks + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Organize results + run: | + mkdir results + mv ${BUILD_WORKSPACE:-.}/build/control/results.json ./results/control.json + mv ${BUILD_WORKSPACE:-.}/build/candidate/results.json ./results/candidate.json + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.platform.id }} + path: results + if: always() + + # Publish the results + publish: + name: Publish results + needs: [ build ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Download test results + uses: actions/download-artifact@v4 + with: + path: results + - name: Generate markdown + run: node ci/compare-benchmarks.js results results.md + - name: Update pull request + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const markdown = fs.readFileSync('results.md'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: markdown + }); diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4ca339806..b6ad39d06 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,28 +1,21 @@ -# Continuous integration and pull request validation builds for the -# main and maintenance branches. -name: CI Build +# A/B testing with benchmarks to compare a control branch (main) to a +# candidate branch (a pull request). +name: A/B Performance Test on: - push: - branches: [ main, maint/* ] - pull_request: - branches: [ main, maint/* ] workflow_dispatch: env: docker-registry: ghcr.io docker-config-path: ci/docker -permissions: - contents: write - packages: write - jobs: # Run our CI/CD builds. We build a matrix with the various build targets # and their details. Then we build either in a docker container (Linux) # or on the actual hosts (macOS, Windows). build: strategy: + fail-fast: false matrix: platform: # All builds: core platforms @@ -34,7 +27,8 @@ jobs: env: CC: gcc CMAKE_GENERATOR: Ninja - CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)" id: noble-clang-mbedtls os: ubuntu-latest @@ -42,167 +36,118 @@ jobs: name: noble env: CC: clang - CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo CMAKE_GENERATOR: Ninja - - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" - id: xenial-gcc-openssl - os: ubuntu-latest - container: - name: xenial - env: - CC: gcc - CMAKE_GENERATOR: Ninja - CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON - - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" - id: xenial-gcc-mbedtls - os: ubuntu-latest - container: - name: xenial - env: - CC: clang - CMAKE_GENERATOR: Ninja - CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 +# - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" +# id: xenial-gcc-openssl +# os: ubuntu-latest +# container: +# name: xenial +# env: +# CC: gcc +# CMAKE_GENERATOR: Ninja +# CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" +# id: xenial-gcc-mbedtls +# os: ubuntu-latest +# container: +# name: xenial +# env: +# CC: clang +# CMAKE_GENERATOR: Ninja +# CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo - name: "macOS" id: macos os: macos-14 setup-script: osx env: CC: clang - CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo CMAKE_GENERATOR: Ninja PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig SKIP_SSH_TESTS: true SKIP_NEGOTIATE_TESTS: true - - name: "Windows (amd64, Visual Studio, Schannel)" - id: windows-amd64-vs - os: windows-2022 - setup-script: win32 - env: - ARCH: amd64 - CMAKE_GENERATOR: Visual Studio 17 2022 - CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 - BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin - BUILD_TEMP: D:\Temp - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - name: "Windows (x86, Visual Studio, WinHTTP)" - id: windows-x86-vs - os: windows-2022 - setup-script: win32 - env: - ARCH: x86 - CMAKE_GENERATOR: Visual Studio 17 2022 - CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 - BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin - BUILD_TEMP: D:\Temp - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - name: "Windows (amd64, mingw, WinHTTP)" - id: windows-amd64-mingw - os: windows-2022 - setup-script: mingw - env: - ARCH: amd64 - CMAKE_GENERATOR: MinGW Makefiles - CMAKE_OPTIONS: -DDEPRECATE_HARD=ON - BUILD_TEMP: D:\Temp - BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - name: "Windows (x86, mingw, Schannel)" - id: windows-x86-mingw - os: windows-2022 - setup-script: mingw - env: - ARCH: x86 - CMAKE_GENERATOR: MinGW Makefiles - CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel - BUILD_TEMP: D:\Temp - BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - # All builds: sanitizers - - name: "Sanitizer (Memory)" - id: sanitizer-memory - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=memory -fsanitize-memory-track-origins=2 -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_C_EXTENSIONS=ON -DCMAKE_PREFIX_PATH=/usr/local/msan -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - - name: "Sanitizer (Address)" - id: sanitizer-address - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=address -ggdb -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - - name: "Sanitizer (UndefinedBehavior)" - id: sanitizer-ub - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=undefined,nullability -fno-sanitize-recover=undefined,nullability -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - - name: "Sanitizer (Thread)" - id: sanitizer-thread - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=thread -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1 - fail-fast: false +# - name: "Windows (amd64, Visual Studio, Schannel)" +# id: windows-amd64-vs +# os: windows-2022 +# setup-script: win32 +# build_prefix: RelWithDebInfo +# env: +# ARCH: amd64 +# CMAKE_GENERATOR: Visual Studio 17 2022 +# CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin +# BUILD_TEMP: D:\Temp +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true +# - name: "Windows (x86, Visual Studio, WinHTTP)" +# id: windows-x86-vs +# os: windows-2022 +# setup-script: win32 +# build_prefix: RelWithDebInfo +# env: +# ARCH: x86 +# CMAKE_GENERATOR: Visual Studio 17 2022 +# CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin +# BUILD_TEMP: D:\Temp +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true +# - name: "Windows (amd64, mingw, WinHTTP)" +# id: windows-amd64-mingw +# os: windows-2022 +# setup-script: mingw +# env: +# ARCH: amd64 +# CMAKE_GENERATOR: MinGW Makefiles +# CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_TEMP: D:\Temp +# BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true +# - name: "Windows (x86, mingw, Schannel)" +# id: windows-x86-mingw +# os: windows-2022 +# setup-script: mingw +# env: +# ARCH: x86 +# CMAKE_GENERATOR: MinGW Makefiles +# CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_TEMP: D:\Temp +# BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true env: ${{ matrix.platform.env }} runs-on: ${{ matrix.platform.os }} - name: "Build: ${{ matrix.platform.name }}" + name: "A/B: ${{ matrix.platform.name }}" steps: - - name: Check out repository + - name: Check out control uses: actions/checkout@v4 with: - path: source - fetch-depth: 0 + path: source/control +# TODO: there are no control benchmarks in main yet +# ref: main + - name: Check out candidate + uses: actions/checkout@v4 + with: + path: source/candidate - name: Set up build environment - run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh + run: source/candidate/ci/setup-${{ matrix.platform.setup-script }}-build.sh shell: bash if: matrix.platform.setup-script != '' - name: Setup QEMU run: docker run --rm --privileged multiarch/qemu-user-static:register --reset if: matrix.platform.container.qemu == true - name: Set up container - uses: ./source/.github/actions/download-or-build-container + uses: ./source/candidate/.github/actions/download-or-build-container with: registry: ${{ env.docker-registry }} config-path: ${{ env.docker-config-path }} @@ -210,49 +155,76 @@ jobs: github_token: ${{ secrets.github_token }} dockerfile: ${{ matrix.platform.container.dockerfile }} if: matrix.platform.container.name != '' - - name: Prepare build - run: mkdir build - - name: Build - uses: ./source/.github/actions/run-build - with: - command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/build.sh - container: ${{ matrix.platform.container.name }} - container-version: ${{ env.docker-registry-container-sha }} - shell: ${{ matrix.platform.shell }} - - name: Test - uses: ./source/.github/actions/run-build - with: - command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/test.sh - container: ${{ matrix.platform.container.name }} - container-version: ${{ env.docker-registry-container-sha }} - shell: ${{ matrix.platform.shell }} - - name: Upload test results - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: test-results-${{ matrix.platform.id }} - path: build/results_*.xml - - documentation: - name: Validate documentation - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v4 - - name: Validate documentation + - name: Prepare builds run: | - (cd script/api-docs && npm install) - script/api-docs/api-generator.js --validate-only --strict --deprecate-hard . + mkdir build + mkdir build/control + mkdir build/candidate + - name: Build control + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ../../source/control/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Build candidate + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ../../source/candidate/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run control benchmarks + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run candidate benchmarks + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Organize results + run: | + mkdir results + mv ${BUILD_WORKSPACE:-.}/build/control/results.json ./results/control.json + mv ${BUILD_WORKSPACE:-.}/build/candidate/results.json ./results/candidate.json + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.platform.id }} + path: results + if: always() - test_results: - name: Test results + # Publish the results + publish: + name: Publish results needs: [ build ] if: always() runs-on: ubuntu-latest steps: + - name: Check out repository + uses: actions/checkout@v4 - name: Download test results uses: actions/download-artifact@v4 - - name: Generate test summary - uses: test-summary/action@v2 with: - paths: 'test-results-*/*.xml' + path: results + - name: Generate markdown + run: node ci/compare-benchmarks.js results results.md + - name: Update pull request + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const markdown = fs.readFileSync('results.md'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: markdown + });