diff --git a/.github/workflows/ab-perf.yml b/.github/workflows/ab-perf.yml new file mode 100644 index 000000000..72a3eac3c --- /dev/null +++ b/.github/workflows/ab-perf.yml @@ -0,0 +1,229 @@ +# A/B testing with benchmarks to compare a control branch (main) to a +# candidate branch (a pull request). +name: A/B Performance Test + +on: + workflow_dispatch: + +env: + docker-registry: ghcr.io + docker-config-path: ci/docker + +jobs: + # Run our CI/CD builds. We build a matrix with the various build targets + # and their details. Then we build either in a docker container (Linux) + # or on the actual hosts (macOS, Windows). + build: + strategy: + fail-fast: false + matrix: + platform: + # All builds: core platforms + - name: "Linux (Noble, GCC, OpenSSL, libssh2)" + id: noble-gcc-openssl + os: ubuntu-latest + container: + name: noble + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)" + id: noble-clang-mbedtls + os: ubuntu-latest + container: + name: noble + env: + CC: clang + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + CMAKE_GENERATOR: Ninja + - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" + id: xenial-gcc-openssl + os: ubuntu-latest + container: + name: xenial + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" + id: xenial-gcc-mbedtls + os: ubuntu-latest + container: + name: xenial + env: + CC: clang + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "macOS" + id: macos + os: macos-14 + setup-script: osx + env: + CC: clang + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + CMAKE_GENERATOR: Ninja + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, Visual Studio, Schannel)" + id: windows-amd64-vs + os: windows-2022 + setup-script: win32 + build_prefix: RelWithDebInfo + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, Visual Studio, WinHTTP)" + id: windows-x86-vs + os: windows-2022 + setup-script: win32 + build_prefix: RelWithDebInfo + env: + ARCH: x86 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, mingw, WinHTTP)" + id: windows-amd64-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: amd64 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, mingw, Schannel)" + id: windows-x86-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: x86 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + env: ${{ matrix.platform.env }} + runs-on: ${{ matrix.platform.os }} + name: "A/B: ${{ matrix.platform.name }}" + steps: + - name: Check out control + uses: actions/checkout@v4 + with: + path: source/control + ref: main + - name: Check out candidate + uses: actions/checkout@v4 + with: + path: source/candidate + - name: Set up build environment + run: source/candidate/ci/setup-${{ matrix.platform.setup-script }}-build.sh + shell: bash + if: matrix.platform.setup-script != '' + - name: Setup QEMU + run: docker run --rm --privileged multiarch/qemu-user-static:register --reset + if: matrix.platform.container.qemu == true + - name: Set up container + uses: ./source/candidate/.github/actions/download-or-build-container + with: + registry: ${{ env.docker-registry }} + config-path: ${{ env.docker-config-path }} + container: ${{ matrix.platform.container.name }} + github_token: ${{ secrets.github_token }} + dockerfile: ${{ matrix.platform.container.dockerfile }} + if: matrix.platform.container.name != '' + - name: Prepare builds + run: | + mkdir build + mkdir build/control + mkdir build/candidate + - name: Build control + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ../../source/control/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Build candidate + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ../../source/candidate/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run control benchmarks + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run candidate benchmarks + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Organize results + run: | + mkdir results + mv ${BUILD_WORKSPACE:-.}/build/control/results.json ./results/control.json + mv ${BUILD_WORKSPACE:-.}/build/candidate/results.json ./results/candidate.json + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.platform.id }} + path: results + if: always() + + # Publish the results + publish: + name: Publish results + needs: [ build ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Download test results + uses: actions/download-artifact@v4 + with: + path: results + - name: Generate markdown + run: node ci/compare-benchmarks.js results results.md + - name: Update pull request + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const markdown = fs.readFileSync('results.md'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: markdown + }); diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 103f4bcd0..1ff7178e9 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -115,7 +115,7 @@ jobs: fi mkdir benchmark && cd benchmark - ../source/tests/benchmarks/benchmark.sh \ + ../source/benchmarks/cli/benchmark.sh \ ${SUITE_FLAG} ${DEBUG_FLAG} \ --admin \ --baseline-cli "git" --cli "${GIT2_CLI}" --name libgit2 \ diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4ca339806..b6ad39d06 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,28 +1,21 @@ -# Continuous integration and pull request validation builds for the -# main and maintenance branches. -name: CI Build +# A/B testing with benchmarks to compare a control branch (main) to a +# candidate branch (a pull request). +name: A/B Performance Test on: - push: - branches: [ main, maint/* ] - pull_request: - branches: [ main, maint/* ] workflow_dispatch: env: docker-registry: ghcr.io docker-config-path: ci/docker -permissions: - contents: write - packages: write - jobs: # Run our CI/CD builds. We build a matrix with the various build targets # and their details. Then we build either in a docker container (Linux) # or on the actual hosts (macOS, Windows). build: strategy: + fail-fast: false matrix: platform: # All builds: core platforms @@ -34,7 +27,8 @@ jobs: env: CC: gcc CMAKE_GENERATOR: Ninja - CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)" id: noble-clang-mbedtls os: ubuntu-latest @@ -42,167 +36,118 @@ jobs: name: noble env: CC: clang - CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo CMAKE_GENERATOR: Ninja - - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" - id: xenial-gcc-openssl - os: ubuntu-latest - container: - name: xenial - env: - CC: gcc - CMAKE_GENERATOR: Ninja - CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON - - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" - id: xenial-gcc-mbedtls - os: ubuntu-latest - container: - name: xenial - env: - CC: clang - CMAKE_GENERATOR: Ninja - CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 +# - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" +# id: xenial-gcc-openssl +# os: ubuntu-latest +# container: +# name: xenial +# env: +# CC: gcc +# CMAKE_GENERATOR: Ninja +# CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" +# id: xenial-gcc-mbedtls +# os: ubuntu-latest +# container: +# name: xenial +# env: +# CC: clang +# CMAKE_GENERATOR: Ninja +# CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo - name: "macOS" id: macos os: macos-14 setup-script: osx env: CC: clang - CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo CMAKE_GENERATOR: Ninja PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig SKIP_SSH_TESTS: true SKIP_NEGOTIATE_TESTS: true - - name: "Windows (amd64, Visual Studio, Schannel)" - id: windows-amd64-vs - os: windows-2022 - setup-script: win32 - env: - ARCH: amd64 - CMAKE_GENERATOR: Visual Studio 17 2022 - CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 - BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin - BUILD_TEMP: D:\Temp - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - name: "Windows (x86, Visual Studio, WinHTTP)" - id: windows-x86-vs - os: windows-2022 - setup-script: win32 - env: - ARCH: x86 - CMAKE_GENERATOR: Visual Studio 17 2022 - CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 - BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin - BUILD_TEMP: D:\Temp - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - name: "Windows (amd64, mingw, WinHTTP)" - id: windows-amd64-mingw - os: windows-2022 - setup-script: mingw - env: - ARCH: amd64 - CMAKE_GENERATOR: MinGW Makefiles - CMAKE_OPTIONS: -DDEPRECATE_HARD=ON - BUILD_TEMP: D:\Temp - BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - name: "Windows (x86, mingw, Schannel)" - id: windows-x86-mingw - os: windows-2022 - setup-script: mingw - env: - ARCH: x86 - CMAKE_GENERATOR: MinGW Makefiles - CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel - BUILD_TEMP: D:\Temp - BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - # All builds: sanitizers - - name: "Sanitizer (Memory)" - id: sanitizer-memory - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=memory -fsanitize-memory-track-origins=2 -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_C_EXTENSIONS=ON -DCMAKE_PREFIX_PATH=/usr/local/msan -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - - name: "Sanitizer (Address)" - id: sanitizer-address - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=address -ggdb -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - - name: "Sanitizer (UndefinedBehavior)" - id: sanitizer-ub - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=undefined,nullability -fno-sanitize-recover=undefined,nullability -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - - name: "Sanitizer (Thread)" - id: sanitizer-thread - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=thread -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1 - fail-fast: false +# - name: "Windows (amd64, Visual Studio, Schannel)" +# id: windows-amd64-vs +# os: windows-2022 +# setup-script: win32 +# build_prefix: RelWithDebInfo +# env: +# ARCH: amd64 +# CMAKE_GENERATOR: Visual Studio 17 2022 +# CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin +# BUILD_TEMP: D:\Temp +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true +# - name: "Windows (x86, Visual Studio, WinHTTP)" +# id: windows-x86-vs +# os: windows-2022 +# setup-script: win32 +# build_prefix: RelWithDebInfo +# env: +# ARCH: x86 +# CMAKE_GENERATOR: Visual Studio 17 2022 +# CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin +# BUILD_TEMP: D:\Temp +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true +# - name: "Windows (amd64, mingw, WinHTTP)" +# id: windows-amd64-mingw +# os: windows-2022 +# setup-script: mingw +# env: +# ARCH: amd64 +# CMAKE_GENERATOR: MinGW Makefiles +# CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_TEMP: D:\Temp +# BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true +# - name: "Windows (x86, mingw, Schannel)" +# id: windows-x86-mingw +# os: windows-2022 +# setup-script: mingw +# env: +# ARCH: x86 +# CMAKE_GENERATOR: MinGW Makefiles +# CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_TEMP: D:\Temp +# BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true env: ${{ matrix.platform.env }} runs-on: ${{ matrix.platform.os }} - name: "Build: ${{ matrix.platform.name }}" + name: "A/B: ${{ matrix.platform.name }}" steps: - - name: Check out repository + - name: Check out control uses: actions/checkout@v4 with: - path: source - fetch-depth: 0 + path: source/control +# TODO: there are no control benchmarks in main yet +# ref: main + - name: Check out candidate + uses: actions/checkout@v4 + with: + path: source/candidate - name: Set up build environment - run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh + run: source/candidate/ci/setup-${{ matrix.platform.setup-script }}-build.sh shell: bash if: matrix.platform.setup-script != '' - name: Setup QEMU run: docker run --rm --privileged multiarch/qemu-user-static:register --reset if: matrix.platform.container.qemu == true - name: Set up container - uses: ./source/.github/actions/download-or-build-container + uses: ./source/candidate/.github/actions/download-or-build-container with: registry: ${{ env.docker-registry }} config-path: ${{ env.docker-config-path }} @@ -210,49 +155,76 @@ jobs: github_token: ${{ secrets.github_token }} dockerfile: ${{ matrix.platform.container.dockerfile }} if: matrix.platform.container.name != '' - - name: Prepare build - run: mkdir build - - name: Build - uses: ./source/.github/actions/run-build - with: - command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/build.sh - container: ${{ matrix.platform.container.name }} - container-version: ${{ env.docker-registry-container-sha }} - shell: ${{ matrix.platform.shell }} - - name: Test - uses: ./source/.github/actions/run-build - with: - command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/test.sh - container: ${{ matrix.platform.container.name }} - container-version: ${{ env.docker-registry-container-sha }} - shell: ${{ matrix.platform.shell }} - - name: Upload test results - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: test-results-${{ matrix.platform.id }} - path: build/results_*.xml - - documentation: - name: Validate documentation - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v4 - - name: Validate documentation + - name: Prepare builds run: | - (cd script/api-docs && npm install) - script/api-docs/api-generator.js --validate-only --strict --deprecate-hard . + mkdir build + mkdir build/control + mkdir build/candidate + - name: Build control + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ../../source/control/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Build candidate + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ../../source/candidate/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run control benchmarks + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run candidate benchmarks + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Organize results + run: | + mkdir results + mv ${BUILD_WORKSPACE:-.}/build/control/results.json ./results/control.json + mv ${BUILD_WORKSPACE:-.}/build/candidate/results.json ./results/candidate.json + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.platform.id }} + path: results + if: always() - test_results: - name: Test results + # Publish the results + publish: + name: Publish results needs: [ build ] if: always() runs-on: ubuntu-latest steps: + - name: Check out repository + uses: actions/checkout@v4 - name: Download test results uses: actions/download-artifact@v4 - - name: Generate test summary - uses: test-summary/action@v2 with: - paths: 'test-results-*/*.xml' + path: results + - name: Generate markdown + run: node ci/compare-benchmarks.js results results.md + - name: Update pull request + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const markdown = fs.readFileSync('results.md'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: markdown + }); diff --git a/CMakeLists.txt b/CMakeLists.txt index 335901d1f..3e9747575 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,8 @@ option(EXPERIMENTAL_SHA256 "Enable experimental SHA256 support (for R&D/test # Optional subsystems option(BUILD_SHARED_LIBS "Build Shared Library (OFF for Static)" ON) -option(BUILD_TESTS "Build Tests using the Clar suite" ON) +option(BUILD_TESTS "Build the test suite" ON) +option(BUILD_BENCHMARKS "Build the benchmark suite" OFF) option(BUILD_CLI "Build the command-line interface" ON) option(BUILD_EXAMPLES "Build library usage example apps" OFF) option(BUILD_FUZZERS "Build the fuzz targets" OFF) @@ -112,6 +113,10 @@ if(BUILD_TESTS) add_subdirectory(tests) endif() +if(BUILD_BENCHMARKS) + add_subdirectory(benchmarks) +endif() + if(BUILD_EXAMPLES) add_subdirectory(examples) endif() diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt new file mode 100644 index 000000000..1b31354e5 --- /dev/null +++ b/benchmarks/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(libgit2) diff --git a/tests/benchmarks/README.md b/benchmarks/cli/README.md similarity index 100% rename from tests/benchmarks/README.md rename to benchmarks/cli/README.md diff --git a/tests/benchmarks/_script/flamegraph/README.md b/benchmarks/cli/_script/flamegraph/README.md similarity index 100% rename from tests/benchmarks/_script/flamegraph/README.md rename to benchmarks/cli/_script/flamegraph/README.md diff --git a/tests/benchmarks/_script/flamegraph/aix-perf.pl b/benchmarks/cli/_script/flamegraph/aix-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/aix-perf.pl rename to benchmarks/cli/_script/flamegraph/aix-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/difffolded.pl b/benchmarks/cli/_script/flamegraph/difffolded.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/difffolded.pl rename to benchmarks/cli/_script/flamegraph/difffolded.pl diff --git a/tests/benchmarks/_script/flamegraph/example-dtrace-stacks.txt b/benchmarks/cli/_script/flamegraph/example-dtrace-stacks.txt similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-dtrace-stacks.txt rename to benchmarks/cli/_script/flamegraph/example-dtrace-stacks.txt diff --git a/tests/benchmarks/_script/flamegraph/example-dtrace.svg b/benchmarks/cli/_script/flamegraph/example-dtrace.svg similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-dtrace.svg rename to benchmarks/cli/_script/flamegraph/example-dtrace.svg diff --git a/tests/benchmarks/_script/flamegraph/example-perf-stacks.txt.gz b/benchmarks/cli/_script/flamegraph/example-perf-stacks.txt.gz similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-perf-stacks.txt.gz rename to benchmarks/cli/_script/flamegraph/example-perf-stacks.txt.gz diff --git a/tests/benchmarks/_script/flamegraph/example-perf.svg b/benchmarks/cli/_script/flamegraph/example-perf.svg similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-perf.svg rename to benchmarks/cli/_script/flamegraph/example-perf.svg diff --git a/tests/benchmarks/_script/flamegraph/files.pl b/benchmarks/cli/_script/flamegraph/files.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/files.pl rename to benchmarks/cli/_script/flamegraph/files.pl diff --git a/tests/benchmarks/_script/flamegraph/flamegraph.pl b/benchmarks/cli/_script/flamegraph/flamegraph.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/flamegraph.pl rename to benchmarks/cli/_script/flamegraph/flamegraph.pl diff --git a/tests/benchmarks/_script/flamegraph/jmaps b/benchmarks/cli/_script/flamegraph/jmaps similarity index 100% rename from tests/benchmarks/_script/flamegraph/jmaps rename to benchmarks/cli/_script/flamegraph/jmaps diff --git a/tests/benchmarks/_script/flamegraph/pkgsplit-perf.pl b/benchmarks/cli/_script/flamegraph/pkgsplit-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/pkgsplit-perf.pl rename to benchmarks/cli/_script/flamegraph/pkgsplit-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/range-perf.pl b/benchmarks/cli/_script/flamegraph/range-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/range-perf.pl rename to benchmarks/cli/_script/flamegraph/range-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/record-test.sh b/benchmarks/cli/_script/flamegraph/record-test.sh similarity index 100% rename from tests/benchmarks/_script/flamegraph/record-test.sh rename to benchmarks/cli/_script/flamegraph/record-test.sh diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-aix.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-aix.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-aix.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-aix.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-bpftrace.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-bpftrace.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-bpftrace.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-bpftrace.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-chrome-tracing.py b/benchmarks/cli/_script/flamegraph/stackcollapse-chrome-tracing.py similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-chrome-tracing.py rename to benchmarks/cli/_script/flamegraph/stackcollapse-chrome-tracing.py diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-elfutils.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-elfutils.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-elfutils.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-elfutils.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-faulthandler.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-faulthandler.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-faulthandler.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-faulthandler.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-gdb.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-gdb.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-gdb.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-gdb.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-go.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-go.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-go.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-go.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-ibmjava.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-ibmjava.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-ibmjava.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-ibmjava.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-instruments.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-instruments.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-instruments.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-instruments.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-java-exceptions.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-java-exceptions.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-java-exceptions.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-java-exceptions.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-jstack.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-jstack.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-jstack.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-jstack.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-ljp.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-ljp.awk similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-ljp.awk rename to benchmarks/cli/_script/flamegraph/stackcollapse-ljp.awk diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-perf-sched.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-perf-sched.awk similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-perf-sched.awk rename to benchmarks/cli/_script/flamegraph/stackcollapse-perf-sched.awk diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-perf.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-perf.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-pmc.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-pmc.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-pmc.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-pmc.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-recursive.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-recursive.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-recursive.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-recursive.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-sample.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-sample.awk similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-sample.awk rename to benchmarks/cli/_script/flamegraph/stackcollapse-sample.awk diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-stap.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-stap.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-stap.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-stap.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-vsprof.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vsprof.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-vsprof.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-vsprof.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-vtune-mc.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vtune-mc.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-vtune-mc.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-vtune-mc.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-vtune.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vtune.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-vtune.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-vtune.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-wcp.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-wcp.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-wcp.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-wcp.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-xdebug.php b/benchmarks/cli/_script/flamegraph/stackcollapse-xdebug.php similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-xdebug.php rename to benchmarks/cli/_script/flamegraph/stackcollapse-xdebug.php diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse.pl b/benchmarks/cli/_script/flamegraph/stackcollapse.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse.pl diff --git a/tests/benchmarks/_script/flamegraph/test.sh b/benchmarks/cli/_script/flamegraph/test.sh similarity index 100% rename from tests/benchmarks/_script/flamegraph/test.sh rename to benchmarks/cli/_script/flamegraph/test.sh diff --git a/tests/benchmarks/benchmark.sh b/benchmarks/cli/benchmark.sh similarity index 100% rename from tests/benchmarks/benchmark.sh rename to benchmarks/cli/benchmark.sh diff --git a/tests/benchmarks/benchmark_helpers.sh b/benchmarks/cli/benchmark_helpers.sh similarity index 99% rename from tests/benchmarks/benchmark_helpers.sh rename to benchmarks/cli/benchmark_helpers.sh index cf0cd5121..f78aabc09 100644 --- a/tests/benchmarks/benchmark_helpers.sh +++ b/benchmarks/cli/benchmark_helpers.sh @@ -105,7 +105,7 @@ fullpath() { } resources_dir() { - cd "$(dirname "$0")/../resources" && pwd + cd "$(dirname "$0")/../../tests/resources" && pwd } temp_dir() { diff --git a/tests/benchmarks/blame__git b/benchmarks/cli/blame__git similarity index 100% rename from tests/benchmarks/blame__git rename to benchmarks/cli/blame__git diff --git a/tests/benchmarks/blame__linux b/benchmarks/cli/blame__linux similarity index 100% rename from tests/benchmarks/blame__linux rename to benchmarks/cli/blame__linux diff --git a/tests/benchmarks/blame__simple b/benchmarks/cli/blame__simple similarity index 100% rename from tests/benchmarks/blame__simple rename to benchmarks/cli/blame__simple diff --git a/tests/benchmarks/hash-object__text_100kb b/benchmarks/cli/hash-object__text_100kb similarity index 100% rename from tests/benchmarks/hash-object__text_100kb rename to benchmarks/cli/hash-object__text_100kb diff --git a/tests/benchmarks/hash-object__text_10mb b/benchmarks/cli/hash-object__text_10mb similarity index 100% rename from tests/benchmarks/hash-object__text_10mb rename to benchmarks/cli/hash-object__text_10mb diff --git a/tests/benchmarks/hash-object__text_1kb b/benchmarks/cli/hash-object__text_1kb similarity index 100% rename from tests/benchmarks/hash-object__text_1kb rename to benchmarks/cli/hash-object__text_1kb diff --git a/tests/benchmarks/hash-object__text_nocache_100kb b/benchmarks/cli/hash-object__text_nocache_100kb similarity index 100% rename from tests/benchmarks/hash-object__text_nocache_100kb rename to benchmarks/cli/hash-object__text_nocache_100kb diff --git a/tests/benchmarks/hash-object__text_nocache_10mb b/benchmarks/cli/hash-object__text_nocache_10mb similarity index 100% rename from tests/benchmarks/hash-object__text_nocache_10mb rename to benchmarks/cli/hash-object__text_nocache_10mb diff --git a/tests/benchmarks/hash-object__text_nocache_1kb b/benchmarks/cli/hash-object__text_nocache_1kb similarity index 100% rename from tests/benchmarks/hash-object__text_nocache_1kb rename to benchmarks/cli/hash-object__text_nocache_1kb diff --git a/tests/benchmarks/hash-object__write_text_100kb b/benchmarks/cli/hash-object__write_text_100kb similarity index 100% rename from tests/benchmarks/hash-object__write_text_100kb rename to benchmarks/cli/hash-object__write_text_100kb diff --git a/tests/benchmarks/hash-object__write_text_10mb b/benchmarks/cli/hash-object__write_text_10mb similarity index 100% rename from tests/benchmarks/hash-object__write_text_10mb rename to benchmarks/cli/hash-object__write_text_10mb diff --git a/tests/benchmarks/hash-object__write_text_1kb b/benchmarks/cli/hash-object__write_text_1kb similarity index 100% rename from tests/benchmarks/hash-object__write_text_1kb rename to benchmarks/cli/hash-object__write_text_1kb diff --git a/tests/benchmarks/hash-object__write_text_nocache_100kb b/benchmarks/cli/hash-object__write_text_nocache_100kb similarity index 100% rename from tests/benchmarks/hash-object__write_text_nocache_100kb rename to benchmarks/cli/hash-object__write_text_nocache_100kb diff --git a/tests/benchmarks/hash-object__write_text_nocache_10mb b/benchmarks/cli/hash-object__write_text_nocache_10mb similarity index 100% rename from tests/benchmarks/hash-object__write_text_nocache_10mb rename to benchmarks/cli/hash-object__write_text_nocache_10mb diff --git a/tests/benchmarks/hash-object__write_text_nocache_1kb b/benchmarks/cli/hash-object__write_text_nocache_1kb similarity index 100% rename from tests/benchmarks/hash-object__write_text_nocache_1kb rename to benchmarks/cli/hash-object__write_text_nocache_1kb diff --git a/tests/benchmarks/indexpack__250mb b/benchmarks/cli/indexpack__250mb similarity index 100% rename from tests/benchmarks/indexpack__250mb rename to benchmarks/cli/indexpack__250mb diff --git a/benchmarks/libgit2/CMakeLists.txt b/benchmarks/libgit2/CMakeLists.txt new file mode 100644 index 000000000..e2668b25e --- /dev/null +++ b/benchmarks/libgit2/CMakeLists.txt @@ -0,0 +1,78 @@ +# util: the unit tests for libgit2's utility library + +if(NOT "${CMAKE_VERSION}" VERSION_LESS 3.27) + cmake_policy(SET CMP0148 OLD) +endif() + +set(Python_ADDITIONAL_VERSIONS 3 2.7) +find_package(PythonInterp) + +if(NOT PYTHONINTERP_FOUND) + message(FATAL_ERROR "Could not find a python interpeter, which is needed to build the tests. " + "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests") +endif() + +set(CLAR_PATH "${PROJECT_SOURCE_DIR}/deps/clar") +set(BENCHMARK_PATH "${CMAKE_CURRENT_SOURCE_DIR}") +add_definitions(-DCLAR_TMPDIR=\"libgit2_bench\") +add_definitions(-DCLAR_WIN32_LONGPATHS) +add_definitions(-DCLAR_HAS_REALPATH) +add_definitions(-D_FILE_OFFSET_BITS=64) + +file(GLOB BENCHMARK_SRC *.c *.h) +list(SORT BENCHMARK_SRC) + +set(CLAR_SRC + "${CLAR_PATH}/clar.c" + "${CLAR_PATH}/clar.h" + "${CLAR_PATH}/clar/fixtures.h" + "${CLAR_PATH}/clar/print.h" + "${CLAR_PATH}/clar/summary.h" + "${CLAR_PATH}/clar/sandbox.h" + "${CLAR_PATH}/clar/fs.h") + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/clar.suite ${CMAKE_CURRENT_BINARY_DIR}/clar_suite.h + COMMAND ${PYTHON_EXECUTABLE} ${CLAR_PATH}/generate.py -p benchmark -o "${CMAKE_CURRENT_BINARY_DIR}" -f . + DEPENDS ${BENCHMARK_SRC} + WORKING_DIRECTORY ${BENCHMARK_PATH} +) + +set_source_files_properties( + ${CLAR_PATH}/clar.c + PROPERTIES OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clar.suite) + +add_executable(libgit2_benchmarks ${CLAR_SRC} + ${BENCHMARK_SRC} + $ + ${LIBGIT2_DEPENDENCY_OBJECTS}) + +target_link_libraries(libgit2_benchmarks libgit2package ${LIBGIT2_SYSTEM_LIBS}) +if(NOT MSVC_IDE) + target_link_libraries(libgit2_benchmarks m) +endif() + +ide_split_sources(libgit2_benchmarks) + +target_include_directories(libgit2_benchmarks PRIVATE + "${CLAR_PATH}" + "${libgit2_BINARY_DIR}/src/util" + "${libgit2_BINARY_DIR}/include" + "${libgit2_SOURCE_DIR}/src/util" + "${libgit2_SOURCE_DIR}/include" + "${CMAKE_CURRENT_BINARY_DIR}" + "${LIBGIT2_DEPENDENCY_INCLUDES}" + "${LIBGIT2_SYSTEM_INCLUDES}") + +# +# Old versions of gcc require us to declare our test functions; don't do +# this on newer compilers to avoid unnecessary recompilation. +# +if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0) + target_compile_options(libgit2_benchmarks PRIVATE -include "clar_suite.h") +endif() + +if(MSVC_IDE) + set_target_properties(libgit2_benchmarks PROPERTIES COMPILE_FLAGS "/Yuprecompiled.h /FIprecompiled.h") + set_source_files_properties("precompiled.c" COMPILE_FLAGS "/Ycprecompiled.h") +endif() diff --git a/benchmarks/libgit2/main.c b/benchmarks/libgit2/main.c new file mode 100644 index 000000000..a8f680f6d --- /dev/null +++ b/benchmarks/libgit2/main.c @@ -0,0 +1,31 @@ +#include + +#include "clar.h" +#include + +#ifdef _WIN32 +int __cdecl main(int argc, char *argv[]) +#else +int main(int argc, char *argv[]) +#endif +{ + int res; + + clar_test_set_mode(CL_TEST_BENCHMARK); + clar_test_init(argc, argv); + + res = git_libgit2_init(); + if (res < 0) { + const git_error *err = git_error_last(); + const char *msg = err ? err->message : "unknown failure"; + fprintf(stderr, "failed to init libgit2: %s\n", msg); + return res; + } + + /* Run the test suite */ + res = clar_test_run(); + + clar_test_shutdown(); + + return res; +} diff --git a/benchmarks/libgit2/oid.c b/benchmarks/libgit2/oid.c new file mode 100644 index 000000000..8e5b157e6 --- /dev/null +++ b/benchmarks/libgit2/oid.c @@ -0,0 +1,151 @@ +#include "clar.h" + +#include +#include +#include + +#include + +#define BENCHMARK_OID_COUNT 256 + +static git_oid sha1_one[BENCHMARK_OID_COUNT]; +static git_oid *sha1_two; + +#ifdef GIT_EXPERIMENTAL_SHA256 +static git_oid sha256_one[BENCHMARK_OID_COUNT]; +static git_oid *sha256_two; +#endif + +static void update_data_to_val(git_oid *out, git_oid_t type, uint32_t val) +{ + unsigned char data[GIT_OID_MAX_SIZE] = {0}; + size_t size; + +#ifdef GIT_EXPERIMENTAL_SHA256 + size = (type == GIT_OID_SHA256) ? GIT_OID_SHA256_SIZE : GIT_OID_SHA1_SIZE; +#else + size = GIT_OID_SHA1_SIZE; + + ((void)(type)); +#endif + + memset(data, 0, GIT_OID_MAX_SIZE); + + data[size - 1] = (unsigned char)(val & 0x000000ff); + data[size - 2] = (unsigned char)((val & 0x0000ff00) >> 8); + data[size - 3] = (unsigned char)((val & 0x00ff0000) >> 16); + data[size - 4] = (unsigned char)((val & 0x00ff0000) >> 24); + +#ifdef GIT_EXPERIMENTAL_SHA256 + cl_assert(git_oid_from_raw(out, data, type) == 0); +#else + cl_assert(git_oid_fromraw(out, data) == 0); +#endif +} + +void benchmark_oid__initialize(void) +{ + uint32_t accum = 0; + size_t i; + + sha1_two = calloc(BENCHMARK_OID_COUNT, sizeof(git_oid)); + cl_assert(sha1_two != NULL); + +#ifdef GIT_EXPERIMENTAL_SHA256 + sha256_two = calloc(BENCHMARK_OID_COUNT, sizeof(git_oid)); + cl_assert(sha256_two != NULL); +#endif + + for (i = 0; i < BENCHMARK_OID_COUNT; i++) { + update_data_to_val(&sha1_one[i], GIT_OID_SHA1, accum++); + update_data_to_val(&sha1_two[i], GIT_OID_SHA1, accum++); + } + +#ifdef GIT_EXPERIMENTAL_SHA256 + for (i = 0; i < BENCHMARK_OID_COUNT; i++) { + update_data_to_val(&sha256_one[i], GIT_OID_SHA256, accum++); + update_data_to_val(&sha256_two[i], GIT_OID_SHA256, accum++); + } +#endif +} + +void benchmark_oid__reset(void) +{ +} + +void benchmark_oid__cleanup(void) +{ + free(sha1_two); + +#ifdef GIT_EXPERIMENTAL_SHA256 + free(sha256_two); +#endif +} + +void benchmark_oid__cmp_sha1(void) +{ + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_cmp(&sha1_one[j], &sha1_two[j]); +} + +void benchmark_oid__cmp_sha256(void) +{ +#ifdef GIT_EXPERIMENTAL_SHA256 + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_cmp(&sha256_one[j], &sha256_two[j]); +#else + clar__skip(); +#endif +} + +void benchmark_oid__cpy_sha1(void) +{ + git_oid dest; + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_cpy(&dest, &sha1_one[j]); +} + +void benchmark_oid__cpy_sha256(void) +{ +#ifdef GIT_EXPERIMENTAL_SHA256 + git_oid dest; + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_cpy(&dest, &sha256_one[j]); +#else + clar__skip(); +#endif +} + +void benchmark_oid__zero_sha1(void) +{ + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_is_zero(&sha1_one[j]); +} + +void benchmark_oid__zero_sha256(void) +{ +#ifdef GIT_EXPERIMENTAL_SHA256 + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_is_zero(&sha256_one[j]); +#else + clar__skip(); +#endif +} diff --git a/benchmarks/libgit2/precompiled.c b/benchmarks/libgit2/precompiled.c new file mode 100644 index 000000000..5f656a45d --- /dev/null +++ b/benchmarks/libgit2/precompiled.c @@ -0,0 +1 @@ +#include "precompiled.h" diff --git a/benchmarks/libgit2/precompiled.h b/benchmarks/libgit2/precompiled.h new file mode 100644 index 000000000..e6b34738d --- /dev/null +++ b/benchmarks/libgit2/precompiled.h @@ -0,0 +1,2 @@ +#include "git2.h" +#include "clar.h" diff --git a/ci/compare-benchmarks.js b/ci/compare-benchmarks.js new file mode 100755 index 000000000..7646ed975 --- /dev/null +++ b/ci/compare-benchmarks.js @@ -0,0 +1,99 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const process = require('node:process'); + +const path = process.argv[2] || '.'; +const output = process.argv[3]; + +const dirs = fs.readdirSync(path); + +tests = [ ]; +results = { }; + +for (const dir of dirs) { + const name = dir.replace(/^results-/, ''); + + const control = JSON.parse(fs.readFileSync(`${path}/${dir}/control.json`)); + const candidate = JSON.parse(fs.readFileSync(`${path}/${dir}/candidate.json`)); + + results[name] = { }; + + const controlTests = control.tests.map((x) => x.name); + const candidateTests = candidate.tests.map((x) => x.name); + tests = [...new Set(tests.concat(controlTests).concat(candidateTests))]; + + for (const test of tests) { + const controlResults = control.tests.filter((x) => x.name === test)[0]; + const candidateResults = candidate.tests.filter((x) => x.name === test)[0]; + + let result; + + if (controlResults.results.status === 'ok' && + candidateResults.results.status === 'ok') { + const delta = (candidateResults.results.mean - controlResults.results.mean) / candidateResults.results.mean; + const deltaPercent = Math.round(Math.abs(delta) * 1000) / 10; + const direction = (delta > 0) ? 'slower' : 'faster'; + let highlight = ''; + + if (delta > 0.1) { + highlight = '**'; + } + + results[name][test] = `${highlight}${deltaPercent}% ${direction}${highlight}`; + } + else if (controlResults.results.status === 'ok') { + results[name][test] = `${candidateResults.results.stauts} in candidate`; + } + else if (candidateResults.results.status === 'ok') { + results[name][test] = `${controlResults.results.stauts} in control`; + } + else { + results[name][test] = controlResults.results.status; + } + } +} + +let markdown = '| Platform \\ Test |'; + +for (const test in Object.values(results)[0]) { + markdown += ` ${test} |`; +} +markdown += '\n'; + +markdown += `| --- |`; +for (const test in Object.values(results)[0]) { + markdown += ` --- |`; +} +markdown += '\n'; + +for (const name in results) { + markdown += `| ${name.replaceAll(/-/g, '‑')} |`; + + for (const test in results[name]) { + markdown += ` ${results[name][test]} |`; + } + + markdown += `\n`; +} + +if (output) { + fs.writeFileSync(output, markdown); +} +else { + console.log(markdown); +} + +function p(arr, q) { + const array = [...arr].sort(); + const pos = (array.length - 1) * q; + const base = Math.floor(pos); + const rest = pos - base; + + if (array[base + 1] !== undefined) { + return array[base] + rest * (array[base + 1] - array[base]); + } + else { + return array[base]; + } +} diff --git a/tests/clar/clar.c b/deps/clar/clar.c similarity index 84% rename from tests/clar/clar.c rename to deps/clar/clar.c index e959a5ae0..0d1ad2812 100644 --- a/tests/clar/clar.c +++ b/deps/clar/clar.c @@ -100,6 +100,7 @@ typedef struct stat STAT_T; #endif +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #include "clar.h" @@ -113,11 +114,11 @@ fixture_path(const char *base, const char *fixture_name); #endif struct clar_error { - const char *file; - const char *function; - uintmax_t line_number; - const char *error_msg; + const char *message; char *description; + const char *function; + const char *file; + uintmax_t line_number; struct clar_error *next; }; @@ -130,13 +131,22 @@ struct clar_explicit { }; struct clar_report { - const char *test; - int test_number; const char *suite; + const char *test; + const char *description; + int test_number; + + int runs; enum cl_test_status status; time_t start; - double elapsed; + + double *times; + double time_min; + double time_max; + double time_mean; + double time_stddev; + double time_total; struct clar_error *errors; struct clar_error *last_error; @@ -150,10 +160,12 @@ struct clar_summary { }; static struct { + enum cl_test_mode test_mode; enum cl_test_status test_status; - const char *active_test; const char *active_suite; + const char *active_test; + const char *active_description; int total_skipped; int total_errors; @@ -162,6 +174,7 @@ static struct { int suites_ran; enum cl_output_format output_format; + enum cl_summary_format summary_format; int exit_on_error; int verbosity; @@ -193,28 +206,32 @@ static struct { struct clar_func { const char *name; + const char *description; + int runs; void (*ptr)(void); }; struct clar_suite { const char *name; struct clar_func initialize; + struct clar_func reset; struct clar_func cleanup; const struct clar_func *tests; size_t test_count; int enabled; }; -/* From clar_print_*.c */ +/* From print.h */ static void clar_print_init(int test_count, int suite_count); static void clar_print_shutdown(int test_count, int suite_count, int error_count); static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error); -static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status failed); -static void clar_print_onsuite(const char *suite_name, int suite_index); +static void clar_print_suite_start(const char *suite_name, int suite_index); +static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number); +static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report); static void clar_print_onabortv(const char *msg, va_list argp); static void clar_print_onabort(const char *msg, ...); -/* From clar_sandbox.c */ +/* From sandbox.c */ static void clar_tempdir_init(void); static void clar_tempdir_shutdown(void); static int clar_sandbox_create(const char *suite_name, const char *test_name); @@ -224,6 +241,8 @@ static int clar_sandbox_cleanup(void); static struct clar_summary *clar_summary_init(const char *filename); static int clar_summary_shutdown(struct clar_summary *fp); +#include "clar/counter.h" + /* Load the declarations for the test suite */ #include "clar.suite" @@ -280,70 +299,122 @@ clar_report_all(void) } } -#ifdef WIN32 -# define clar_time DWORD - -static void clar_time_now(clar_time *out) +static void +compute_times(void) { - *out = GetTickCount(); -} + double total_squares = 0; + int i; -static double clar_time_diff(clar_time *start, clar_time *end) -{ - return ((double)*end - (double)*start) / 1000; -} -#else -# include + _clar.last_report->time_min = _clar.last_report->times[0]; + _clar.last_report->time_max = _clar.last_report->times[0]; + _clar.last_report->time_total = _clar.last_report->times[0]; -# define clar_time struct timeval + for (i = 1; i < _clar.last_report->runs; i++) { + if (_clar.last_report->times[i] < _clar.last_report->time_min) + _clar.last_report->time_min = _clar.last_report->times[i]; -static void clar_time_now(clar_time *out) -{ - gettimeofday(out, NULL); -} + if (_clar.last_report->times[i] > _clar.last_report->time_max) + _clar.last_report->time_max = _clar.last_report->times[i]; -static double clar_time_diff(clar_time *start, clar_time *end) -{ - return ((double)end->tv_sec + (double)end->tv_usec / 1.0E6) - - ((double)start->tv_sec + (double)start->tv_usec / 1.0E6); + _clar.last_report->time_total += _clar.last_report->times[i]; + } + + if (_clar.last_report->runs <= 1) { + _clar.last_report->time_stddev = 0; + } else { + _clar.last_report->time_mean = _clar.last_report->time_total / _clar.last_report->runs; + + for (i = 0; i < _clar.last_report->runs; i++) { + double dev = (_clar.last_report->times[i] > _clar.last_report->time_mean) ? + _clar.last_report->times[i] - _clar.last_report->time_mean : + _clar.last_report->time_mean - _clar.last_report->times[i]; + + total_squares += (dev * dev); + } + + _clar.last_report->time_stddev = sqrt(total_squares / _clar.last_report->runs); + } } -#endif static void clar_run_test( const struct clar_suite *suite, const struct clar_func *test, const struct clar_func *initialize, + const struct clar_func *reset, const struct clar_func *cleanup) { - clar_time start, end; + int runs = test->runs; + volatile int i = 0; - _clar.trampoline_enabled = 1; + _clar.last_report->start = time(NULL); + _clar.last_report->times = &_clar.last_report->time_mean; CL_TRACE(CL_TRACE__TEST__BEGIN); clar_sandbox_create(suite->name, test->name); - _clar.last_report->start = time(NULL); - clar_time_now(&start); + clar_print_test_start(suite->name, test->name, _clar.tests_ran); + + _clar.trampoline_enabled = 1; if (setjmp(_clar.trampoline) == 0) { if (initialize->ptr != NULL) initialize->ptr(); CL_TRACE(CL_TRACE__TEST__RUN_BEGIN); - test->ptr(); + + do { + struct clar_counter start, end; + double elapsed; + + if (i > 0 && reset->ptr != NULL) { + reset->ptr(); + } else if (i > 0) { + if (_clar.local_cleanup != NULL) + _clar.local_cleanup(_clar.local_cleanup_payload); + if (cleanup->ptr != NULL) + cleanup->ptr(); + if (initialize->ptr != NULL) + initialize->ptr(); + } + + clar_counter_now(&start); + test->ptr(); + clar_counter_now(&end); + + elapsed = clar_counter_diff(&start, &end); + + /* + * unless the number of runs was explicitly given + * in benchmark mode, use the first run as a sample + * to determine how many runs we should attempt + */ + if (_clar.test_mode == CL_TEST_BENCHMARK && !runs) { + runs = MAX(CLAR_BENCHMARK_RUN_MIN, (int)(CLAR_BENCHMARK_RUN_TIME / elapsed)); + runs = MIN(CLAR_BENCHMARK_RUN_MAX, runs); + } + + if (i == 0 && runs > 1) { + _clar.last_report->times = calloc(runs, sizeof(double)); + + if (_clar.last_report->times == NULL) + clar_abort("Failed to allocate report times.\n"); + } + + _clar.last_report->runs++; + _clar.last_report->times[i] = elapsed; + } while(++i < runs); + CL_TRACE(CL_TRACE__TEST__RUN_END); } - clar_time_now(&end); - _clar.trampoline_enabled = 0; if (_clar.last_report->status == CL_TEST_NOTRUN) _clar.last_report->status = CL_TEST_OK; - _clar.last_report->elapsed = clar_time_diff(&start, &end); + compute_times(); if (_clar.local_cleanup != NULL) _clar.local_cleanup(_clar.local_cleanup_payload); @@ -363,7 +434,7 @@ clar_run_test( _clar.local_cleanup = NULL; _clar.local_cleanup_payload = NULL; - clar_print_ontest(suite->name, test->name, _clar.tests_ran, _clar.last_report->status); + clar_print_test_finish(suite->name, test->name, _clar.tests_ran, _clar.last_report); } static void @@ -380,10 +451,11 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) if (_clar.exit_on_error && _clar.total_errors) return; - clar_print_onsuite(suite->name, ++_clar.suites_ran); + clar_print_suite_start(suite->name, ++_clar.suites_ran); _clar.active_suite = suite->name; _clar.active_test = NULL; + _clar.active_description = NULL; CL_TRACE(CL_TRACE__SUITE_BEGIN); if (filter) { @@ -412,11 +484,13 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) continue; _clar.active_test = test[i].name; + _clar.active_description = test[i].description; if ((report = calloc(1, sizeof(*report))) == NULL) clar_abort("Failed to allocate report.\n"); report->suite = _clar.active_suite; report->test = _clar.active_test; + report->description = _clar.active_description; report->test_number = _clar.tests_ran; report->status = CL_TEST_NOTRUN; @@ -428,13 +502,14 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) _clar.last_report = report; - clar_run_test(suite, &test[i], &suite->initialize, &suite->cleanup); + clar_run_test(suite, &test[i], &suite->initialize, &suite->reset, &suite->cleanup); if (_clar.exit_on_error && _clar.total_errors) return; } _clar.active_test = NULL; + _clar.active_description = NULL; CL_TRACE(CL_TRACE__SUITE_END); } @@ -600,6 +675,14 @@ clar_test_init(int argc, char **argv) { const char *summary_env; + if (_clar.test_mode == CL_TEST_BENCHMARK) { + _clar.output_format = CL_OUTPUT_TIMING; + _clar.summary_format = CL_SUMMARY_JSON; + } else { + _clar.output_format = CL_OUTPUT_CLAP; + _clar.summary_format = CL_SUMMARY_JUNIT; + } + if (argc > 1) clar_parse_args(argc, argv); @@ -622,6 +705,12 @@ clar_test_init(int argc, char **argv) clar_tempdir_init(); } +void +clar_test_set_mode(enum cl_test_mode mode) +{ + _clar.test_mode = mode; +} + int clar_test_run(void) { @@ -671,6 +760,9 @@ clar_test_shutdown(void) free(error); } + if (report->times != &report->time_mean) + free(report->times); + report_next = report->next; free(report); } @@ -735,7 +827,7 @@ static void clar__failv( error->file = _clar.invoke_file ? _clar.invoke_file : file; error->function = _clar.invoke_func ? _clar.invoke_func : function; error->line_number = _clar.invoke_line ? _clar.invoke_line : line; - error->error_msg = error_msg; + error->message = error_msg; if (description != NULL) { va_list args_copy; @@ -791,14 +883,14 @@ void clar__assert( const char *file, const char *function, size_t line, - const char *error_msg, - const char *description, + const char *error_message, + const char *error_description, int should_abort) { if (condition) return; - clar__fail(file, function, line, error_msg, description, should_abort); + clar__fail(file, function, line, error_message, error_description, should_abort); } void clar__assert_equal( diff --git a/tests/clar/clar.h b/deps/clar/clar.h similarity index 95% rename from tests/clar/clar.h rename to deps/clar/clar.h index 9ea91d3d0..47609df3a 100644 --- a/tests/clar/clar.h +++ b/deps/clar/clar.h @@ -21,6 +21,16 @@ # define CLAR_MAX_PATH 4096 #endif +/* + * In benchmark mode, by default, clar will run the test repeatedly for + * approximately `CLAR_BENCHMARK_RUN_TIME` seconds, and at least + * `CLAR_BENCHMARK_RUN_MIN` iterations. + */ + +#define CLAR_BENCHMARK_RUN_TIME 3.0 +#define CLAR_BENCHMARK_RUN_MIN 10 +#define CLAR_BENCHMARK_RUN_MAX 30000000 + #ifndef CLAR_SELFTEST # define CLAR_CURRENT_FILE __FILE__ # define CLAR_CURRENT_LINE __LINE__ @@ -31,6 +41,11 @@ # define CLAR_CURRENT_FUNC "func" #endif +enum cl_test_mode { + CL_TEST_STANDARD, + CL_TEST_BENCHMARK, +}; + enum cl_test_status { CL_TEST_OK, CL_TEST_FAILURE, @@ -41,10 +56,17 @@ enum cl_test_status { enum cl_output_format { CL_OUTPUT_CLAP, CL_OUTPUT_TAP, + CL_OUTPUT_TIMING, +}; + +enum cl_summary_format { + CL_SUMMARY_JUNIT, + CL_SUMMARY_JSON, }; /** Setup clar environment */ void clar_test_init(int argc, char *argv[]); +void clar_test_set_mode(enum cl_test_mode mode); int clar_test_run(void); void clar_test_shutdown(void); diff --git a/deps/clar/clar/counter.h b/deps/clar/clar/counter.h new file mode 100644 index 000000000..329f7481e --- /dev/null +++ b/deps/clar/clar/counter.h @@ -0,0 +1,167 @@ +#define CLAR_COUNTER_TV_DIFF(out_sec, out_usec, start_sec, start_usec, end_sec, end_usec) \ + if (start_usec > end_usec) { \ + out_sec = (end_sec - 1) - start_sec; \ + out_usec = (end_usec + 1000000) - start_usec; \ + } else { \ + out_sec = end_sec - start_sec; \ + out_usec = end_usec - start_usec; \ + } + +#ifdef _WIN32 + +struct clar_counter { + LARGE_INTEGER value; +}; + +static void clar_counter_now(struct clar_counter *out) +{ + QueryPerformanceCounter(&out->value); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + LARGE_INTEGER freq; + + QueryPerformanceFrequency(&freq); + return (double)(end->value.QuadPart - start->value.QuadPart)/(double)freq.QuadPart; +} + +#elif __APPLE__ + +#include +#include + +static double clar_counter_scaling_factor = -1; + +struct clar_counter { + union { + uint64_t absolute_time; + struct timeval tv; + } val; +}; + +static void clar_counter_now(struct clar_counter *out) +{ + if (clar_counter_scaling_factor == 0) { + mach_timebase_info_data_t info; + + clar_counter_scaling_factor = + mach_timebase_info(&info) == KERN_SUCCESS ? + ((double)info.numer / (double)info.denom) / 1.0E6 : + -1; + } + + /* mach_timebase_info failed; fall back to gettimeofday */ + if (clar_counter_scaling_factor < 0) + gettimeofday(&out->val.tv, NULL); + else + out->val.absolute_time = mach_absolute_time(); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + if (clar_counter_scaling_factor < 0) { + time_t sec; + suseconds_t usec; + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->val.tv.tv_sec, start->val.tv.tv_usec, + end->val.tv.tv_sec, end->val.tv.tv_usec); + + return (double)sec + ((double)usec / 1000000.0); + } else { + return (double)(end->val.absolute_time - start->val.absolute_time) * + clar_counter_scaling_factor; + } +} + +#elif defined(__amigaos4__) + +#include + +struct clar_counter { + struct TimeVal tv; +} + +static void clar_counter_now(struct clar_counter *out) +{ + ITimer->GetUpTime(&out->tv); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + uint32_t sec, usec; + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->tv.Seconds, start->tv.Microseconds, + end->tv.Seconds, end->tv.Microseconds); + + return (double)sec + ((double)usec / 1000000.0); +} + +#else + +#include + +struct clar_counter { + int type; + union { +#ifdef CLOCK_MONOTONIC + struct timespec tp; +#endif + struct timeval tv; + } val; +}; + +static void clar_counter_now(struct clar_counter *out) +{ +#ifdef CLOCK_MONOTONIC + if (clock_gettime(CLOCK_MONOTONIC, &out->val.tp) == 0) { + out->type = 0; + return; + } +#endif + + /* Fall back to using gettimeofday */ + out->type = 1; + gettimeofday(&out->val.tv, NULL); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + time_t sec; + suseconds_t usec; + +#ifdef CLOCK_MONOTONIC + if (start->type == 0) { + time_t sec; + long nsec; + + if (start->val.tp.tv_sec > end->val.tp.tv_sec) { + sec = (end->val.tp.tv_sec - 1) - start->val.tp.tv_sec; + nsec = (end->val.tp.tv_nsec + 1000000000) - start->val.tp.tv_nsec; + } else { + sec = end->val.tp.tv_sec - start->val.tp.tv_sec; + nsec = end->val.tp.tv_nsec - start->val.tp.tv_nsec; + } + + return (double)sec + ((double)nsec / 1000000000.0); + } +#endif + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->val.tv.tv_sec, start->val.tv.tv_usec, + end->val.tv.tv_sec, end->val.tv.tv_usec); + + return (double)sec + ((double)usec / 1000000.0); +} + +#endif diff --git a/tests/clar/clar/fixtures.h b/deps/clar/clar/fixtures.h similarity index 100% rename from tests/clar/clar/fixtures.h rename to deps/clar/clar/fixtures.h diff --git a/tests/clar/clar/fs.h b/deps/clar/clar/fs.h similarity index 100% rename from tests/clar/clar/fs.h rename to deps/clar/clar/fs.h diff --git a/tests/clar/clar/print.h b/deps/clar/clar/print.h similarity index 54% rename from tests/clar/clar/print.h rename to deps/clar/clar/print.h index 59b7dc14a..55aff5f4f 100644 --- a/tests/clar/clar/print.h +++ b/deps/clar/clar/print.h @@ -48,7 +48,7 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con error->file, error->line_number); - clar_print_indented(error->error_msg, 2); + clar_print_indented(error->message, 2); if (error->description != NULL) clar_print_indented(error->description, 2); @@ -57,9 +57,8 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con fflush(stdout); } -static void clar_print_clap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_clap_test_start(const char *suite_name, const char *test_name, int test_number) { - (void)test_name; (void)test_number; if (_clar.verbosity < 0) @@ -67,15 +66,18 @@ static void clar_print_clap_ontest(const char *suite_name, const char *test_name if (_clar.verbosity > 1) { printf("%s::%s: ", suite_name, test_name); + fflush(stdout); + } +} - switch (status) { - case CL_TEST_OK: printf("ok\n"); break; - case CL_TEST_FAILURE: printf("fail\n"); break; - case CL_TEST_SKIP: printf("skipped\n"); break; - case CL_TEST_NOTRUN: printf("notrun\n"); break; - } - } else { - switch (status) { +static void clar_print_clap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) +{ + (void)suite_name; + (void)test_name; + (void)test_number; + + if (_clar.verbosity == 0) { + switch (report->status) { case CL_TEST_OK: printf("."); break; case CL_TEST_FAILURE: printf("F"); break; case CL_TEST_SKIP: printf("S"); break; @@ -83,10 +85,17 @@ static void clar_print_clap_ontest(const char *suite_name, const char *test_name } fflush(stdout); + } else if (_clar.verbosity > 1) { + switch (report->status) { + case CL_TEST_OK: printf("ok\n"); break; + case CL_TEST_FAILURE: printf("fail\n"); break; + case CL_TEST_SKIP: printf("skipped\n"); break; + case CL_TEST_NOTRUN: printf("notrun\n"); break; + } } } -static void clar_print_clap_onsuite(const char *suite_name, int suite_index) +static void clar_print_clap_suite_start(const char *suite_name, int suite_index) { if (_clar.verbosity < 0) return; @@ -127,7 +136,7 @@ static void clar_print_tap_error(int num, const struct clar_report *report, cons static void print_escaped(const char *str) { - const char *c; + char *c; while ((c = strchr(str, '\'')) != NULL) { printf("%.*s", (int)(c - str), str); @@ -138,14 +147,21 @@ static void print_escaped(const char *str) printf("%s", str); } -static void clar_print_tap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_tap_test_start(const char *suite_name, const char *test_name, int test_number) +{ + (void)suite_name; + (void)test_name; + (void)test_number; +} + +static void clar_print_tap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) { const struct clar_error *error = _clar.last_report->errors; (void)test_name; (void)test_number; - switch(status) { + switch(report->status) { case CL_TEST_OK: printf("ok %d - %s::%s\n", test_number, suite_name, test_name); break; @@ -155,7 +171,7 @@ static void clar_print_tap_ontest(const char *suite_name, const char *test_name, if (_clar.verbosity >= 0) { printf(" ---\n"); printf(" reason: |\n"); - clar_print_indented(error->error_msg, 6); + clar_print_indented(error->message, 6); if (error->description) clar_print_indented(error->description, 6); @@ -177,7 +193,7 @@ static void clar_print_tap_ontest(const char *suite_name, const char *test_name, fflush(stdout); } -static void clar_print_tap_onsuite(const char *suite_name, int suite_index) +static void clar_print_tap_suite_start(const char *suite_name, int suite_index) { if (_clar.verbosity < 0) return; @@ -191,6 +207,114 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg) fflush(stdout); } +/* timings format: useful for benchmarks */ + +static void clar_print_timing_init(int test_count, int suite_count) +{ + (void)test_count; + (void)suite_count; + + printf("Started benchmarks (mean time ± stddev / min time … max time):\n\n"); +} + +static void clar_print_timing_shutdown(int test_count, int suite_count, int error_count) +{ + (void)test_count; + (void)suite_count; + (void)error_count; +} + +static void clar_print_timing_error(int num, const struct clar_report *report, const struct clar_error *error) +{ + (void)num; + (void)report; + (void)error; +} + +static void clar_print_timing_test_start(const char *suite_name, const char *test_name, int test_number) +{ + (void)test_number; + + printf("%s::%s: ", suite_name, test_name); + fflush(stdout); +} + +static void clar_print_timing_time(double t) +{ + static const char *units[] = { "sec", "ms", "μs", "ns" }; + static const int units_len = sizeof(units) / sizeof(units[0]); + int unit = 0, exponent = 0, digits; + + while (t < 1.0 && unit < units_len - 1) { + t *= 1000.0; + unit++; + } + + while (t > 0.0 && t < 1.0 && exponent < 10) { + t *= 10.0; + exponent++; + } + + digits = (t < 10.0) ? 3 : ((t < 100.0) ? 2 : 1); + + printf("%.*f", digits, t); + + if (exponent > 0) + printf("e-%d", exponent); + + printf(" %s", units[unit]); +} + +static void clar_print_timing_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) +{ + const struct clar_error *error = _clar.last_report->errors; + + (void)suite_name; + (void)test_name; + (void)test_number; + + switch(report->status) { + case CL_TEST_OK: + clar_print_timing_time(report->time_mean); + + if (report->runs > 1) { + printf(" ± "); + clar_print_timing_time(report->time_stddev); + + printf(" / range: "); + clar_print_timing_time(report->time_min); + printf(" … "); + clar_print_timing_time(report->time_max); + printf(" (%d runs)", report->runs); + } + + printf("\n"); + break; + case CL_TEST_FAILURE: + printf("failed: %s\n", error->message); + break; + case CL_TEST_SKIP: + case CL_TEST_NOTRUN: + printf("skipped\n"); + break; + } + + fflush(stdout); +} + +static void clar_print_timing_suite_start(const char *suite_name, int suite_index) +{ + if (_clar.verbosity == 1) + printf("\n%s", suite_name); + + (void)suite_index; +} + +static void clar_print_timing_onabort(const char *fmt, va_list arg) +{ + vfprintf(stderr, fmt, arg); +} + /* indirection between protocol output selection */ #define PRINT(FN, ...) do { \ @@ -201,6 +325,9 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg) case CL_OUTPUT_TAP: \ clar_print_tap_##FN (__VA_ARGS__); \ break; \ + case CL_OUTPUT_TIMING: \ + clar_print_timing_##FN (__VA_ARGS__); \ + break; \ default: \ abort(); \ } \ @@ -221,14 +348,19 @@ static void clar_print_error(int num, const struct clar_report *report, const st PRINT(error, num, report, error); } -static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number) { - PRINT(ontest, suite_name, test_name, test_number, status); + PRINT(test_start, suite_name, test_name, test_number); } -static void clar_print_onsuite(const char *suite_name, int suite_index) +static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) { - PRINT(onsuite, suite_name, suite_index); + PRINT(test_finish, suite_name, test_name, test_number, report); +} + +static void clar_print_suite_start(const char *suite_name, int suite_index) +{ + PRINT(suite_start, suite_name, suite_index); } static void clar_print_onabortv(const char *msg, va_list argp) diff --git a/tests/clar/clar/sandbox.h b/deps/clar/clar/sandbox.h similarity index 100% rename from tests/clar/clar/sandbox.h rename to deps/clar/clar/sandbox.h diff --git a/deps/clar/clar/summary.h b/deps/clar/clar/summary.h new file mode 100644 index 000000000..f2be3c13a --- /dev/null +++ b/deps/clar/clar/summary.h @@ -0,0 +1,311 @@ +#include +#include + +static int clar_summary_time_digits(double t) +{ + int digits = 3; + + if (t >= 100.0) + return 1; + else if (t >= 10.0) + return 2; + + while (t > 0.0 && t < 1.0 && digits < 10) { + t *= 10.0; + digits++; + } + + return digits; +} + +static int clar_summary_junit_close_tag( + struct clar_summary *summary, const char *tag, int indent) +{ + const char *indt; + + if (indent == 0) indt = ""; + else if (indent == 1) indt = "\t"; + else indt = "\t\t"; + + return fprintf(summary->fp, "%s\n", indt, tag); +} + +static int clar_summary_junit_testsuites(struct clar_summary *summary) +{ + return fprintf(summary->fp, "\n"); +} + +static int clar_summary_junit_testsuite(struct clar_summary *summary, + int idn, const char *name, time_t timestamp, + int test_count, int fail_count, int error_count) +{ + struct tm tm; + char iso_dt[20]; + + localtime_r(×tamp, &tm); + if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", &tm) == 0) + return -1; + + return fprintf(summary->fp, "\t\n", + idn, name, iso_dt, test_count, fail_count, error_count); +} + +static int clar_summary_junit_testcase(struct clar_summary *summary, + const char *name, const char *classname, double elapsed) +{ + return fprintf(summary->fp, + "\t\t\n", + name, classname, clar_summary_time_digits(elapsed), elapsed); +} + +static int clar_summary_junit_failure(struct clar_summary *summary, + const char *type, const char *message, const char *desc) +{ + return fprintf(summary->fp, + "\t\t\t\n", + type, message, desc); +} + +static int clar_summary_junit_skipped(struct clar_summary *summary) +{ + return fprintf(summary->fp, "\t\t\t\n"); +} + +static struct clar_summary *clar_summary_junit_init(const char *filename) +{ + struct clar_summary *summary; + FILE *fp; + + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); + return NULL; + } + + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); + fclose(fp); + return NULL; + } + + summary->filename = filename; + summary->fp = fp; + + return summary; +} + +static int clar_summary_junit_shutdown(struct clar_summary *summary) +{ + struct clar_report *report; + const char *last_suite = NULL; + + if (clar_summary_junit_testsuites(summary) < 0) + goto on_error; + + report = _clar.reports; + while (report != NULL) { + struct clar_error *error = report->errors; + + if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) { + if (clar_summary_junit_testsuite(summary, 0, report->suite, + report->start, _clar.tests_ran, _clar.total_errors, 0) < 0) + goto on_error; + } + + last_suite = report->suite; + + clar_summary_junit_testcase(summary, report->test, report->suite, report->time_total); + + while (error != NULL) { + if (clar_summary_junit_failure(summary, "assert", + error->message, error->description) < 0) + goto on_error; + + error = error->next; + } + + if (report->status == CL_TEST_SKIP) + clar_summary_junit_skipped(summary); + + if (clar_summary_junit_close_tag(summary, "testcase", 2) < 0) + goto on_error; + + report = report->next; + + if (!report || strcmp(last_suite, report->suite) != 0) { + if (clar_summary_junit_close_tag(summary, "testsuite", 1) < 0) + goto on_error; + } + } + + if (clar_summary_junit_close_tag(summary, "testsuites", 0) < 0 || + fclose(summary->fp) != 0) + goto on_error; + + printf("written summary file to %s\n", summary->filename); + + free(summary); + return 0; + +on_error: + fclose(summary->fp); + free(summary); + return -1; +} + +static struct clar_summary *clar_summary_json_init(const char *filename) +{ + struct clar_summary *summary; + FILE *fp; + + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); + return NULL; + } + + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); + fclose(fp); + return NULL; + } + + summary->filename = filename; + summary->fp = fp; + + return summary; +} + +static int clar_summary_json_shutdown(struct clar_summary *summary) +{ + struct clar_report *report; + int i; + + fprintf(summary->fp, "{\n"); + fprintf(summary->fp, " \"tests\": [\n"); + + report = _clar.reports; + while (report != NULL) { + struct clar_error *error = report->errors; + + if (report != _clar.reports) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " {\n"); + fprintf(summary->fp, " \"name\": \"%s::%s\",\n", report->suite, report->test); + + if (report->description) + fprintf(summary->fp, " \"description\": \"%s\",\n", report->description); + + fprintf(summary->fp, " \"results\": {\n"); + + fprintf(summary->fp, " \"status\": "); + if (report->status == CL_TEST_OK) + fprintf(summary->fp, "\"ok\",\n"); + else if (report->status == CL_TEST_FAILURE) + fprintf(summary->fp, "\"failed\",\n"); + else if (report->status == CL_TEST_SKIP) + fprintf(summary->fp, "\"skipped\"\n"); + else + clar_abort("unknown test status %d", report->status); + + if (report->status == CL_TEST_OK) { + fprintf(summary->fp, " \"mean\": %.*f,\n", + clar_summary_time_digits(report->time_mean), report->time_mean); + fprintf(summary->fp, " \"stddev\": %.*f,\n", + clar_summary_time_digits(report->time_stddev), report->time_stddev); + fprintf(summary->fp, " \"min\": %.*f,\n", + clar_summary_time_digits(report->time_min), report->time_min); + fprintf(summary->fp, " \"max\": %.*f,\n", + clar_summary_time_digits(report->time_max), report->time_max); + fprintf(summary->fp, " \"times\": [\n"); + + for (i = 0; i < report->runs; i++) { + if (i > 0) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " %.*f", + clar_summary_time_digits(report->times[i]), report->times[i]); + } + + fprintf(summary->fp, "\n ]\n"); + } + + if (report->status == CL_TEST_FAILURE) { + fprintf(summary->fp, " \"errors\": [\n"); + + while (error != NULL) { + if (error != report->errors) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " {\n"); + fprintf(summary->fp, " \"message\": \"%s\",\n", error->message); + + if (error->description) + fprintf(summary->fp, " \"description\": \"%s\",\n", error->description); + + fprintf(summary->fp, " \"function\": \"%s\",\n", error->function); + fprintf(summary->fp, " \"file\": \"%s\",\n", error->file); + fprintf(summary->fp, " \"line\": %" PRIuMAX "\n", error->line_number); + fprintf(summary->fp, " }"); + + error = error->next; + } + + fprintf(summary->fp, "\n"); + fprintf(summary->fp, " ]\n"); + } + + fprintf(summary->fp, " }\n"); + fprintf(summary->fp, " }"); + + report = report->next; + } + + fprintf(summary->fp, "\n"); + fprintf(summary->fp, " ]\n"); + fprintf(summary->fp, "}\n"); + + if (fclose(summary->fp) != 0) + goto on_error; + + printf("written summary file to %s\n", summary->filename); + + free(summary); + return 0; + +on_error: + fclose(summary->fp); + free(summary); + return -1; +} + +/* indirection between protocol output selection */ + +#define SUMMARY(FN, ...) do { \ + switch (_clar.summary_format) { \ + case CL_SUMMARY_JUNIT: \ + return clar_summary_junit_##FN (__VA_ARGS__); \ + break; \ + case CL_SUMMARY_JSON: \ + return clar_summary_json_##FN (__VA_ARGS__); \ + break; \ + default: \ + abort(); \ + } \ + } while(0) + +struct clar_summary *clar_summary_init(const char *filename) +{ + SUMMARY(init, filename); +} + +int clar_summary_shutdown(struct clar_summary *summary) +{ + SUMMARY(shutdown, summary); +} diff --git a/tests/clar/clar_libgit2.c b/deps/clar/clar_libgit2.c similarity index 100% rename from tests/clar/clar_libgit2.c rename to deps/clar/clar_libgit2.c diff --git a/tests/clar/clar_libgit2.h b/deps/clar/clar_libgit2.h similarity index 100% rename from tests/clar/clar_libgit2.h rename to deps/clar/clar_libgit2.h diff --git a/tests/clar/clar_libgit2_alloc.c b/deps/clar/clar_libgit2_alloc.c similarity index 100% rename from tests/clar/clar_libgit2_alloc.c rename to deps/clar/clar_libgit2_alloc.c diff --git a/tests/clar/clar_libgit2_alloc.h b/deps/clar/clar_libgit2_alloc.h similarity index 100% rename from tests/clar/clar_libgit2_alloc.h rename to deps/clar/clar_libgit2_alloc.h diff --git a/tests/clar/clar_libgit2_timer.c b/deps/clar/clar_libgit2_timer.c similarity index 100% rename from tests/clar/clar_libgit2_timer.c rename to deps/clar/clar_libgit2_timer.c diff --git a/tests/clar/clar_libgit2_timer.h b/deps/clar/clar_libgit2_timer.h similarity index 100% rename from tests/clar/clar_libgit2_timer.h rename to deps/clar/clar_libgit2_timer.h diff --git a/tests/clar/clar_libgit2_trace.c b/deps/clar/clar_libgit2_trace.c similarity index 100% rename from tests/clar/clar_libgit2_trace.c rename to deps/clar/clar_libgit2_trace.c diff --git a/tests/clar/clar_libgit2_trace.h b/deps/clar/clar_libgit2_trace.h similarity index 100% rename from tests/clar/clar_libgit2_trace.h rename to deps/clar/clar_libgit2_trace.h diff --git a/tests/clar/generate.py b/deps/clar/generate.py similarity index 66% rename from tests/clar/generate.py rename to deps/clar/generate.py index 2357b2d6d..2e4eaa0a9 100644 --- a/tests/clar/generate.py +++ b/deps/clar/generate.py @@ -17,8 +17,13 @@ class Module(object): def _render_callback(self, cb): if not cb: - return ' { NULL, NULL }' - return ' { "%s", &%s }' % (cb['short_name'], cb['symbol']) + return ' { NULL, NULL, 0, NULL }' + + return ' { "%s", %s, %d, &%s }' % \ + (cb['short_name'], \ + '"' + cb['description'] + '"' if cb['description'] != None else "NULL", \ + cb['runs'], \ + cb['symbol']) class DeclarationTemplate(Template): def render(self): @@ -27,6 +32,9 @@ class Module(object): for initializer in self.module.initializers: out += "extern %s;\n" % initializer['declaration'] + if self.module.reset: + out += "extern %s;\n" % self.module.reset['declaration'] + if self.module.cleanup: out += "extern %s;\n" % self.module.cleanup['declaration'] @@ -34,7 +42,7 @@ class Module(object): class CallbacksTemplate(Template): def render(self): - out = "static const struct clar_func _clar_cb_%s[] = {\n" % self.module.name + out = "static const struct %s_func _%s_cb_%s[] = {\n" % (self.module.app_name, self.module.app_name, self.module.name) out += ",\n".join(self._render_callback(cb) for cb in self.module.callbacks) out += "\n};\n" return out @@ -58,14 +66,16 @@ class Module(object): { "${clean_name}", ${initialize}, + ${reset}, ${cleanup}, ${cb_ptr}, ${cb_count}, ${enabled} }""" ).substitute( clean_name = name, initialize = self._render_callback(initializer), + reset = self._render_callback(self.module.reset), cleanup = self._render_callback(self.module.cleanup), - cb_ptr = "_clar_cb_%s" % self.module.name, + cb_ptr = "_%s_cb_%s" % (self.module.app_name, self.module.name), cb_count = len(self.module.callbacks), enabled = int(self.module.enabled) ) @@ -73,8 +83,10 @@ class Module(object): return ','.join(templates) - def __init__(self, name): + def __init__(self, name, app_name, prefix): self.name = name + self.app_name = app_name + self.prefix = prefix self.mtime = None self.enabled = True @@ -85,7 +97,7 @@ class Module(object): def _skip_comments(self, text): SKIP_COMMENTS_REGEX = re.compile( - r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', + r'//.*?$|/\*(?!\s*\[clar\]:).*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE) def _replacer(match): @@ -95,24 +107,63 @@ class Module(object): return re.sub(SKIP_COMMENTS_REGEX, _replacer, text) def parse(self, contents): - TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\s*\(\s*void\s*\))\s*\{" + TEST_FUNC_REGEX = r"^(void\s+(%s_%s__(\w+))\s*\(\s*void\s*\))(?:\s*/\*\s*\[clar\]:\s*(.*?)\s*\*/)?\s*\{" contents = self._skip_comments(contents) - regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE) + regex = re.compile(TEST_FUNC_REGEX % (self.prefix, self.name), re.MULTILINE) self.callbacks = [] self.initializers = [] + self.reset = None self.cleanup = None - for (declaration, symbol, short_name) in regex.findall(contents): + for (declaration, symbol, short_name, options) in regex.findall(contents): + runs = 0 + description = None + + while options != '': + match = re.search(r'^([a-zA-Z0-9]+)=(\"[^"]*\"|[a-zA-Z0-9_\-]+|\d+)(?:,\s*|\Z)(.*)', options) + + if match == None: + print("Invalid options: '%s' for '%s'" % (options, symbol)) + sys.exit(1) + + key = match.group(1) + value = match.group(2) + options = match.group(3) + + match = re.search(r'^\"(.*)\"$', value) + if match != None: + value = match.group(1) + + match = re.search(r'([^a-zA-Z0-9 _\-,\.])', value) + if match != None: + print("Invalid character '%s' in %s for '%s'" % (match.group(1), key, symbol)) + sys.exit(1) + + if key == "description": + description = value + elif key == "runs": + if not value.isnumeric(): + print("Invalid option: '%s' in runs for '%s'" % (option, symbol)) + sys.exit(1) + runs = int(value) + else: + print("Invalid option: '%s' for '%s'" % (key, symbol)) + sys.exit(1) + data = { "short_name" : short_name, "declaration" : declaration, - "symbol" : symbol + "symbol" : symbol, + "description" : description, + "runs" : runs } if short_name.startswith('initialize'): self.initializers.append(data) + elif short_name == 'reset': + self.reset = data elif short_name == 'cleanup': self.cleanup = data else: @@ -179,8 +230,8 @@ class TestSuite(object): return modules - def load_cache(self): - path = os.path.join(self.output, '.clarcache') + def load_cache(self, app_name): + path = os.path.join(self.output, ".%scache" % app_name) cache = {} try: @@ -192,18 +243,18 @@ class TestSuite(object): return cache - def save_cache(self): - path = os.path.join(self.output, '.clarcache') + def save_cache(self, app_name): + path = os.path.join(self.output, ".%scache" % app_name) with open(path, 'wb') as cache: pickle.dump(self.modules, cache) - def load(self, force = False): + def load(self, app_name, prefix, force = False): module_data = self.find_modules() - self.modules = {} if force else self.load_cache() + self.modules = {} if force else self.load_cache(app_name) for path, name in module_data: if name not in self.modules: - self.modules[name] = Module(name) + self.modules[name] = Module(name, app_name, prefix) if not self.modules[name].refresh(path): del self.modules[name] @@ -222,15 +273,15 @@ class TestSuite(object): def callback_count(self): return sum(len(module.callbacks) for module in self.modules.values()) - def write(self): + def write(self, name): if not os.path.exists(self.output): os.makedirs(self.output) - wrote_suite = self.write_suite() - wrote_header = self.write_header() + wrote_suite = self.write_suite(name) + wrote_header = self.write_header(name) if wrote_suite or wrote_header: - self.save_cache() + self.save_cache(name) return True return False @@ -257,8 +308,8 @@ class TestSuite(object): return True - def write_suite(self): - suite_fn = os.path.join(self.output, 'clar.suite') + def write_suite(self, name): + suite_fn = os.path.join(self.output, '%s.suite' % name) with io.StringIO() as suite_file: modules = sorted(self.modules.values(), key=lambda module: module.name) @@ -271,25 +322,26 @@ class TestSuite(object): t = Module.CallbacksTemplate(module) suite_file.write(t.render()) - suites = "static struct clar_suite _clar_suites[] = {" + ','.join( + suites = "static struct %s_suite _%s_suites[] = {" % (name, name) + suites += ','.join( Module.InfoTemplate(module).render() for module in modules ) + "\n};\n" suite_file.write(suites) - suite_file.write(u"static const size_t _clar_suite_count = %d;\n" % self.suite_count()) - suite_file.write(u"static const size_t _clar_callback_count = %d;\n" % self.callback_count()) + suite_file.write(u"static const size_t _%s_suite_count = %d;\n" % (name, self.suite_count())) + suite_file.write(u"static const size_t _%s_callback_count = %d;\n" % (name, self.callback_count())) return self.write_output(suite_fn, suite_file.getvalue()) return False - def write_header(self): - header_fn = os.path.join(self.output, 'clar_suite.h') + def write_header(self, name): + header_fn = os.path.join(self.output, '%s_suite.h' % name) with io.StringIO() as header_file: - header_file.write(u"#ifndef _____clar_suite_h_____\n") - header_file.write(u"#define _____clar_suite_h_____\n") + header_file.write(u"#ifndef _____%s_suite_h_____\n" % name) + header_file.write(u"#define _____%s_suite_h_____\n" % name) modules = sorted(self.modules.values(), key=lambda module: module.name) @@ -310,6 +362,8 @@ if __name__ == '__main__': parser.add_option('-f', '--force', action="store_true", dest='force', default=False) parser.add_option('-x', '--exclude', dest='excluded', action='append', default=[]) parser.add_option('-o', '--output', dest='output') + parser.add_option('-n', '--name', dest='name', default='clar') + parser.add_option('-p', '--prefix', dest='prefix', default='test') options, args = parser.parse_args() if len(args) > 1: @@ -323,7 +377,8 @@ if __name__ == '__main__': output = options.output or path suite = TestSuite(path, output) - suite.load(options.force) + suite.load(options.name, options.prefix, options.force) suite.disable(options.excluded) - if suite.write(): - print("Written `clar.suite`, `clar_suite.h` (%d tests in %d suites)" % (suite.callback_count(), suite.suite_count())) + + if suite.write(options.name): + print("Written `%s.suite`, `%s_suite.h` (%d tests in %d suites)" % (options.name, options.name, suite.callback_count(), suite.suite_count())) diff --git a/tests/clar/main.c b/deps/clar/main.c similarity index 100% rename from tests/clar/main.c rename to deps/clar/main.c diff --git a/tests/README.md b/tests/README.md index 460e045e3..78e625242 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,10 +2,6 @@ These are the unit and integration tests for the libgit2 projects. -* `benchmarks` - These are benchmark tests that excercise the CLI. -* `clar` - This is [clar](https://github.com/clar-test/clar) the common test framework. * `headertest` This is a simple project that ensures that our public headers are compatible with extremely strict compilation options. diff --git a/tests/clar/clar/summary.h b/tests/clar/clar/summary.h deleted file mode 100644 index 7b85f162d..000000000 --- a/tests/clar/clar/summary.h +++ /dev/null @@ -1,140 +0,0 @@ - -#include -#include - -static int clar_summary_close_tag( - struct clar_summary *summary, const char *tag, int indent) -{ - const char *indt; - - if (indent == 0) indt = ""; - else if (indent == 1) indt = "\t"; - else indt = "\t\t"; - - return fprintf(summary->fp, "%s\n", indt, tag); -} - -static int clar_summary_testsuites(struct clar_summary *summary) -{ - return fprintf(summary->fp, "\n"); -} - -static int clar_summary_testsuite(struct clar_summary *summary, - int idn, const char *name, time_t timestamp, - int test_count, int fail_count, int error_count) -{ - struct tm tm; - char iso_dt[20]; - - localtime_r(×tamp, &tm); - if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", &tm) == 0) - return -1; - - return fprintf(summary->fp, "\t\n", - idn, name, iso_dt, test_count, fail_count, error_count); -} - -static int clar_summary_testcase(struct clar_summary *summary, - const char *name, const char *classname, double elapsed) -{ - return fprintf(summary->fp, - "\t\t\n", - name, classname, elapsed); -} - -static int clar_summary_failure(struct clar_summary *summary, - const char *type, const char *message, const char *desc) -{ - return fprintf(summary->fp, - "\t\t\t\n", - type, message, desc); -} - -static int clar_summary_skipped(struct clar_summary *summary) -{ - return fprintf(summary->fp, "\t\t\t\n"); -} - -struct clar_summary *clar_summary_init(const char *filename) -{ - struct clar_summary *summary; - FILE *fp; - - if ((fp = fopen(filename, "w")) == NULL) - clar_abort("Failed to open the summary file '%s': %s.\n", - filename, strerror(errno)); - - if ((summary = malloc(sizeof(struct clar_summary))) == NULL) - clar_abort("Failed to allocate summary.\n"); - - summary->filename = filename; - summary->fp = fp; - - return summary; -} - -int clar_summary_shutdown(struct clar_summary *summary) -{ - struct clar_report *report; - const char *last_suite = NULL; - - if (clar_summary_testsuites(summary) < 0) - goto on_error; - - report = _clar.reports; - while (report != NULL) { - struct clar_error *error = report->errors; - - if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_testsuite(summary, 0, report->suite, - report->start, _clar.tests_ran, _clar.total_errors, 0) < 0) - goto on_error; - } - - last_suite = report->suite; - - clar_summary_testcase(summary, report->test, report->suite, report->elapsed); - - while (error != NULL) { - if (clar_summary_failure(summary, "assert", - error->error_msg, error->description) < 0) - goto on_error; - - error = error->next; - } - - if (report->status == CL_TEST_SKIP) - clar_summary_skipped(summary); - - if (clar_summary_close_tag(summary, "testcase", 2) < 0) - goto on_error; - - report = report->next; - - if (!report || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_close_tag(summary, "testsuite", 1) < 0) - goto on_error; - } - } - - if (clar_summary_close_tag(summary, "testsuites", 0) < 0 || - fclose(summary->fp) != 0) - goto on_error; - - printf("written summary file to %s\n", summary->filename); - - free(summary); - return 0; - -on_error: - fclose(summary->fp); - free(summary); - return -1; -} diff --git a/tests/libgit2/CMakeLists.txt b/tests/libgit2/CMakeLists.txt index 7a4bb476c..681635a35 100644 --- a/tests/libgit2/CMakeLists.txt +++ b/tests/libgit2/CMakeLists.txt @@ -10,9 +10,9 @@ find_package(PythonInterp) if(NOT PYTHONINTERP_FOUND) message(FATAL_ERROR "Could not find a python interpreter, which is needed to build the tests. " "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests") -ENDIF() +endif() -set(CLAR_PATH "${PROJECT_SOURCE_DIR}/tests/clar") +set(CLAR_PATH "${PROJECT_SOURCE_DIR}/deps/clar") set(CLAR_FIXTURES "${PROJECT_SOURCE_DIR}/tests/resources/") set(TEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}") add_definitions(-DCLAR_FIXTURE_PATH=\"${CLAR_FIXTURES}\") @@ -48,7 +48,11 @@ add_executable(libgit2_tests ${SRC_CLAR} ${SRC_TEST} ${LIBGIT2_OBJECTS}) set_target_properties(libgit2_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) target_include_directories(libgit2_tests PRIVATE ${TEST_INCLUDES} ${LIBGIT2_INCLUDES} ${LIBGIT2_DEPENDENCY_INCLUDES}) target_include_directories(libgit2_tests SYSTEM PRIVATE ${LIBGIT2_SYSTEM_INCLUDES}) + target_link_libraries(libgit2_tests ${LIBGIT2_SYSTEM_LIBS}) +if(NOT MSVC_IDE) + target_link_libraries(libgit2_tests m) +endif() ide_split_sources(libgit2_tests) diff --git a/tests/util/CMakeLists.txt b/tests/util/CMakeLists.txt index 425c27dcd..ba861aa1c 100644 --- a/tests/util/CMakeLists.txt +++ b/tests/util/CMakeLists.txt @@ -10,9 +10,9 @@ find_package(PythonInterp) if(NOT PYTHONINTERP_FOUND) message(FATAL_ERROR "Could not find a python interpeter, which is needed to build the tests. " "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests") -ENDIF() +endif() -set(CLAR_PATH "${libgit2_SOURCE_DIR}/tests/clar") +set(CLAR_PATH "${libgit2_SOURCE_DIR}/deps/clar") set(CLAR_FIXTURES "${libgit2_SOURCE_DIR}/tests/resources/") set(TEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}") add_definitions(-DCLAR_FIXTURE_PATH=\"${CLAR_FIXTURES}\") @@ -48,7 +48,11 @@ set_target_properties(util_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${libgit2_B target_include_directories(util_tests PRIVATE ${TEST_INCLUDES} ${LIBGIT2_INCLUDES} ${LIBGIT2_DEPENDENCY_INCLUDES}) target_include_directories(util_tests SYSTEM PRIVATE ${LIBGIT2_SYSTEM_INCLUDES}) + target_link_libraries(util_tests ${LIBGIT2_SYSTEM_LIBS}) +if(NOT MSVC_IDE) + target_link_libraries(util_tests m) +endif() ide_split_sources(util_tests)