Merge remote-tracking branch 'origin/main' into stat64

.github/actions/run-build/action.yml

@@ -18,11 +18,16 @@ inputs:
     type: string
     required: true
     default: 'bash'
+  cmake-global-options:
+    description: CMAKE_GLOBAL_OPTIONS to pass
+    type: string
 
 runs:
   using: 'composite'
   steps:
   - run: |
+      export CMAKE_GLOBAL_OPTIONS="${{ inputs.cmake-global-options }}"
+
       if [ -n "${{ inputs.container }}" ]; then
         docker run \
           --rm \
@@ -35,6 +40,7 @@ runs:
           -e CFLAGS \
           -e CMAKE_GENERATOR \
           -e CMAKE_OPTIONS \
+          -e CMAKE_GLOBAL_OPTIONS \
           -e GITTEST_NEGOTIATE_PASSWORD \
           -e PKG_CONFIG_PATH \
           -e SKIP_NEGOTIATE_TESTS \

.github/workflows/ab-perf.yml

@@ -0,0 +1,231 @@
+# A/B testing with benchmarks to compare a control branch (main) to a
+# candidate branch (a pull request).
+name: A/B Performance Test
+
+on:
+  workflow_dispatch:
+
+env:
+  docker-registry: ghcr.io
+  docker-config-path: ci/docker
+
+jobs:
+  # Run our CI/CD builds.  We build a matrix with the various build targets
+  # and their details.  Then we build either in a docker container (Linux)
+  # or on the actual hosts (macOS, Windows).
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+        # All builds: core platforms
+        - name: "Linux (Noble, GCC, OpenSSL, libssh2)"
+          id: noble-gcc-openssl
+          os: ubuntu-latest
+          container:
+            name: noble
+          env:
+            CC: gcc
+            CMAKE_GENERATOR: Ninja
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DUSE_GSSAPI=ON -DUSE_SSH=libssh2
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+        - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)"
+          id: noble-clang-mbedtls
+          os: ubuntu-latest
+          container:
+            name: noble
+          env:
+            CC: clang
+            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            CMAKE_GENERATOR: Ninja
+        - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)"
+          id: xenial-gcc-openssl
+          os: ubuntu-latest
+          container:
+            name: xenial
+          env:
+            CC: gcc
+            CMAKE_GENERATOR: Ninja
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DUSE_GSSAPI=ON -DUSE_SSH=exec
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+        - name: "Linux (Xenial, Clang, mbedTLS, libssh2)"
+          id: xenial-gcc-mbedtls
+          os: ubuntu-latest
+          container:
+            name: xenial
+          env:
+            CC: clang
+            CMAKE_GENERATOR: Ninja
+            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DUSE_GSSAPI=ON -DUSE_SSH=libssh2
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+        - name: "macOS"
+          id: macos
+          os: macos-14
+          setup-script: osx
+          env:
+            CC: clang
+            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DUSE_GSSAPI=ON
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            CMAKE_GENERATOR: Ninja
+            PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+        - name: "Windows (amd64, Visual Studio, Schannel)"
+          id: windows-amd64-vs
+          os: windows-2022
+          setup-script: win32
+          build_prefix: RelWithDebInfo
+          env:
+            ARCH: amd64
+            CMAKE_GENERATOR: Visual Studio 17 2022
+            CMAKE_OPTIONS: -A x64 -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
+            BUILD_TEMP: D:\Temp
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+        - name: "Windows (x86, Visual Studio, WinHTTP)"
+          id: windows-x86-vs
+          os: windows-2022
+          setup-script: win32
+          build_prefix: RelWithDebInfo
+          env:
+            ARCH: x86
+            CMAKE_GENERATOR: Visual Studio 17 2022
+            CMAKE_OPTIONS: -A Win32 -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
+            BUILD_TEMP: D:\Temp
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+        - name: "Windows (amd64, mingw, WinHTTP)"
+          id: windows-amd64-mingw
+          os: windows-2022
+          setup-script: mingw
+          env:
+            ARCH: amd64
+            CMAKE_GENERATOR: MinGW Makefiles
+            CMAKE_OPTIONS:
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            BUILD_TEMP: D:\Temp
+            BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+        - name: "Windows (x86, mingw, Schannel)"
+          id: windows-x86-mingw
+          os: windows-2022
+          setup-script: mingw
+          env:
+            ARCH: x86
+            CMAKE_GENERATOR: MinGW Makefiles
+            CMAKE_OPTIONS: -DUSE_HTTPS=Schannel
+            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            BUILD_TEMP: D:\Temp
+            BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+    env: ${{ matrix.platform.env }}
+    runs-on: ${{ matrix.platform.os }}
+    name: "A/B: ${{ matrix.platform.name }}"
+    steps:
+    - name: Check out control
+      uses: actions/checkout@v4
+      with:
+        path: source/control
+        ref: main
+    - name: Check out candidate
+      uses: actions/checkout@v4
+      with:
+        path: source/candidate
+    - name: Set up build environment
+      run: source/candidate/ci/setup-${{ matrix.platform.setup-script }}-build.sh
+      shell: bash
+      if: matrix.platform.setup-script != ''
+    - name: Setup QEMU
+      run: docker run --rm --privileged multiarch/qemu-user-static:register --reset
+      if: matrix.platform.container.qemu == true
+    - name: Set up container
+      uses: ./source/candidate/.github/actions/download-or-build-container
+      with:
+        registry: ${{ env.docker-registry }}
+        config-path: ${{ env.docker-config-path }}
+        container: ${{ matrix.platform.container.name }}
+        github_token: ${{ secrets.github_token }}
+        dockerfile: ${{ matrix.platform.container.dockerfile }}
+      if: matrix.platform.container.name != ''
+    - name: Prepare builds
+      run: |
+        mkdir build
+        mkdir build/control
+        mkdir build/candidate
+    - name: Build control
+      uses: ./source/control/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/control && ../../source/control/ci/build.sh
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+        cmake-global-options: -DDEPRECATE_HARD=ON -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON
+    - name: Build candidate
+      uses: ./source/candidate/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ../../source/candidate/ci/build.sh
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+        cmake-global-options: -DDEPRECATE_HARD=ON -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+    - name: Run control benchmarks
+      uses: ./source/control/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/control && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json )
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+    - name: Run candidate benchmarks
+      uses: ./source/candidate/.github/actions/run-build
+      with:
+        command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json )
+        container: ${{ matrix.platform.container.name }}
+        container-version: ${{ env.docker-registry-container-sha }}
+        shell: ${{ matrix.platform.shell }}
+    - name: Organize results
+      run: |
+        mkdir results
+        mv ${BUILD_WORKSPACE:-.}/build/control/results.json ./results/control.json
+        mv ${BUILD_WORKSPACE:-.}/build/candidate/results.json ./results/candidate.json
+    - name: Upload results
+      uses: actions/upload-artifact@v4
+      with:
+        name: results-${{ matrix.platform.id }}
+        path: results
+      if: always()
+
+  # Publish the results
+  publish:
+    name: Publish results
+    needs: [ build ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+    - name: Download test results
+      uses: actions/download-artifact@v4
+      with:
+        path: results
+    - name: Generate markdown
+      run: node ci/compare-benchmarks.js results results.md
+    - name: Update pull request
+      uses: actions/github-script@v7
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const fs = require('fs');
+          const markdown = fs.readFileSync('results.md');
+          github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: markdown
+          });

.github/workflows/benchmark.yml

@@ -37,16 +37,14 @@ jobs:
           setup-script: ubuntu
           env:
             CC: clang
-            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
-            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DUSE_GSSAPI=ON
         - name: "macOS"
           id: macos
           os: macos-latest
           setup-script: osx
           env:
             CC: clang
-            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
-            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DUSE_GSSAPI=ON
             PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
         - name: "Windows (amd64, Visual Studio)"
           id: windows
@@ -55,8 +53,7 @@ jobs:
           env:
             ARCH: amd64
             CMAKE_GENERATOR: Visual Studio 17 2022
-            CMAKE_OPTIONS: -A x64 -DDEPRECATE_HARD=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
-            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
+            CMAKE_OPTIONS: -A x64
       fail-fast: false
     name: "Benchmark ${{ matrix.platform.name }}"
     env: ${{ matrix.platform.env }}
@@ -89,6 +86,9 @@ jobs:
       run: |
         mkdir build && cd build
         ../source/ci/build.sh
+      env:
+        CMAKE_GLOBAL_OPTIONS: -DDEPRECATE_HARD=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
+        CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
       shell: bash
     - name: Benchmark
       run: |
@@ -115,7 +115,7 @@ jobs:
         fi
 
         mkdir benchmark && cd benchmark
-        ../source/tests/benchmarks/benchmark.sh \
+        ../source/benchmarks/cli/benchmark.sh \
             ${SUITE_FLAG} ${DEBUG_FLAG} \
             --admin \
             --baseline-cli "git" --cli "${GIT2_CLI}" --name libgit2 \

.github/workflows/experimental.yml

@@ -34,14 +34,14 @@ jobs:
           env:
             CC: clang
             CMAKE_GENERATOR: Ninja
-            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON -DEXPERIMENTAL_SHA256=ON
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON
         - name: "macOS (SHA256)"
           id: macos-sha256
-          os: macos-13
+          os: macos-14
           setup-script: osx
           env:
             CC: clang
-            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DEXPERIMENTAL_SHA256=ON
+            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON
             CMAKE_GENERATOR: Ninja
             PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
             SKIP_SSH_TESTS: true
@@ -52,7 +52,7 @@ jobs:
           env:
             ARCH: amd64
             CMAKE_GENERATOR: Visual Studio 17 2022
-            CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DEXPERIMENTAL_SHA256=ON
+            CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
             # TODO: this is a temporary removal
@@ -92,6 +92,7 @@ jobs:
         container: ${{ matrix.platform.container.name }}
         container-version: ${{ env.docker-registry-container-sha }}
         shell: ${{ matrix.platform.shell }}
+        cmake-global-options: -DDEPRECATE_HARD=ON -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -DEXPERIMENTAL_SHA256=ON
     - name: Test
       uses: ./source/.github/actions/run-build
       with:

.github/workflows/main.yml

@@ -34,7 +34,7 @@ jobs:
           env:
             CC: gcc
             CMAKE_GENERATOR: Ninja
-            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON
         - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)"
           id: noble-clang-mbedtls
           os: ubuntu-latest
@@ -42,7 +42,7 @@ jobs:
             name: noble
           env:
             CC: clang
-            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser
+            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser
             CMAKE_GENERATOR: Ninja
         - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)"
           id: xenial-gcc-openssl
@@ -52,7 +52,7 @@ jobs:
           env:
             CC: gcc
             CMAKE_GENERATOR: Ninja
-            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON
         - name: "Linux (Xenial, Clang, mbedTLS, libssh2)"
           id: xenial-gcc-mbedtls
           os: ubuntu-latest
@@ -61,14 +61,14 @@ jobs:
           env:
             CC: clang
             CMAKE_GENERATOR: Ninja
-            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2
+            CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2
         - name: "macOS"
           id: macos
-          os: macos-13
+          os: macos-14
           setup-script: osx
           env:
             CC: clang
-            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON
+            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON
             CMAKE_GENERATOR: Ninja
             PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
             SKIP_SSH_TESTS: true
@@ -80,7 +80,7 @@ jobs:
           env:
             ARCH: amd64
             CMAKE_GENERATOR: Visual Studio 17 2022
-            CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2
+            CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2
             BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
             BUILD_TEMP: D:\Temp
             SKIP_SSH_TESTS: true
@@ -92,7 +92,7 @@ jobs:
           env:
             ARCH: x86
             CMAKE_GENERATOR: Visual Studio 17 2022
-            CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2
+            CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2
             BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
             BUILD_TEMP: D:\Temp
             SKIP_SSH_TESTS: true
@@ -104,7 +104,7 @@ jobs:
           env:
             ARCH: amd64
             CMAKE_GENERATOR: MinGW Makefiles
-            CMAKE_OPTIONS: -DDEPRECATE_HARD=ON
+            CMAKE_OPTIONS:
             BUILD_TEMP: D:\Temp
             BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
             SKIP_SSH_TESTS: true
@@ -116,12 +116,49 @@ jobs:
           env:
             ARCH: x86
             CMAKE_GENERATOR: MinGW Makefiles
-            CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel
+            CMAKE_OPTIONS: -DUSE_HTTPS=Schannel
             BUILD_TEMP: D:\Temp
             BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
 
+        # All builds: reftable
+        - name: "Linux (Noble, GCC, OpenSSL, libssh2, reftable)"
+          id: noble-gcc-openssl-reftable
+          os: ubuntu-latest
+          container:
+            name: noble
+          env:
+            CC: gcc
+            CLAR_REF_FORMAT: reftable
+            CMAKE_GENERATOR: Ninja
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON
+        - name: "macOS (reftable)"
+          id: macos-reftable
+          os: macos-14
+          setup-script: osx
+          env:
+            CC: clang
+            CLAR_REF_FORMAT: reftable
+            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON
+            CMAKE_GENERATOR: Ninja
+            PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+        - name: "Windows (amd64, Visual Studio, Schannel, reftable)"
+          id: windows-amd64-vs-reftable
+          os: windows-2022
+          setup-script: win32
+          env:
+            ARCH: amd64
+            CLAR_REF_FORMAT: reftable
+            CMAKE_GENERATOR: Visual Studio 17 2022
+            CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2
+            BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
+            BUILD_TEMP: D:\Temp
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+
         # All builds: sanitizers
         - name: "Sanitizer (Memory)"
           id: sanitizer-memory
@@ -132,7 +169,7 @@ jobs:
           env:
             CC: clang
             CFLAGS: -fsanitize=memory -fsanitize-memory-track-origins=2 -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer
-            CMAKE_OPTIONS: -DCMAKE_C_EXTENSIONS=ON -DCMAKE_PREFIX_PATH=/usr/local/msan -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
+            CMAKE_OPTIONS: -DCMAKE_C_EXTENSIONS=ON -DCMAKE_PREFIX_PATH=/usr/local/msan -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
             CMAKE_GENERATOR: Ninja
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
@@ -147,7 +184,7 @@ jobs:
           env:
             CC: clang
             CFLAGS: -fsanitize=address -ggdb -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer
-            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
+            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
             CMAKE_GENERATOR: Ninja
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
@@ -162,7 +199,7 @@ jobs:
           env:
             CC: clang
             CFLAGS: -fsanitize=undefined,nullability -fno-sanitize-recover=undefined,nullability -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer
-            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
+            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
             CMAKE_GENERATOR: Ninja
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
@@ -177,7 +214,7 @@ jobs:
           env:
             CC: clang
             CFLAGS: -fsanitize=thread -fno-optimize-sibling-calls -fno-omit-frame-pointer
-            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
+            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON
             CMAKE_GENERATOR: Ninja
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
@@ -219,6 +256,7 @@ jobs:
         container: ${{ matrix.platform.container.name }}
         container-version: ${{ env.docker-registry-container-sha }}
         shell: ${{ matrix.platform.shell }}
+        cmake-global-options: -DDEPRECATE_HARD=ON -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON
     - name: Test
       uses: ./source/.github/actions/run-build
       with:

.github/workflows/nightly.yml

@@ -65,7 +65,7 @@ jobs:
             CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2
         - name: "macOS"
           id: macos
-          os: macos-13
+          os: macos-14
           setup-script: osx
           env:
             CC: clang
@@ -76,7 +76,7 @@ jobs:
             SKIP_NEGOTIATE_TESTS: true
         - name: "iOS"
           id: ios
-          os: macos-13
+          os: macos-14
           setup-script: ios
           env:
             CC: clang
@@ -355,7 +355,7 @@ jobs:
           os: ubuntu-latest
         - name: "macOS (SHA256)"
           id: macos-sha256
-          os: macos-13
+          os: macos-14
           setup-script: osx
           env:
             CC: clang
@@ -418,6 +418,7 @@ jobs:
         container: ${{ matrix.platform.container.name }}
         container-version: ${{ env.docker-registry-container-sha }}
         shell: ${{ matrix.platform.shell }}
+        cmake-global-options: -DDEPRECATE_HARD=ON -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON
     - name: Test
       uses: ./source/.github/actions/run-build
       with:

CMakeLists.txt

@@ -20,7 +20,8 @@ option(EXPERIMENTAL_SHA256     "Enable experimental SHA256 support (for R&D/test
 
 # Optional subsystems
 option(BUILD_SHARED_LIBS       "Build Shared Library (OFF for Static)"                  ON)
-option(BUILD_TESTS             "Build Tests using the Clar suite"                       ON)
+option(BUILD_TESTS             "Build the test suite"                                   ON)
+option(BUILD_BENCHMARKS        "Build the benchmark suite"                             OFF)
 option(BUILD_CLI               "Build the command-line interface"                       ON)
 option(BUILD_EXAMPLES          "Build library usage example apps"                      OFF)
 option(BUILD_FUZZERS           "Build the fuzz targets"                                OFF)
@@ -77,7 +78,7 @@ if(MSVC)
 endif()
 
 if(NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE)
-	set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE)
+	set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE)
 endif()
 
 
@@ -112,6 +113,10 @@ if(BUILD_TESTS)
 	add_subdirectory(tests)
 endif()
 
+if(BUILD_BENCHMARKS)
+	add_subdirectory(benchmarks)
+endif()
+
 if(BUILD_EXAMPLES)
 	add_subdirectory(examples)
 endif()

README.md

@@ -7,7 +7,7 @@ libgit2 - the Git linkable library
 | **main** branch builds | [![CI Build](https://github.com/libgit2/libgit2/actions/workflows/main.yml/badge.svg?branch=main&event=push)](https://github.com/libgit2/libgit2/actions/workflows/main.yml?query=event%3Apush+branch%3Amain) [![Experimental Features](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml/badge.svg?branch=main)](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml?query=event%3Apush+branch%3Amain) |
 | **v1.9 branch** builds | [![CI Build](https://github.com/libgit2/libgit2/actions/workflows/main.yml/badge.svg?branch=maint%2Fv1.9&event=push)](https://github.com/libgit2/libgit2/actions/workflows/main.yml?query=event%3Apush+branch%3Amaint%2Fv1.9) [![Experimental Features](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml/badge.svg?branch=maint%2Fv1.9)](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml?query=event%3Apush+branch%3Amaint%2Fv1.9) |
 | **v1.8 branch** builds | [![CI Build](https://github.com/libgit2/libgit2/actions/workflows/main.yml/badge.svg?branch=maint%2Fv1.8&event=push)](https://github.com/libgit2/libgit2/actions/workflows/main.yml?query=event%3Apush+branch%3Amaint%2Fv1.8) [![Experimental Features](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml/badge.svg?branch=maint%2Fv1.8)](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml?query=event%3Apush+branch%3Amaint%2Fv1.8) |
-| **Nightly** builds | [![Nightly Build](https://github.com/libgit2/libgit2/actions/workflows/nightly.yml/badge.svg?branch=main&event=schedule)](https://github.com/libgit2/libgit2/actions/workflows/nightly.yml) [![Coverity Scan Status](https://scan.coverity.com/projects/639/badge.svg)](https://scan.coverity.com/projects/639) |
+| **Nightly** builds | [![Nightly Build](https://github.com/libgit2/libgit2/actions/workflows/nightly.yml/badge.svg?branch=main&event=schedule)](https://github.com/libgit2/libgit2/actions/workflows/nightly.yml) |
 
 `libgit2` is a portable, pure C implementation of the Git core methods
 provided as a linkable library with a solid API, allowing to build Git
@@ -541,6 +541,7 @@ Here are the bindings to libgit2 that are currently available:
 * Swift
     * SwiftGit2 <https://github.com/SwiftGit2/SwiftGit2>
     * SwiftGitX <https://github.com/ibrahimcetin/SwiftGitX>
+    * swift-libgit2 <https://github.com/swift-developer-tools/swift-libgit2>
 * Tcl
     * lg2 <https://github.com/apnadkarni/tcl-libgit2>
 * Vala

benchmarks/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(libgit2)

benchmarks/cli/benchmark_helpers.sh

@@ -105,7 +105,7 @@ fullpath() {
 }
 
 resources_dir() {
-	cd "$(dirname "$0")/../resources" && pwd
+	cd "$(dirname "$0")/../../tests/resources" && pwd
 }
 
 temp_dir() {

benchmarks/libgit2/CMakeLists.txt

@@ -0,0 +1,78 @@
+# util: the unit tests for libgit2's utility library
+
+if(NOT "${CMAKE_VERSION}" VERSION_LESS 3.27)
+        cmake_policy(SET CMP0148 OLD)
+endif()
+
+set(Python_ADDITIONAL_VERSIONS 3 2.7)
+find_package(PythonInterp)
+
+if(NOT PYTHONINTERP_FOUND)
+        message(FATAL_ERROR "Could not find a python interpeter, which is needed to build the tests. "
+                             "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests")
+endif()
+
+set(CLAR_PATH "${PROJECT_SOURCE_DIR}/deps/clar")
+set(BENCHMARK_PATH "${CMAKE_CURRENT_SOURCE_DIR}")
+add_definitions(-DCLAR_TMPDIR=\"libgit2_bench\")
+add_definitions(-DCLAR_WIN32_LONGPATHS)
+add_definitions(-DCLAR_HAS_REALPATH)
+add_definitions(-D_FILE_OFFSET_BITS=64)
+
+file(GLOB BENCHMARK_SRC *.c *.h)
+list(SORT BENCHMARK_SRC)
+
+set(CLAR_SRC
+	"${CLAR_PATH}/clar.c"
+	"${CLAR_PATH}/clar.h"
+	"${CLAR_PATH}/clar/fixtures.h"
+	"${CLAR_PATH}/clar/print.h"
+	"${CLAR_PATH}/clar/summary.h"
+	"${CLAR_PATH}/clar/sandbox.h"
+	"${CLAR_PATH}/clar/fs.h")
+
+add_custom_command(
+	OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/clar.suite ${CMAKE_CURRENT_BINARY_DIR}/clar_suite.h
+	COMMAND ${PYTHON_EXECUTABLE} ${CLAR_PATH}/generate.py -p benchmark -o "${CMAKE_CURRENT_BINARY_DIR}" -f .
+	DEPENDS ${BENCHMARK_SRC}
+	WORKING_DIRECTORY ${BENCHMARK_PATH}
+)
+
+set_source_files_properties(
+	${CLAR_PATH}/clar.c
+	PROPERTIES OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clar.suite)
+
+add_executable(libgit2_benchmarks ${CLAR_SRC}
+	${BENCHMARK_SRC}
+	$<TARGET_OBJECTS:util>
+	${LIBGIT2_DEPENDENCY_OBJECTS})
+
+target_link_libraries(libgit2_benchmarks libgit2package ${LIBGIT2_SYSTEM_LIBS})
+if(NOT MSVC)
+	target_link_libraries(libgit2_benchmarks m)
+endif()
+
+ide_split_sources(libgit2_benchmarks)
+
+target_include_directories(libgit2_benchmarks PRIVATE
+        "${CLAR_PATH}"
+        "${libgit2_BINARY_DIR}/src/util"
+        "${libgit2_BINARY_DIR}/include"
+        "${libgit2_SOURCE_DIR}/src/util"
+        "${libgit2_SOURCE_DIR}/include"
+	"${CMAKE_CURRENT_BINARY_DIR}"
+        "${LIBGIT2_DEPENDENCY_INCLUDES}"
+        "${LIBGIT2_SYSTEM_INCLUDES}")
+
+#
+# Old versions of gcc require us to declare our test functions; don't do
+# this on newer compilers to avoid unnecessary recompilation.
+#
+if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0)
+	target_compile_options(libgit2_benchmarks PRIVATE -include "clar_suite.h")
+endif()
+
+if(MSVC_IDE)
+	set_target_properties(libgit2_benchmarks PROPERTIES COMPILE_FLAGS "/Yuprecompiled.h /FIprecompiled.h")
+	set_source_files_properties("precompiled.c" COMPILE_FLAGS "/Ycprecompiled.h")
+endif()

benchmarks/libgit2/main.c

@@ -0,0 +1,31 @@
+#include <stdio.h>
+
+#include "clar.h"
+#include <git2.h>
+
+#ifdef _WIN32
+int __cdecl main(int argc, char *argv[])
+#else
+int main(int argc, char *argv[])
+#endif
+{
+	int res;
+
+	clar_test_set_mode(CL_TEST_BENCHMARK);
+	clar_test_init(argc, argv);
+
+	res = git_libgit2_init();
+	if (res < 0) {
+		const git_error *err = git_error_last();
+		const char *msg = err ? err->message : "unknown failure";
+		fprintf(stderr, "failed to init libgit2: %s\n", msg);
+		return res;
+	}
+
+	/* Run the test suite */
+	res = clar_test_run();
+
+	clar_test_shutdown();
+
+	return res;
+}

benchmarks/libgit2/oid.c

@@ -0,0 +1,151 @@
+#include "clar.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include <git2.h>
+
+#define BENCHMARK_OID_COUNT 256
+
+static git_oid sha1_one[BENCHMARK_OID_COUNT];
+static git_oid *sha1_two;
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+static git_oid sha256_one[BENCHMARK_OID_COUNT];
+static git_oid *sha256_two;
+#endif
+
+static void update_data_to_val(git_oid *out, git_oid_t type, uint32_t val)
+{
+	unsigned char data[GIT_OID_MAX_SIZE] = {0};
+	size_t size;
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	size = (type == GIT_OID_SHA256) ? GIT_OID_SHA256_SIZE : GIT_OID_SHA1_SIZE;
+#else
+	size = GIT_OID_SHA1_SIZE;
+
+	((void)(type));
+#endif
+
+	memset(data, 0, GIT_OID_MAX_SIZE);
+
+	data[size - 1] = (unsigned char)(val & 0x000000ff);
+	data[size - 2] = (unsigned char)((val & 0x0000ff00) >> 8);
+	data[size - 3] = (unsigned char)((val & 0x00ff0000) >> 16);
+	data[size - 4] = (unsigned char)((val & 0x00ff0000) >> 24);
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	cl_assert(git_oid_from_raw(out, data, type) == 0);
+#else
+	cl_assert(git_oid_fromraw(out, data) == 0);
+#endif
+}
+
+void benchmark_oid__initialize(void)
+{
+	uint32_t accum = 0;
+	size_t i;
+
+	sha1_two = calloc(BENCHMARK_OID_COUNT, sizeof(git_oid));
+	cl_assert(sha1_two != NULL);
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	sha256_two = calloc(BENCHMARK_OID_COUNT, sizeof(git_oid));
+	cl_assert(sha256_two != NULL);
+#endif
+
+	for (i = 0; i < BENCHMARK_OID_COUNT; i++) {
+		update_data_to_val(&sha1_one[i], GIT_OID_SHA1, accum++);
+		update_data_to_val(&sha1_two[i], GIT_OID_SHA1, accum++);
+	}
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	for (i = 0; i < BENCHMARK_OID_COUNT; i++) {
+		update_data_to_val(&sha256_one[i], GIT_OID_SHA256, accum++);
+		update_data_to_val(&sha256_two[i], GIT_OID_SHA256, accum++);
+	}
+#endif
+}
+
+void benchmark_oid__reset(void)
+{
+}
+
+void benchmark_oid__cleanup(void)
+{
+	free(sha1_two);
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	free(sha256_two);
+#endif
+}
+
+void benchmark_oid__cmp_sha1(void)
+{
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_cmp(&sha1_one[j], &sha1_two[j]);
+}
+
+void benchmark_oid__cmp_sha256(void)
+{
+#ifdef GIT_EXPERIMENTAL_SHA256
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_cmp(&sha256_one[j], &sha256_two[j]);
+#else
+	clar__skip();
+#endif
+}
+
+void benchmark_oid__cpy_sha1(void)
+{
+	git_oid dest;
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_cpy(&dest, &sha1_one[j]);
+}
+
+void benchmark_oid__cpy_sha256(void)
+{
+#ifdef GIT_EXPERIMENTAL_SHA256
+	git_oid dest;
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_cpy(&dest, &sha256_one[j]);
+#else
+	clar__skip();
+#endif
+}
+
+void benchmark_oid__zero_sha1(void)
+{
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_is_zero(&sha1_one[j]);
+}
+
+void benchmark_oid__zero_sha256(void)
+{
+#ifdef GIT_EXPERIMENTAL_SHA256
+	size_t i, j;
+
+	for (i = 0; i < 1024 * 16; i++)
+		for (j = 0; j < BENCHMARK_OID_COUNT; j++)
+			git_oid_is_zero(&sha256_one[j]);
+#else
+	clar__skip();
+#endif
+}

benchmarks/libgit2/precompiled.c

@@ -0,0 +1 @@
+#include "precompiled.h"

benchmarks/libgit2/precompiled.h

@@ -0,0 +1,2 @@
+#include "git2.h"
+#include "clar.h"

ci/build.sh

@@ -75,8 +75,8 @@ echo "##########################################################################
 echo "## Configuring build environment"
 echo "##############################################################################"
 
-echo "${CMAKE}" -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS} -S \"${SOURCE_DIR}\"
-env PATH="${BUILD_PATH}" "${CMAKE}" -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G "${CMAKE_GENERATOR}" ${CMAKE_OPTIONS} -S "${SOURCE_DIR}"
+echo "${CMAKE}" -G \"${CMAKE_GENERATOR}\" ${CMAKE_GLOBAL_OPTIONS} ${CMAKE_OPTIONS} -S \"${SOURCE_DIR}\"
+env PATH="${BUILD_PATH}" "${CMAKE}" -G "${CMAKE_GENERATOR}" ${CMAKE_GLOBAL_OPTIONS} ${CMAKE_OPTIONS} -S "${SOURCE_DIR}"
 
 echo ""
 echo "##############################################################################"

ci/compare-benchmarks.js

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+
+const fs = require('fs');
+const process = require('node:process');
+
+const path = process.argv[2] || '.';
+const output = process.argv[3];
+
+const dirs = fs.readdirSync(path);
+
+tests = [ ];
+results = { };
+
+for (const dir of dirs) {
+    const name = dir.replace(/^results-/, '');
+
+    const control = JSON.parse(fs.readFileSync(`${path}/${dir}/control.json`));
+    const candidate = JSON.parse(fs.readFileSync(`${path}/${dir}/candidate.json`));
+
+    results[name] = { };
+
+    const controlTests = control.tests.map((x) => x.name);
+    const candidateTests = candidate.tests.map((x) => x.name);
+    tests = [...new Set(tests.concat(controlTests).concat(candidateTests))];
+
+    for (const test of tests) {
+        const controlResults = control.tests.filter((x) => x.name === test)[0];
+        const candidateResults = candidate.tests.filter((x) => x.name === test)[0];
+
+        let result;
+
+        if (controlResults.results.status === 'ok' &&
+            candidateResults.results.status === 'ok') {
+            const delta = (candidateResults.results.mean - controlResults.results.mean) / candidateResults.results.mean;
+            const deltaPercent = Math.round(Math.abs(delta) * 1000) / 10;
+            const direction = (delta > 0) ? 'slower' : 'faster';
+            let highlight = '';
+
+            if (delta > 0.1) {
+                highlight = '**';
+            }
+
+            results[name][test] = `${highlight}${deltaPercent}% ${direction}${highlight}`;
+        }
+        else if (controlResults.results.status === 'ok') {
+            results[name][test] = `${candidateResults.results.stauts} in candidate`;
+        }
+        else if (candidateResults.results.status === 'ok') {
+            results[name][test] = `${controlResults.results.stauts} in control`;
+        }
+        else {
+            results[name][test] = controlResults.results.status;
+        }
+    }
+}
+
+let markdown = '| Platform \\ Test |';
+
+for (const test in Object.values(results)[0]) {
+    markdown += ` ${test} |`;
+}
+markdown += '\n';
+
+markdown += `| --- |`;
+for (const test in Object.values(results)[0]) {
+    markdown += ` --- |`;
+}
+markdown += '\n';
+
+for (const name in results) {
+    markdown += `| ${name.replaceAll(/-/g, '‑')} |`;
+
+    for (const test in results[name]) {
+        markdown += ` ${results[name][test]} |`;
+    }
+
+    markdown += `\n`;
+}
+
+if (output) {
+    fs.writeFileSync(output, markdown);
+}
+else {
+    console.log(markdown);
+}
+
+function p(arr, q) {
+    const array = [...arr].sort();
+    const pos = (array.length - 1) * q;
+    const base = Math.floor(pos);
+    const rest = pos - base;
+
+    if (array[base + 1] !== undefined) {
+        return array[base] + rest * (array[base + 1] - array[base]);
+    }
+    else {
+        return array[base];
+    }
+}

ci/test.sh

@@ -198,7 +198,7 @@ if should_run "PROXY_TESTS"; then
 fi
 
 if should_run "NTLM_TESTS" || should_run "ONLINE_TESTS"; then
-	curl --location --silent --show-error https://github.com/ethomson/poxygit/releases/download/v0.6.0/poxygit-0.6.0.jar >poxygit.jar
+	curl --location --silent --show-error https://github.com/ethomson/poxygit/releases/download/v0.8.1/poxygit-0.8.1.jar >poxygit.jar
 
 	echo "Starting HTTP server..."
 	HTTP_DIR=`mktemp -d ${TMPDIR}/http.XXXXXXXX`
@@ -216,18 +216,13 @@ if should_run "SSH_TESTS"; then
 	cat >"${SSHD_DIR}/sshd_config" <<-EOF
 	Port 2222
 	ListenAddress 0.0.0.0
-	Protocol 2
 	HostKey ${SSHD_DIR}/id_${GITTEST_SSH_KEYTYPE}
 	PidFile ${SSHD_DIR}/pid
 	AuthorizedKeysFile ${HOME}/.ssh/authorized_keys
 	LogLevel DEBUG
-	RSAAuthentication yes
 	PasswordAuthentication yes
 	PubkeyAuthentication yes
-	ChallengeResponseAuthentication no
 	StrictModes no
-	HostCertificate ${SSHD_DIR}/id_${GITTEST_SSH_KEYTYPE}.pub
-	HostKey ${SSHD_DIR}/id_${GITTEST_SSH_KEYTYPE}
 	# Required here as sshd will simply close connection otherwise
 	UsePAM no
 	EOF
@@ -304,10 +299,10 @@ if should_run "ONLINE_TESTS"; then
 	echo "## Running networking (online) tests"
 	echo "##############################################################################"
 
-	export GITTEST_REMOTE_REDIRECT_INITIAL="http://localhost:9000/initial-redirect/libgit2/TestGitRepository"
+	export GITTEST_REMOTE_REDIRECT_INITIAL="http://localhost:9000/initial-redirect:none/libgit2/TestGitRepository"
 	export GITTEST_REMOTE_REDIRECT_SUBSEQUENT="http://localhost:9000/subsequent-redirect/libgit2/TestGitRepository"
-	export GITTEST_REMOTE_SPEED_SLOW="http://localhost:9000/speed-9600/test.git"
-	export GITTEST_REMOTE_SPEED_TIMESOUT="http://localhost:9000/speed-0.5/test.git"
+	export GITTEST_REMOTE_SPEED_SLOW="http://localhost:9000/speed:9600/test.git"
+	export GITTEST_REMOTE_SPEED_TIMESOUT="http://localhost:9000/speed:0.5/test.git"
 	run_test online
 	unset GITTEST_REMOTE_REDIRECT_INITIAL
 	unset GITTEST_REMOTE_REDIRECT_SUBSEQUENT

cmake/SelectSSH.cmake

@@ -6,7 +6,9 @@ if(USE_SSH STREQUAL "exec")
 elseif(USE_SSH STREQUAL ON OR USE_SSH STREQUAL "libssh2")
 	find_pkglibraries(LIBSSH2 libssh2)
 
-	if(NOT LIBSSH2_FOUND)
+	if(LIBSSH2_FOUND)
+		set(LIBSSH2_FOUND_PKGCONFIG 1)
+	else()
 		find_package(LibSSH2)
 		set(LIBSSH2_INCLUDE_DIRS ${LIBSSH2_INCLUDE_DIR})
 		get_filename_component(LIBSSH2_LIBRARY_DIRS "${LIBSSH2_LIBRARY}" DIRECTORY)
@@ -20,7 +22,11 @@ elseif(USE_SSH STREQUAL ON OR USE_SSH STREQUAL "libssh2")
 
 	list(APPEND LIBGIT2_SYSTEM_INCLUDES ${LIBSSH2_INCLUDE_DIRS})
 	list(APPEND LIBGIT2_SYSTEM_LIBS ${LIBSSH2_LIBRARIES})
-	list(APPEND LIBGIT2_PC_LIBS ${LIBSSH2_LDFLAGS})
+	if(LIBSSH2_FOUND_PKGCONFIG)
+		list(APPEND LIBGIT2_PC_REQUIRES "libssh2")
+	else()
+		list(APPEND LIBGIT2_PC_LIBS ${LIBSSH2_LDFLAGS})
+	endif()
 
 	check_library_exists("${LIBSSH2_LIBRARIES}" libssh2_userauth_publickey_frommemory "${LIBSSH2_LIBRARY_DIRS}" HAVE_LIBSSH2_MEMORY_CREDENTIALS)
 	if(HAVE_LIBSSH2_MEMORY_CREDENTIALS)

deps/clar/clar.c

@@ -24,6 +24,14 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#ifndef va_copy
+#	ifdef __va_copy
+#		define va_copy(dst, src) __va_copy(dst, src)
+#	else
+#		define va_copy(dst, src) ((dst) = (src))
+#	endif
+#endif
+
 #if defined(__UCLIBC__) && ! defined(__UCLIBC_HAS_WCHAR__)
 	/*
 	 * uClibc can optionally be built without wchar support, in which case
@@ -76,17 +84,23 @@
 #			define S_ISDIR(x) ((x & _S_IFDIR) != 0)
 #		endif
 #		define p_snprintf(buf,sz,fmt,...) _snprintf_s(buf,sz,_TRUNCATE,fmt,__VA_ARGS__)
+#		define p_vsnprintf _vsnprintf
 #	else
 #		define p_snprintf snprintf
+#		define p_vsnprintf vsnprintf
 #	endif
+
+#	define localtime_r(timer, buf) (localtime_s(buf, timer) == 0 ? buf : NULL)
 #else
 #	include <sys/wait.h> /* waitpid(2) */
 #	include <unistd.h>
 #	define _MAIN_CC
 #	define p_snprintf snprintf
+#	define p_vsnprintf vsnprintf
 	typedef struct stat STAT_T;
 #endif
 
+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
 #define MAX(x, y) (((x) > (y)) ? (x) : (y))
 
 #include "clar.h"
@@ -100,11 +114,11 @@ fixture_path(const char *base, const char *fixture_name);
 #endif
 
 struct clar_error {
-	const char *file;
-	const char *function;
-	uintmax_t line_number;
-	const char *error_msg;
+	const char *message;
 	char *description;
+	const char *function;
+	const char *file;
+	uintmax_t line_number;
 
 	struct clar_error *next;
 };
@@ -117,13 +131,22 @@ struct clar_explicit {
 };
 
 struct clar_report {
-	const char *test;
-	int test_number;
 	const char *suite;
+	const char *test;
+	const char *description;
+	int test_number;
+
+	int runs;
 
 	enum cl_test_status status;
 	time_t start;
-	double elapsed;
+
+	double *times;
+	double time_min;
+	double time_max;
+	double time_mean;
+	double time_stddev;
+	double time_total;
 
 	struct clar_error *errors;
 	struct clar_error *last_error;
@@ -137,10 +160,12 @@ struct clar_summary {
 };
 
 static struct {
+	enum cl_test_mode test_mode;
 	enum cl_test_status test_status;
 
-	const char *active_test;
 	const char *active_suite;
+	const char *active_test;
+	const char *active_description;
 
 	int total_skipped;
 	int total_errors;
@@ -149,8 +174,8 @@ static struct {
 	int suites_ran;
 
 	enum cl_output_format output_format;
+	enum cl_summary_format summary_format;
 
-	int report_errors_only;
 	int exit_on_error;
 	int verbosity;
 
@@ -181,28 +206,32 @@ static struct {
 
 struct clar_func {
 	const char *name;
+	const char *description;
+	int runs;
 	void (*ptr)(void);
 };
 
 struct clar_suite {
 	const char *name;
 	struct clar_func initialize;
+	struct clar_func reset;
 	struct clar_func cleanup;
 	const struct clar_func *tests;
 	size_t test_count;
 	int enabled;
 };
 
-/* From clar_print_*.c */
-static void clar_print_init(int test_count, int suite_count, const char *suite_names);
+/* From print.h */
+static void clar_print_init(int test_count, int suite_count);
 static void clar_print_shutdown(int test_count, int suite_count, int error_count);
 static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error);
-static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status failed);
-static void clar_print_onsuite(const char *suite_name, int suite_index);
+static void clar_print_suite_start(const char *suite_name, int suite_index);
+static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number);
+static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report);
 static void clar_print_onabortv(const char *msg, va_list argp);
 static void clar_print_onabort(const char *msg, ...);
 
-/* From clar_sandbox.c */
+/* From sandbox.c */
 static void clar_tempdir_init(void);
 static void clar_tempdir_shutdown(void);
 static int clar_sandbox_create(const char *suite_name, const char *test_name);
@@ -212,6 +241,8 @@ static int clar_sandbox_cleanup(void);
 static struct clar_summary *clar_summary_init(const char *filename);
 static int clar_summary_shutdown(struct clar_summary *fp);
 
+#include "clar/counter.h"
+
 /* Load the declarations for the test suite */
 #include "clar.suite"
 
@@ -268,70 +299,122 @@ clar_report_all(void)
 	}
 }
 
-#ifdef WIN32
-# define clar_time DWORD
-
-static void clar_time_now(clar_time *out)
+static void
+compute_times(void)
 {
-	*out = GetTickCount();
-}
+	double total_squares = 0;
+	int i;
 
-static double clar_time_diff(clar_time *start, clar_time *end)
-{
-	return ((double)*end - (double)*start) / 1000;
-}
-#else
-# include <sys/time.h>
+	_clar.last_report->time_min = _clar.last_report->times[0];
+	_clar.last_report->time_max = _clar.last_report->times[0];
+	_clar.last_report->time_total = _clar.last_report->times[0];
 
-# define clar_time struct timeval
+	for (i = 1; i < _clar.last_report->runs; i++) {
+		if (_clar.last_report->times[i] < _clar.last_report->time_min)
+			_clar.last_report->time_min = _clar.last_report->times[i];
 
-static void clar_time_now(clar_time *out)
-{
-	gettimeofday(out, NULL);
-}
+		if (_clar.last_report->times[i] > _clar.last_report->time_max)
+			_clar.last_report->time_max = _clar.last_report->times[i];
 
-static double clar_time_diff(clar_time *start, clar_time *end)
-{
-	return ((double)end->tv_sec + (double)end->tv_usec / 1.0E6) -
-	       ((double)start->tv_sec + (double)start->tv_usec / 1.0E6);
+		_clar.last_report->time_total += _clar.last_report->times[i];
+	}
+
+	if (_clar.last_report->runs <= 1) {
+		_clar.last_report->time_stddev = 0;
+	} else {
+		_clar.last_report->time_mean = _clar.last_report->time_total / _clar.last_report->runs;
+
+		for (i = 0; i < _clar.last_report->runs; i++) {
+			double dev = (_clar.last_report->times[i] > _clar.last_report->time_mean) ?
+				_clar.last_report->times[i] - _clar.last_report->time_mean :
+				_clar.last_report->time_mean - _clar.last_report->times[i];
+
+			total_squares += (dev * dev);
+		}
+
+		_clar.last_report->time_stddev = sqrt(total_squares / _clar.last_report->runs);
+	}
 }
-#endif
 
 static void
 clar_run_test(
 	const struct clar_suite *suite,
 	const struct clar_func *test,
 	const struct clar_func *initialize,
+	const struct clar_func *reset,
 	const struct clar_func *cleanup)
 {
-	clar_time start, end;
+	int runs = test->runs;
+	volatile int i = 0;
 
-	_clar.trampoline_enabled = 1;
+	_clar.last_report->start = time(NULL);
+	_clar.last_report->times = &_clar.last_report->time_mean;
 
 	CL_TRACE(CL_TRACE__TEST__BEGIN);
 
 	clar_sandbox_create(suite->name, test->name);
 
-	_clar.last_report->start = time(NULL);
-	clar_time_now(&start);
+	clar_print_test_start(suite->name, test->name, _clar.tests_ran);
+
+	_clar.trampoline_enabled = 1;
 
 	if (setjmp(_clar.trampoline) == 0) {
 		if (initialize->ptr != NULL)
 			initialize->ptr();
 
 		CL_TRACE(CL_TRACE__TEST__RUN_BEGIN);
-		test->ptr();
+
+		do {
+			struct clar_counter start, end;
+			double elapsed;
+
+			if (i > 0 && reset->ptr != NULL) {
+				reset->ptr();
+			} else if (i > 0) {
+				if (_clar.local_cleanup != NULL)
+					_clar.local_cleanup(_clar.local_cleanup_payload);
+				if (cleanup->ptr != NULL)
+					cleanup->ptr();
+				if (initialize->ptr != NULL)
+					initialize->ptr();
+			}
+
+			clar_counter_now(&start);
+			test->ptr();
+			clar_counter_now(&end);
+
+			elapsed = clar_counter_diff(&start, &end);
+
+			/*
+			 * unless the number of runs was explicitly given
+			 * in benchmark mode, use the first run as a sample
+			 * to determine how many runs we should attempt
+			 */
+			if (_clar.test_mode == CL_TEST_BENCHMARK && !runs) {
+				runs = MAX(CLAR_BENCHMARK_RUN_MIN, (int)(CLAR_BENCHMARK_RUN_TIME / elapsed));
+				runs = MIN(CLAR_BENCHMARK_RUN_MAX, runs);
+			}
+
+			if (i == 0 && runs > 1) {
+				_clar.last_report->times = calloc(runs, sizeof(double));
+
+				if (_clar.last_report->times == NULL)
+					clar_abort("Failed to allocate report times.\n");
+			}
+
+			_clar.last_report->runs++;
+			_clar.last_report->times[i] = elapsed;
+		} while(++i < runs);
+
 		CL_TRACE(CL_TRACE__TEST__RUN_END);
 	}
 
-	clar_time_now(&end);
-
 	_clar.trampoline_enabled = 0;
 
 	if (_clar.last_report->status == CL_TEST_NOTRUN)
 		_clar.last_report->status = CL_TEST_OK;
 
-	_clar.last_report->elapsed = clar_time_diff(&start, &end);
+	compute_times();
 
 	if (_clar.local_cleanup != NULL)
 		_clar.local_cleanup(_clar.local_cleanup_payload);
@@ -351,18 +434,14 @@ clar_run_test(
 	_clar.local_cleanup = NULL;
 	_clar.local_cleanup_payload = NULL;
 
-	if (_clar.report_errors_only) {
-		clar_report_errors(_clar.last_report);
-	} else {
-		clar_print_ontest(suite->name, test->name, _clar.tests_ran, _clar.last_report->status);
-	}
+	clar_print_test_finish(suite->name, test->name, _clar.tests_ran, _clar.last_report);
 }
 
 static void
 clar_run_suite(const struct clar_suite *suite, const char *filter)
 {
 	const struct clar_func *test = suite->tests;
-	size_t i, matchlen;
+	size_t i, matchlen = 0;
 	struct clar_report *report;
 	int exact = 0;
 
@@ -372,11 +451,11 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)
 	if (_clar.exit_on_error && _clar.total_errors)
 		return;
 
-	if (!_clar.report_errors_only)
-		clar_print_onsuite(suite->name, ++_clar.suites_ran);
+	clar_print_suite_start(suite->name, ++_clar.suites_ran);
 
 	_clar.active_suite = suite->name;
 	_clar.active_test = NULL;
+	_clar.active_description = NULL;
 	CL_TRACE(CL_TRACE__SUITE_BEGIN);
 
 	if (filter) {
@@ -405,11 +484,13 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)
 			continue;
 
 		_clar.active_test = test[i].name;
+		_clar.active_description = test[i].description;
 
 		if ((report = calloc(1, sizeof(*report))) == NULL)
 			clar_abort("Failed to allocate report.\n");
 		report->suite = _clar.active_suite;
 		report->test = _clar.active_test;
+		report->description = _clar.active_description;
 		report->test_number = _clar.tests_ran;
 		report->status = CL_TEST_NOTRUN;
 
@@ -421,13 +502,14 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)
 
 		_clar.last_report = report;
 
-		clar_run_test(suite, &test[i], &suite->initialize, &suite->cleanup);
+		clar_run_test(suite, &test[i], &suite->initialize, &suite->reset, &suite->cleanup);
 
 		if (_clar.exit_on_error && _clar.total_errors)
 			return;
 	}
 
 	_clar.active_test = NULL;
+	_clar.active_description = NULL;
 	CL_TRACE(CL_TRACE__SUITE_END);
 }
 
@@ -440,7 +522,7 @@ clar_usage(const char *arg)
 	printf("  -iname        Include the suite with `name`\n");
 	printf("  -xname        Exclude the suite with `name`\n");
 	printf("  -v            Increase verbosity (show suite names)\n");
-	printf("  -q            Only report tests that had an error\n");
+	printf("  -q            Decrease verbosity, inverse to -v\n");
 	printf("  -Q            Quit as soon as a test fails\n");
 	printf("  -t            Display results in tap format\n");
 	printf("  -l            Print suite names\n");
@@ -532,7 +614,7 @@ clar_parse_args(int argc, char **argv)
 			if (argument[2] != '\0')
 				clar_usage(argv[0]);
 
-			_clar.report_errors_only = 1;
+			_clar.verbosity--;
 			break;
 
 		case 'Q':
@@ -593,14 +675,18 @@ clar_test_init(int argc, char **argv)
 {
 	const char *summary_env;
 
+	if (_clar.test_mode == CL_TEST_BENCHMARK) {
+		_clar.output_format = CL_OUTPUT_TIMING;
+		_clar.summary_format = CL_SUMMARY_JSON;
+	} else {
+		_clar.output_format = CL_OUTPUT_CLAP;
+		_clar.summary_format = CL_SUMMARY_JUNIT;
+	}
+
 	if (argc > 1)
 		clar_parse_args(argc, argv);
 
-	clar_print_init(
-		(int)_clar_callback_count,
-		(int)_clar_suite_count,
-		""
-	);
+	clar_print_init((int)_clar_callback_count, (int)_clar_suite_count);
 
 	if (!_clar.summary_filename &&
 	    (summary_env = getenv("CLAR_SUMMARY")) != NULL) {
@@ -619,6 +705,12 @@ clar_test_init(int argc, char **argv)
 	clar_tempdir_init();
 }
 
+void
+clar_test_set_mode(enum cl_test_mode mode)
+{
+	_clar.test_mode = mode;
+}
+
 int
 clar_test_run(void)
 {
@@ -668,6 +760,9 @@ clar_test_shutdown(void)
 			free(error);
 		}
 
+		if (report->times != &report->time_mean)
+			free(report->times);
+
 		report_next = report->next;
 		free(report);
 	}
@@ -707,13 +802,14 @@ void clar__skip(void)
 	abort_test();
 }
 
-void clar__fail(
+static void clar__failv(
 	const char *file,
 	const char *function,
 	size_t line,
+	int should_abort,
 	const char *error_msg,
 	const char *description,
-	int should_abort)
+	va_list args)
 {
 	struct clar_error *error;
 
@@ -731,11 +827,21 @@ void clar__fail(
 	error->file = _clar.invoke_file ? _clar.invoke_file : file;
 	error->function = _clar.invoke_func ? _clar.invoke_func : function;
 	error->line_number = _clar.invoke_line ? _clar.invoke_line : line;
-	error->error_msg = error_msg;
+	error->message = error_msg;
 
-	if (description != NULL &&
-	    (error->description = strdup(description)) == NULL)
-		clar_abort("Failed to allocate description.\n");
+	if (description != NULL) {
+		va_list args_copy;
+		int len;
+
+		va_copy(args_copy, args);
+		if ((len = p_vsnprintf(NULL, 0, description, args_copy)) < 0)
+			clar_abort("Failed to compute description.");
+		va_end(args_copy);
+
+		if ((error->description = calloc(1, len + 1)) == NULL)
+			clar_abort("Failed to allocate buffer.");
+		p_vsnprintf(error->description, len + 1, description, args);
+	}
 
 	_clar.total_errors++;
 	_clar.last_report->status = CL_TEST_FAILURE;
@@ -744,19 +850,47 @@ void clar__fail(
 		abort_test();
 }
 
-void clar__assert(
-	int condition,
+void clar__failf(
+	const char *file,
+	const char *function,
+	size_t line,
+	int should_abort,
+	const char *error_msg,
+	const char *description,
+	...)
+{
+	va_list args;
+	va_start(args, description);
+	clar__failv(file, function, line, should_abort, error_msg,
+		    description, args);
+	va_end(args);
+}
+
+void clar__fail(
 	const char *file,
 	const char *function,
 	size_t line,
 	const char *error_msg,
 	const char *description,
 	int should_abort)
+{
+	clar__failf(file, function, line, should_abort, error_msg,
+		    description ? "%s" : NULL, description);
+}
+
+void clar__assert(
+	int condition,
+	const char *file,
+	const char *function,
+	size_t line,
+	const char *error_message,
+	const char *error_description,
+	int should_abort)
 {
 	if (condition)
 		return;
 
-	clar__fail(file, function, line, error_msg, description, should_abort);
+	clar__fail(file, function, line, error_message, error_description, should_abort);
 }
 
 void clar__assert_equal(
@@ -787,7 +921,12 @@ void clar__assert_equal(
 				p_snprintf(buf, sizeof(buf), "'%s' != '%s' (at byte %d)",
 					s1, s2, pos);
 			} else {
-				p_snprintf(buf, sizeof(buf), "'%s' != '%s'", s1, s2);
+				const char *q1 = s1 ? "'" : "";
+				const char *q2 = s2 ? "'" : "";
+				s1 = s1 ? s1 : "NULL";
+				s2 = s2 ? s2 : "NULL";
+				p_snprintf(buf, sizeof(buf), "%s%s%s != %s%s%s",
+					   q1, s1, q1, q2, s2, q2);
 			}
 		}
 	}
@@ -800,12 +939,17 @@ void clar__assert_equal(
 		if (!is_equal) {
 			if (s1 && s2) {
 				int pos;
-				for (pos = 0; s1[pos] == s2[pos] && pos < len; ++pos)
+				for (pos = 0; pos < len && s1[pos] == s2[pos]; ++pos)
 					/* find differing byte offset */;
 				p_snprintf(buf, sizeof(buf), "'%.*s' != '%.*s' (at byte %d)",
 					len, s1, len, s2, pos);
 			} else {
-				p_snprintf(buf, sizeof(buf), "'%.*s' != '%.*s'", len, s1, len, s2);
+				const char *q1 = s1 ? "'" : "";
+				const char *q2 = s2 ? "'" : "";
+				s1 = s1 ? s1 : "NULL";
+				s2 = s2 ? s2 : "NULL";
+				p_snprintf(buf, sizeof(buf), "%s%.*s%s != %s%.*s%s",
+					   q1, len, s1, q1, q2, len, s2, q2);
 			}
 		}
 	}
@@ -823,7 +967,12 @@ void clar__assert_equal(
 				p_snprintf(buf, sizeof(buf), "'%ls' != '%ls' (at byte %d)",
 					wcs1, wcs2, pos);
 			} else {
-				p_snprintf(buf, sizeof(buf), "'%ls' != '%ls'", wcs1, wcs2);
+				const char *q1 = wcs1 ? "'" : "";
+				const char *q2 = wcs2 ? "'" : "";
+				wcs1 = wcs1 ? wcs1 : L"NULL";
+				wcs2 = wcs2 ? wcs2 : L"NULL";
+				p_snprintf(buf, sizeof(buf), "%s%ls%s != %s%ls%s",
+					   q1, wcs1, q1, q2, wcs2, q2);
 			}
 		}
 	}
@@ -836,12 +985,17 @@ void clar__assert_equal(
 		if (!is_equal) {
 			if (wcs1 && wcs2) {
 				int pos;
-				for (pos = 0; wcs1[pos] == wcs2[pos] && pos < len; ++pos)
+				for (pos = 0; pos < len && wcs1[pos] == wcs2[pos]; ++pos)
 					/* find differing byte offset */;
 				p_snprintf(buf, sizeof(buf), "'%.*ls' != '%.*ls' (at byte %d)",
 					len, wcs1, len, wcs2, pos);
 			} else {
-				p_snprintf(buf, sizeof(buf), "'%.*ls' != '%.*ls'", len, wcs1, len, wcs2);
+				const char *q1 = wcs1 ? "'" : "";
+				const char *q2 = wcs2 ? "'" : "";
+				wcs1 = wcs1 ? wcs1 : L"NULL";
+				wcs2 = wcs2 ? wcs2 : L"NULL";
+				p_snprintf(buf, sizeof(buf), "%s%.*ls%s != %s%.*ls%s",
+					   q1, len, wcs1, q1, q2, len, wcs2, q2);
 			}
 		}
 	}
@@ -859,8 +1013,7 @@ void clar__assert_equal(
 		void *p1 = va_arg(args, void *), *p2 = va_arg(args, void *);
 		is_equal = (p1 == p2);
 		if (!is_equal)
-			p_snprintf(buf, sizeof(buf), "0x%"PRIxPTR" != 0x%"PRIxPTR,
-				   (uintptr_t)p1, (uintptr_t)p2);
+			p_snprintf(buf, sizeof(buf), "%p != %p", p1, p2);
 	}
 	else {
 		int i1 = va_arg(args, int), i2 = va_arg(args, int);
@@ -878,6 +1031,92 @@ void clar__assert_equal(
 		clar__fail(file, function, line, err, buf, should_abort);
 }
 
+void clar__assert_compare_i(
+	const char *file,
+	const char *func,
+	size_t line,
+	int should_abort,
+	enum clar_comparison cmp,
+	intmax_t value1,
+	intmax_t value2,
+	const char *error,
+	const char *description,
+	...)
+{
+	int fulfilled;
+	switch (cmp) {
+	case CLAR_COMPARISON_EQ:
+		fulfilled = value1 == value2;
+		break;
+	case CLAR_COMPARISON_LT:
+		fulfilled = value1 < value2;
+		break;
+	case CLAR_COMPARISON_LE:
+		fulfilled = value1 <= value2;
+		break;
+	case CLAR_COMPARISON_GT:
+		fulfilled = value1 > value2;
+		break;
+	case CLAR_COMPARISON_GE:
+		fulfilled = value1 >= value2;
+		break;
+	default:
+		cl_assert(0);
+		return;
+	}
+
+	if (!fulfilled) {
+		va_list args;
+		va_start(args, description);
+		clar__failv(file, func, line, should_abort, error,
+			    description, args);
+		va_end(args);
+	}
+}
+
+void clar__assert_compare_u(
+	const char *file,
+	const char *func,
+	size_t line,
+	int should_abort,
+	enum clar_comparison cmp,
+	uintmax_t value1,
+	uintmax_t value2,
+	const char *error,
+	const char *description,
+	...)
+{
+	int fulfilled;
+	switch (cmp) {
+	case CLAR_COMPARISON_EQ:
+		fulfilled = value1 == value2;
+		break;
+	case CLAR_COMPARISON_LT:
+		fulfilled = value1 < value2;
+		break;
+	case CLAR_COMPARISON_LE:
+		fulfilled = value1 <= value2;
+		break;
+	case CLAR_COMPARISON_GT:
+		fulfilled = value1 > value2;
+		break;
+	case CLAR_COMPARISON_GE:
+		fulfilled = value1 >= value2;
+		break;
+	default:
+		cl_assert(0);
+		return;
+	}
+
+	if (!fulfilled) {
+		va_list args;
+		va_start(args, description);
+		clar__failv(file, func, line, should_abort, error,
+			    description, args);
+		va_end(args);
+	}
+}
+
 void cl_set_cleanup(void (*cleanup)(void *), void *opaque)
 {
 	_clar.local_cleanup = cleanup;

deps/clar/clar.h

@@ -7,6 +7,7 @@
 #ifndef __CLAR_TEST_H__
 #define __CLAR_TEST_H__
 
+#include <inttypes.h>
 #include <stdlib.h>
 #include <limits.h>
 
@@ -14,10 +15,22 @@
 # define CLAR_MAX_PATH 4096
 #elif defined(_WIN32)
 # define CLAR_MAX_PATH MAX_PATH
-#else
+#elif defined(PATH_MAX)
 # define CLAR_MAX_PATH PATH_MAX
+#else
+# define CLAR_MAX_PATH 4096
 #endif
 
+/*
+ * In benchmark mode, by default, clar will run the test repeatedly for
+ * approximately `CLAR_BENCHMARK_RUN_TIME` seconds, and at least
+ * `CLAR_BENCHMARK_RUN_MIN` iterations.
+ */
+
+#define CLAR_BENCHMARK_RUN_TIME 3.0
+#define CLAR_BENCHMARK_RUN_MIN  10
+#define CLAR_BENCHMARK_RUN_MAX  30000000
+
 #ifndef CLAR_SELFTEST
 # define CLAR_CURRENT_FILE __FILE__
 # define CLAR_CURRENT_LINE __LINE__
@@ -28,6 +41,11 @@
 # define CLAR_CURRENT_FUNC "func"
 #endif
 
+enum cl_test_mode {
+	CL_TEST_STANDARD,
+	CL_TEST_BENCHMARK,
+};
+
 enum cl_test_status {
 	CL_TEST_OK,
 	CL_TEST_FAILURE,
@@ -38,10 +56,17 @@ enum cl_test_status {
 enum cl_output_format {
 	CL_OUTPUT_CLAP,
 	CL_OUTPUT_TAP,
+	CL_OUTPUT_TIMING,
+};
+
+enum cl_summary_format {
+	CL_SUMMARY_JUNIT,
+	CL_SUMMARY_JSON,
 };
 
 /** Setup clar environment */
 void clar_test_init(int argc, char *argv[]);
+void clar_test_set_mode(enum cl_test_mode mode);
 int clar_test_run(void);
 void clar_test_shutdown(void);
 
@@ -149,6 +174,7 @@ const char *cl_fixture_basename(const char *fixture_name);
  * Forced failure/warning
  */
 #define cl_fail(desc) clar__fail(CLAR_CURRENT_FILE, CLAR_CURRENT_FUNC, CLAR_CURRENT_LINE, "Test failed.", desc, 1)
+#define cl_failf(desc,...) clar__failf(CLAR_CURRENT_FILE, CLAR_CURRENT_FUNC, CLAR_CURRENT_LINE, 1, "Test failed.", desc, __VA_ARGS__)
 #define cl_warning(desc) clar__fail(CLAR_CURRENT_FILE, CLAR_CURRENT_FUNC, CLAR_CURRENT_LINE, "Warning during test execution:", desc, 0)
 
 #define cl_skip() clar__skip()
@@ -168,9 +194,42 @@ const char *cl_fixture_basename(const char *fixture_name);
 #define cl_assert_equal_wcsn(wcs1,wcs2,len) clar__assert_equal(CLAR_CURRENT_FILE,CLAR_CURRENT_FUNC,CLAR_CURRENT_LINE,"String mismatch: " #wcs1 " != " #wcs2, 1, "%.*ls", (wcs1), (wcs2), (int)(len))
 #define cl_assert_equal_wcsn_(wcs1,wcs2,len,note) clar__assert_equal(CLAR_CURRENT_FILE,CLAR_CURRENT_FUNC,CLAR_CURRENT_LINE,"String mismatch: " #wcs1 " != " #wcs2 " (" #note ")", 1, "%.*ls", (wcs1), (wcs2), (int)(len))
 
-#define cl_assert_equal_i(i1,i2) clar__assert_equal(CLAR_CURRENT_FILE,CLAR_CURRENT_FUNC,CLAR_CURRENT_LINE,#i1 " != " #i2, 1, "%d", (int)(i1), (int)(i2))
-#define cl_assert_equal_i_(i1,i2,note) clar__assert_equal(CLAR_CURRENT_FILE,CLAR_CURRENT_FUNC,CLAR_CURRENT_LINE,#i1 " != " #i2 " (" #note ")", 1, "%d", (i1), (i2))
-#define cl_assert_equal_i_fmt(i1,i2,fmt) clar__assert_equal(CLAR_CURRENT_FILE,CLAR_CURRENT_FUNC,CLAR_CURRENT_LINE,#i1 " != " #i2, 1, (fmt), (int)(i1), (int)(i2))
+#define cl_assert_compare_i_(i1, i2, cmp, error, ...) clar__assert_compare_i(CLAR_CURRENT_FILE, CLAR_CURRENT_FUNC, CLAR_CURRENT_LINE, 1, cmp, \
+									     (i1), (i2), "Expected comparison to hold: " error, __VA_ARGS__)
+#define cl_assert_compare_i(i1, i2, cmp, error, fmt) do { \
+	intmax_t v1 = (i1), v2 = (i2); \
+	clar__assert_compare_i(CLAR_CURRENT_FILE, CLAR_CURRENT_FUNC, CLAR_CURRENT_LINE, 1, cmp, \
+			       v1, v2, "Expected comparison to hold: " error, fmt, v1, v2); \
+} while (0)
+#define cl_assert_equal_i_(i1, i2, ...)    cl_assert_compare_i_(i1, i2, CLAR_COMPARISON_EQ, #i1 " == " #i2, __VA_ARGS__)
+#define cl_assert_equal_i(i1, i2)          cl_assert_compare_i (i1, i2, CLAR_COMPARISON_EQ, #i1 " == " #i2, "%"PRIdMAX " != %"PRIdMAX)
+#define cl_assert_equal_i_fmt(i1, i2, fmt) cl_assert_compare_i_(i1, i2, CLAR_COMPARISON_EQ, #i1 " == " #i2,  fmt " != " fmt, (int)(i1), (int)(i2))
+#define cl_assert_lt_i_(i1, i2, ...) cl_assert_compare_i_(i1, i2, CLAR_COMPARISON_LT, #i1 " < " #i2, __VA_ARGS__)
+#define cl_assert_lt_i(i1, i2)       cl_assert_compare_i (i1, i2, CLAR_COMPARISON_LT, #i1 " < " #i2, "%"PRIdMAX " >= %"PRIdMAX)
+#define cl_assert_le_i_(i1, i2, ...) cl_assert_compare_i_(i1, i2, CLAR_COMPARISON_LE, #i1 " <= " #i2, __VA_ARGS__)
+#define cl_assert_le_i(i1, i2)       cl_assert_compare_i (i1, i2, CLAR_COMPARISON_LE, #i1 " <= " #i2, "%"PRIdMAX " > %"PRIdMAX)
+#define cl_assert_gt_i_(i1, i2, ...) cl_assert_compare_i_(i1, i2, CLAR_COMPARISON_GT, #i1 " > " #i2, __VA_ARGS__)
+#define cl_assert_gt_i(i1, i2)       cl_assert_compare_i (i1, i2, CLAR_COMPARISON_GT, #i1 " > " #i2, "%"PRIdMAX " <= %"PRIdMAX)
+#define cl_assert_ge_i_(i1, i2, ...) cl_assert_compare_i_(i1, i2, CLAR_COMPARISON_GE, #i1 " >= " #i2, __VA_ARGS__)
+#define cl_assert_ge_i(i1, i2)       cl_assert_compare_i (i1, i2, CLAR_COMPARISON_GE, #i1 " >= " #i2, "%"PRIdMAX " < %"PRIdMAX)
+
+#define cl_assert_compare_u_(u1, u2, cmp, error, ...) clar__assert_compare_u(CLAR_CURRENT_FILE, CLAR_CURRENT_FUNC, CLAR_CURRENT_LINE, 1, cmp, \
+									     (u1), (u2), "Expected comparison to hold: " error, __VA_ARGS__)
+#define cl_assert_compare_u(u1, u2, cmp, error, fmt) do { \
+	uintmax_t v1 = (u1), v2 = (u2); \
+	clar__assert_compare_u(CLAR_CURRENT_FILE, CLAR_CURRENT_FUNC, CLAR_CURRENT_LINE, 1, cmp, \
+			       v1, v2, "Expected comparison to hold: " error, fmt, v1, v2); \
+} while (0)
+#define cl_assert_equal_u_(u1, u2, ...) cl_assert_compare_u_(u1, u2, CLAR_COMPARISON_EQ, #u1 " == " #u2, __VA_ARGS__)
+#define cl_assert_equal_u(u1, u2)       cl_assert_compare_u (u1, u2, CLAR_COMPARISON_EQ, #u1 " == " #u2, "%"PRIuMAX " != %"PRIuMAX)
+#define cl_assert_lt_u_(u1, u2, ...) cl_assert_compare_u_(u1, u2, CLAR_COMPARISON_LT, #u1 " < " #u2, __VA_ARGS__)
+#define cl_assert_lt_u(u1, u2)       cl_assert_compare_u (u1, u2, CLAR_COMPARISON_LT, #u1 " < " #u2, "%"PRIuMAX " >= %"PRIuMAX)
+#define cl_assert_le_u_(u1, u2, ...) cl_assert_compare_u_(u1, u2, CLAR_COMPARISON_LE, #u1 " <= " #u2, __VA_ARGS__)
+#define cl_assert_le_u(u1, u2)       cl_assert_compare_u (u1, u2, CLAR_COMPARISON_LE, #u1 " <= " #u2, "%"PRIuMAX " > %"PRIuMAX)
+#define cl_assert_gt_u_(u1, u2, ...) cl_assert_compare_u_(u1, u2, CLAR_COMPARISON_GT, #u1 " > " #u2, __VA_ARGS__)
+#define cl_assert_gt_u(u1, u2)       cl_assert_compare_u (u1, u2, CLAR_COMPARISON_GT, #u1 " > " #u2, "%"PRIuMAX " <= %"PRIuMAX)
+#define cl_assert_ge_u_(u1, u2, ...) cl_assert_compare_u_(u1, u2, CLAR_COMPARISON_GE, #u1 " >= " #u2, __VA_ARGS__)
+#define cl_assert_ge_u(u1, u2)       cl_assert_compare_u (u1, u2, CLAR_COMPARISON_GE, #u1 " >= " #u2, "%"PRIuMAX " < %"PRIuMAX)
 
 #define cl_assert_equal_b(b1,b2) clar__assert_equal(CLAR_CURRENT_FILE,CLAR_CURRENT_FUNC,CLAR_CURRENT_LINE,#b1 " != " #b2, 1, "%d", (int)((b1) != 0),(int)((b2) != 0))
 
@@ -186,6 +245,15 @@ void clar__fail(
 	const char *description,
 	int should_abort);
 
+void clar__failf(
+	const char *file,
+	const char *func,
+	size_t line,
+	int should_abort,
+	const char *error,
+	const char *description,
+	...);
+
 void clar__assert(
 	int condition,
 	const char *file,
@@ -204,6 +272,38 @@ void clar__assert_equal(
 	const char *fmt,
 	...);
 
+enum clar_comparison {
+	CLAR_COMPARISON_EQ,
+	CLAR_COMPARISON_LT,
+	CLAR_COMPARISON_LE,
+	CLAR_COMPARISON_GT,
+	CLAR_COMPARISON_GE,
+};
+
+void clar__assert_compare_i(
+	const char *file,
+	const char *func,
+	size_t line,
+	int should_abort,
+	enum clar_comparison cmp,
+	intmax_t value1,
+	intmax_t value2,
+	const char *error,
+	const char *description,
+	...);
+
+void clar__assert_compare_u(
+	const char *file,
+	const char *func,
+	size_t line,
+	int should_abort,
+	enum clar_comparison cmp,
+	uintmax_t value1,
+	uintmax_t value2,
+	const char *error,
+	const char *description,
+	...);
+
 void clar__set_invokepoint(
 	const char *file,
 	const char *func,

deps/clar/clar/counter.h

@@ -0,0 +1,167 @@
+#define CLAR_COUNTER_TV_DIFF(out_sec, out_usec, start_sec, start_usec, end_sec, end_usec) \
+	if (start_usec > end_usec) { \
+		out_sec = (end_sec - 1) - start_sec; \
+		out_usec = (end_usec + 1000000) - start_usec; \
+	} else { \
+		out_sec = end_sec - start_sec; \
+		out_usec = end_usec - start_usec; \
+	}
+
+#ifdef _WIN32
+
+struct clar_counter {
+	LARGE_INTEGER value;
+};
+
+static void clar_counter_now(struct clar_counter *out)
+{
+	QueryPerformanceCounter(&out->value);
+}
+
+static double clar_counter_diff(
+	struct clar_counter *start,
+	struct clar_counter *end)
+{
+	LARGE_INTEGER freq;
+
+	QueryPerformanceFrequency(&freq);
+	return (double)(end->value.QuadPart - start->value.QuadPart)/(double)freq.QuadPart;
+}
+
+#elif __APPLE__
+
+#include <mach/mach_time.h>
+#include <sys/time.h>
+
+static double clar_counter_scaling_factor = -1;
+
+struct clar_counter {
+	union {
+		uint64_t absolute_time;
+		struct timeval tv;
+	} val;
+};
+
+static void clar_counter_now(struct clar_counter *out)
+{
+	if (clar_counter_scaling_factor == 0) {
+		mach_timebase_info_data_t info;
+
+		clar_counter_scaling_factor =
+			mach_timebase_info(&info) == KERN_SUCCESS ?
+			((double)info.numer / (double)info.denom) / 1.0E6 :
+			-1;
+	}
+
+	/* mach_timebase_info failed; fall back to gettimeofday */
+	if (clar_counter_scaling_factor < 0)
+		gettimeofday(&out->val.tv, NULL);
+	else
+		out->val.absolute_time = mach_absolute_time();
+}
+
+static double clar_counter_diff(
+	struct clar_counter *start,
+	struct clar_counter *end)
+{
+	if (clar_counter_scaling_factor < 0) {
+		time_t sec;
+		suseconds_t usec;
+
+		CLAR_COUNTER_TV_DIFF(sec, usec,
+			start->val.tv.tv_sec, start->val.tv.tv_usec,
+			end->val.tv.tv_sec, end->val.tv.tv_usec);
+
+		return (double)sec + ((double)usec / 1000000.0);
+	} else {
+		return (double)(end->val.absolute_time - start->val.absolute_time) *
+			clar_counter_scaling_factor;
+	}
+}
+
+#elif defined(__amigaos4__)
+
+#include <proto/timer.h>
+
+struct clar_counter {
+	struct TimeVal tv;
+}
+
+static void clar_counter_now(struct clar_counter *out)
+{
+	ITimer->GetUpTime(&out->tv);
+}
+
+static double clar_counter_diff(
+	struct clar_counter *start,
+	struct clar_counter *end)
+{
+	uint32_t sec, usec;
+
+	CLAR_COUNTER_TV_DIFF(sec, usec,
+		start->tv.Seconds, start->tv.Microseconds,
+		end->tv.Seconds, end->tv.Microseconds);
+
+	return (double)sec + ((double)usec / 1000000.0);
+}
+
+#else
+
+#include <sys/time.h>
+
+struct clar_counter {
+	int type;
+	union {
+#ifdef CLOCK_MONOTONIC
+		struct timespec tp;
+#endif
+		struct timeval tv;
+	} val;
+};
+
+static void clar_counter_now(struct clar_counter *out)
+{
+#ifdef CLOCK_MONOTONIC
+	if (clock_gettime(CLOCK_MONOTONIC, &out->val.tp) == 0) {
+		out->type = 0;
+		return;
+	}
+#endif
+
+	/* Fall back to using gettimeofday */
+	out->type = 1;
+	gettimeofday(&out->val.tv, NULL);
+}
+
+static double clar_counter_diff(
+	struct clar_counter *start,
+	struct clar_counter *end)
+{
+	time_t sec;
+	suseconds_t usec;
+
+#ifdef CLOCK_MONOTONIC
+	if (start->type == 0) {
+		time_t sec;
+		long nsec;
+
+		if (start->val.tp.tv_sec > end->val.tp.tv_sec) {
+			sec = (end->val.tp.tv_sec - 1) - start->val.tp.tv_sec;
+			nsec = (end->val.tp.tv_nsec + 1000000000) - start->val.tp.tv_nsec;
+		} else {
+			sec = end->val.tp.tv_sec - start->val.tp.tv_sec;
+			nsec = end->val.tp.tv_nsec - start->val.tp.tv_nsec;
+		}
+
+		return (double)sec + ((double)nsec / 1000000000.0);
+	}
+#endif
+
+	CLAR_COUNTER_TV_DIFF(sec, usec,
+		start->val.tv.tv_sec, start->val.tv.tv_usec,
+		end->val.tv.tv_sec, end->val.tv.tv_usec);
+
+	return (double)sec + ((double)usec / 1000000.0);
+}
+
+#endif

deps/clar/clar/fs.h

@@ -365,14 +365,19 @@ static void
 fs_copydir_helper(const char *source, const char *dest, int dest_mode)
 {
 	DIR *source_dir;
-	struct dirent *d;
 
 	mkdir(dest, dest_mode);
 
 	cl_assert_(source_dir = opendir(source), "Could not open source dir");
-	while ((d = (errno = 0, readdir(source_dir))) != NULL) {
+	while (1) {
+		struct dirent *d;
 		char *child;
 
+		errno = 0;
+		d = readdir(source_dir);
+		if (!d)
+			break;
+
 		if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
 			continue;
 
@@ -470,12 +475,18 @@ static void
 fs_rmdir_helper(const char *path)
 {
 	DIR *dir;
-	struct dirent *d;
 
 	cl_assert_(dir = opendir(path), "Could not open dir");
-	while ((d = (errno = 0, readdir(dir))) != NULL) {
+
+	while (1) {
+		struct dirent *d;
 		char *child;
 
+		errno = 0;
+		d = readdir(dir);
+		if (!d)
+			break;
+
 		if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
 			continue;
 

deps/clar/clar/print.h

@@ -0,0 +1,377 @@
+/* clap: clar protocol, the traditional clar output format */
+
+static void clar_print_clap_init(int test_count, int suite_count)
+{
+	(void)test_count;
+
+	if (_clar.verbosity < 0)
+		return;
+
+	printf("Loaded %d suites:\n", (int)suite_count);
+	printf("Started (test status codes: OK='.' FAILURE='F' SKIPPED='S')\n");
+}
+
+static void clar_print_clap_shutdown(int test_count, int suite_count, int error_count)
+{
+	(void)test_count;
+	(void)suite_count;
+	(void)error_count;
+
+	if (_clar.verbosity >= 0)
+		printf("\n\n");
+	clar_report_all();
+}
+
+
+static void clar_print_indented(const char *str, int indent)
+{
+	const char *bol, *eol;
+
+	for (bol = str; *bol; bol = eol) {
+		eol = strchr(bol, '\n');
+		if (eol)
+			eol++;
+		else
+			eol = bol + strlen(bol);
+		printf("%*s%.*s", indent, "", (int)(eol - bol), bol);
+	}
+	putc('\n', stdout);
+}
+
+static void clar_print_clap_error(int num, const struct clar_report *report, const struct clar_error *error)
+{
+	printf("  %d) Failure:\n", num);
+
+	printf("%s::%s [%s:%"PRIuMAX"]\n",
+		report->suite,
+		report->test,
+		error->file,
+		error->line_number);
+
+	clar_print_indented(error->message, 2);
+
+	if (error->description != NULL)
+		clar_print_indented(error->description, 2);
+
+	printf("\n");
+	fflush(stdout);
+}
+
+static void clar_print_clap_test_start(const char *suite_name, const char *test_name, int test_number)
+{
+	(void)test_number;
+
+	if (_clar.verbosity < 0)
+		return;
+
+	if (_clar.verbosity > 1) {
+		printf("%s::%s: ", suite_name, test_name);
+		fflush(stdout);
+	}
+}
+
+static void clar_print_clap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report)
+{
+	(void)suite_name;
+	(void)test_name;
+	(void)test_number;
+
+	if (_clar.verbosity == 0) {
+		switch (report->status) {
+		case CL_TEST_OK: printf("."); break;
+		case CL_TEST_FAILURE: printf("F"); break;
+		case CL_TEST_SKIP: printf("S"); break;
+		case CL_TEST_NOTRUN: printf("N"); break;
+		}
+
+		fflush(stdout);
+	} else if (_clar.verbosity > 1) {
+		switch (report->status) {
+		case CL_TEST_OK: printf("ok\n"); break;
+		case CL_TEST_FAILURE: printf("fail\n"); break;
+		case CL_TEST_SKIP: printf("skipped\n"); break;
+		case CL_TEST_NOTRUN: printf("notrun\n"); break;
+		}
+	}
+}
+
+static void clar_print_clap_suite_start(const char *suite_name, int suite_index)
+{
+	if (_clar.verbosity < 0)
+		return;
+	if (_clar.verbosity == 1)
+		printf("\n%s", suite_name);
+
+	(void)suite_index;
+}
+
+static void clar_print_clap_onabort(const char *fmt, va_list arg)
+{
+	vfprintf(stderr, fmt, arg);
+}
+
+/* tap: test anywhere protocol format */
+
+static void clar_print_tap_init(int test_count, int suite_count)
+{
+	(void)test_count;
+	(void)suite_count;
+	printf("TAP version 13\n");
+}
+
+static void clar_print_tap_shutdown(int test_count, int suite_count, int error_count)
+{
+	(void)suite_count;
+	(void)error_count;
+
+	printf("1..%d\n", test_count);
+}
+
+static void clar_print_tap_error(int num, const struct clar_report *report, const struct clar_error *error)
+{
+	(void)num;
+	(void)report;
+	(void)error;
+}
+
+static void print_escaped(const char *str)
+{
+	const char *c;
+
+	while ((c = strchr(str, '\'')) != NULL) {
+		printf("%.*s", (int)(c - str), str);
+		printf("''");
+		str = c + 1;
+	}
+
+	printf("%s", str);
+}
+
+static void clar_print_tap_test_start(const char *suite_name, const char *test_name, int test_number)
+{
+	(void)suite_name;
+	(void)test_name;
+	(void)test_number;
+}
+
+static void clar_print_tap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report)
+{
+	const struct clar_error *error = _clar.last_report->errors;
+
+	(void)test_name;
+	(void)test_number;
+
+	switch(report->status) {
+	case CL_TEST_OK:
+		printf("ok %d - %s::%s\n", test_number, suite_name, test_name);
+		break;
+	case CL_TEST_FAILURE:
+		printf("not ok %d - %s::%s\n", test_number, suite_name, test_name);
+
+		if (_clar.verbosity >= 0) {
+			printf("    ---\n");
+			printf("    reason: |\n");
+			clar_print_indented(error->message, 6);
+
+			if (error->description)
+				clar_print_indented(error->description, 6);
+
+			printf("    at:\n");
+			printf("      file: '"); print_escaped(error->file); printf("'\n");
+			printf("      line: %" PRIuMAX "\n", error->line_number);
+			printf("      function: '%s'\n", error->function);
+			printf("    ...\n");
+		}
+
+		break;
+	case CL_TEST_SKIP:
+	case CL_TEST_NOTRUN:
+		printf("ok %d - # SKIP %s::%s\n", test_number, suite_name, test_name);
+		break;
+	}
+
+	fflush(stdout);
+}
+
+static void clar_print_tap_suite_start(const char *suite_name, int suite_index)
+{
+	if (_clar.verbosity < 0)
+		return;
+	printf("# start of suite %d: %s\n", suite_index, suite_name);
+}
+
+static void clar_print_tap_onabort(const char *fmt, va_list arg)
+{
+	printf("Bail out! ");
+	vprintf(fmt, arg);
+	fflush(stdout);
+}
+
+/* timings format: useful for benchmarks */
+
+static void clar_print_timing_init(int test_count, int suite_count)
+{
+	(void)test_count;
+	(void)suite_count;
+
+	printf("Started benchmarks (mean time ± stddev / min time … max time):\n\n");
+}
+
+static void clar_print_timing_shutdown(int test_count, int suite_count, int error_count)
+{
+	(void)test_count;
+	(void)suite_count;
+	(void)error_count;
+}
+
+static void clar_print_timing_error(int num, const struct clar_report *report, const struct clar_error *error)
+{
+	(void)num;
+	(void)report;
+	(void)error;
+}
+
+static void clar_print_timing_test_start(const char *suite_name, const char *test_name, int test_number)
+{
+	(void)test_number;
+
+	printf("%s::%s:  ", suite_name, test_name);
+	fflush(stdout);
+}
+
+static void clar_print_timing_time(double t)
+{
+	static const char *units[] = { "sec", "ms", "μs", "ns" };
+	static const int units_len = sizeof(units) / sizeof(units[0]);
+	int unit = 0, exponent = 0, digits;
+
+	while (t < 1.0 && unit < units_len - 1) {
+		t *= 1000.0;
+		unit++;
+	}
+
+	while (t > 0.0 && t < 1.0 && exponent < 10) {
+		t *= 10.0;
+		exponent++;
+	}
+
+	digits = (t < 10.0) ? 3 : ((t < 100.0) ? 2 : 1);
+
+	printf("%.*f", digits, t);
+
+	if (exponent > 0)
+		printf("e-%d", exponent);
+
+	printf(" %s", units[unit]);
+}
+
+static void clar_print_timing_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report)
+{
+	const struct clar_error *error = _clar.last_report->errors;
+
+	(void)suite_name;
+	(void)test_name;
+	(void)test_number;
+
+	switch(report->status) {
+	case CL_TEST_OK:
+		clar_print_timing_time(report->time_mean);
+
+		if (report->runs > 1) {
+			printf(" ± ");
+			clar_print_timing_time(report->time_stddev);
+
+			printf(" / range: ");
+			clar_print_timing_time(report->time_min);
+			printf(" … ");
+			clar_print_timing_time(report->time_max);
+			printf("  (%d runs)", report->runs);
+		}
+
+		printf("\n");
+		break;
+	case CL_TEST_FAILURE:
+		printf("failed: %s\n", error->message);
+		break;
+	case CL_TEST_SKIP:
+	case CL_TEST_NOTRUN:
+		printf("skipped\n");
+		break;
+	}
+
+	fflush(stdout);
+}
+
+static void clar_print_timing_suite_start(const char *suite_name, int suite_index)
+{
+	if (_clar.verbosity == 1)
+		printf("\n%s", suite_name);
+
+	(void)suite_index;
+}
+
+static void clar_print_timing_onabort(const char *fmt, va_list arg)
+{
+	vfprintf(stderr, fmt, arg);
+}
+
+/* indirection between protocol output selection */
+
+#define PRINT(FN, ...) do { \
+		switch (_clar.output_format) { \
+			case CL_OUTPUT_CLAP: \
+				clar_print_clap_##FN (__VA_ARGS__); \
+				break; \
+			case CL_OUTPUT_TAP: \
+				clar_print_tap_##FN (__VA_ARGS__); \
+				break; \
+			case CL_OUTPUT_TIMING: \
+				clar_print_timing_##FN (__VA_ARGS__); \
+				break; \
+			default: \
+				abort(); \
+		} \
+	} while (0)
+
+static void clar_print_init(int test_count, int suite_count)
+{
+	PRINT(init, test_count, suite_count);
+}
+
+static void clar_print_shutdown(int test_count, int suite_count, int error_count)
+{
+	PRINT(shutdown, test_count, suite_count, error_count);
+}
+
+static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error)
+{
+	PRINT(error, num, report, error);
+}
+
+static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number)
+{
+	PRINT(test_start, suite_name, test_name, test_number);
+}
+
+static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report)
+{
+	PRINT(test_finish, suite_name, test_name, test_number, report);
+}
+
+static void clar_print_suite_start(const char *suite_name, int suite_index)
+{
+	PRINT(suite_start, suite_name, suite_index);
+}
+
+static void clar_print_onabortv(const char *msg, va_list argp)
+{
+	PRINT(onabort, msg, argp);
+}
+
+static void clar_print_onabort(const char *msg, ...)
+{
+	va_list argp;
+	va_start(argp, msg);
+	clar_print_onabortv(msg, argp);
+	va_end(argp);
+}

deps/clar/clar/sandbox.h

@@ -164,7 +164,7 @@ static int build_tempdir_path(void)
 
 	if (mkdir(_clar_tempdir, 0700) != 0)
 		return -1;
-#elif defined(__sun) || defined(__TANDEM)
+#elif defined(__sun) || defined(__TANDEM) || defined(__hpux)
 	if (mktemp(_clar_tempdir) == NULL)
 		return -1;
 
@@ -191,7 +191,7 @@ static void clar_tempdir_init(void)
 #if !defined(CLAR_SANDBOX_TEST_NAMES) && defined(_WIN32)
 	srand(clock() ^ (unsigned int)time(NULL) ^ GetCurrentProcessId() ^ GetCurrentThreadId());
 #elif !defined(CLAR_SANDBOX_TEST_NAMES)
-	srand(clock() ^ time(NULL) ^ (getpid() << 16));
+	srand(clock() ^ time(NULL) ^ ((unsigned)getpid() << 16));
 #endif
 }
 

deps/clar/clar/summary.h

@@ -0,0 +1,311 @@
+#include <stdio.h>
+#include <time.h>
+
+static int clar_summary_time_digits(double t)
+{
+	int digits = 3;
+
+	if (t >= 100.0)
+		return 1;
+	else if (t >= 10.0)
+		return 2;
+
+	while (t > 0.0 && t < 1.0 && digits < 10) {
+		t *= 10.0;
+		digits++;
+	}
+
+	return digits;
+}
+
+static int clar_summary_junit_close_tag(
+    struct clar_summary *summary, const char *tag, int indent)
+{
+	const char *indt;
+
+	if (indent == 0) indt = "";
+	else if (indent == 1) indt = "\t";
+	else indt = "\t\t";
+
+	return fprintf(summary->fp, "%s</%s>\n", indt, tag);
+}
+
+static int clar_summary_junit_testsuites(struct clar_summary *summary)
+{
+	return fprintf(summary->fp, "<testsuites>\n");
+}
+
+static int clar_summary_junit_testsuite(struct clar_summary *summary,
+    int idn, const char *name, time_t timestamp,
+    int test_count, int fail_count, int error_count)
+{
+	struct tm tm;
+	char iso_dt[20];
+
+	localtime_r(&timestamp, &tm);
+	if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", &tm) == 0)
+		return -1;
+
+	return fprintf(summary->fp, "\t<testsuite"
+		       " id=\"%d\""
+		       " name=\"%s\""
+		       " hostname=\"localhost\""
+		       " timestamp=\"%s\""
+		       " tests=\"%d\""
+		       " failures=\"%d\""
+		       " errors=\"%d\">\n",
+		       idn, name, iso_dt, test_count, fail_count, error_count);
+}
+
+static int clar_summary_junit_testcase(struct clar_summary *summary,
+    const char *name, const char *classname, double elapsed)
+{
+	return fprintf(summary->fp,
+	    "\t\t<testcase name=\"%s\" classname=\"%s\" time=\"%.*f\">\n",
+		name, classname, clar_summary_time_digits(elapsed), elapsed);
+}
+
+static int clar_summary_junit_failure(struct clar_summary *summary,
+    const char *type, const char *message, const char *desc)
+{
+	return fprintf(summary->fp,
+	    "\t\t\t<failure type=\"%s\"><![CDATA[%s\n%s]]></failure>\n",
+	    type, message, desc);
+}
+
+static int clar_summary_junit_skipped(struct clar_summary *summary)
+{
+	return fprintf(summary->fp, "\t\t\t<skipped />\n");
+}
+
+static struct clar_summary *clar_summary_junit_init(const char *filename)
+{
+	struct clar_summary *summary;
+	FILE *fp;
+
+	if ((fp = fopen(filename, "w")) == NULL) {
+		perror("fopen");
+		return NULL;
+	}
+
+	if ((summary = malloc(sizeof(struct clar_summary))) == NULL) {
+		perror("malloc");
+		fclose(fp);
+		return NULL;
+	}
+
+	summary->filename = filename;
+	summary->fp = fp;
+
+	return summary;
+}
+
+static int clar_summary_junit_shutdown(struct clar_summary *summary)
+{
+	struct clar_report *report;
+	const char *last_suite = NULL;
+
+	if (clar_summary_junit_testsuites(summary) < 0)
+		goto on_error;
+
+	report = _clar.reports;
+	while (report != NULL) {
+		struct clar_error *error = report->errors;
+
+		if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) {
+			if (clar_summary_junit_testsuite(summary, 0, report->suite,
+			    report->start, _clar.tests_ran, _clar.total_errors, 0) < 0)
+				goto on_error;
+		}
+
+		last_suite = report->suite;
+
+		clar_summary_junit_testcase(summary, report->test, report->suite, report->time_total);
+
+		while (error != NULL) {
+			if (clar_summary_junit_failure(summary, "assert",
+			    error->message, error->description) < 0)
+				goto on_error;
+
+			error = error->next;
+		}
+
+		if (report->status == CL_TEST_SKIP)
+			clar_summary_junit_skipped(summary);
+
+		if (clar_summary_junit_close_tag(summary, "testcase", 2) < 0)
+			goto on_error;
+
+		report = report->next;
+
+		if (!report || strcmp(last_suite, report->suite) != 0) {
+			if (clar_summary_junit_close_tag(summary, "testsuite", 1) < 0)
+				goto on_error;
+		}
+	}
+
+	if (clar_summary_junit_close_tag(summary, "testsuites", 0) < 0 ||
+	    fclose(summary->fp) != 0)
+		goto on_error;
+
+	printf("written summary file to %s\n", summary->filename);
+
+	free(summary);
+	return 0;
+
+on_error:
+	fclose(summary->fp);
+	free(summary);
+	return -1;
+}
+
+static struct clar_summary *clar_summary_json_init(const char *filename)
+{
+	struct clar_summary *summary;
+	FILE *fp;
+
+	if ((fp = fopen(filename, "w")) == NULL) {
+		perror("fopen");
+		return NULL;
+	}
+
+	if ((summary = malloc(sizeof(struct clar_summary))) == NULL) {
+		perror("malloc");
+		fclose(fp);
+		return NULL;
+	}
+
+	summary->filename = filename;
+	summary->fp = fp;
+
+	return summary;
+}
+
+static int clar_summary_json_shutdown(struct clar_summary *summary)
+{
+	struct clar_report *report;
+	int i;
+
+	fprintf(summary->fp, "{\n");
+	fprintf(summary->fp, "  \"tests\": [\n");
+
+	report = _clar.reports;
+	while (report != NULL) {
+		struct clar_error *error = report->errors;
+
+		if (report != _clar.reports)
+			fprintf(summary->fp, ",\n");
+
+		fprintf(summary->fp, "    {\n");
+		fprintf(summary->fp, "      \"name\": \"%s::%s\",\n", report->suite, report->test);
+
+		if (report->description)
+			fprintf(summary->fp, "      \"description\": \"%s\",\n", report->description);
+
+		fprintf(summary->fp, "      \"results\": {\n");
+
+		fprintf(summary->fp, "        \"status\": ");
+		if (report->status == CL_TEST_OK)
+			fprintf(summary->fp, "\"ok\",\n");
+		else if (report->status == CL_TEST_FAILURE)
+			fprintf(summary->fp, "\"failed\",\n");
+		else if (report->status == CL_TEST_SKIP)
+			fprintf(summary->fp, "\"skipped\"\n");
+		else
+			clar_abort("unknown test status %d", report->status);
+
+		if (report->status == CL_TEST_OK) {
+			fprintf(summary->fp, "        \"mean\": %.*f,\n",
+				clar_summary_time_digits(report->time_mean), report->time_mean);
+			fprintf(summary->fp, "        \"stddev\": %.*f,\n",
+				clar_summary_time_digits(report->time_stddev), report->time_stddev);
+			fprintf(summary->fp, "        \"min\": %.*f,\n",
+				clar_summary_time_digits(report->time_min), report->time_min);
+			fprintf(summary->fp, "        \"max\": %.*f,\n",
+				clar_summary_time_digits(report->time_max), report->time_max);
+			fprintf(summary->fp, "        \"times\": [\n");
+
+			for (i = 0; i < report->runs; i++) {
+				if (i > 0)
+					fprintf(summary->fp, ",\n");
+
+				fprintf(summary->fp, "          %.*f",
+					clar_summary_time_digits(report->times[i]), report->times[i]);
+			}
+
+			fprintf(summary->fp, "\n        ]\n");
+		}
+
+		if (report->status == CL_TEST_FAILURE) {
+			fprintf(summary->fp, "        \"errors\": [\n");
+
+			while (error != NULL) {
+				if (error != report->errors)
+					fprintf(summary->fp, ",\n");
+
+				fprintf(summary->fp, "          {\n");
+				fprintf(summary->fp, "            \"message\": \"%s\",\n", error->message);
+
+				if (error->description)
+					fprintf(summary->fp, "            \"description\": \"%s\",\n", error->description);
+
+				fprintf(summary->fp, "            \"function\": \"%s\",\n", error->function);
+				fprintf(summary->fp, "            \"file\": \"%s\",\n", error->file);
+				fprintf(summary->fp, "            \"line\": %" PRIuMAX "\n", error->line_number);
+				fprintf(summary->fp, "          }");
+
+				error = error->next;
+			}
+
+			fprintf(summary->fp, "\n");
+			fprintf(summary->fp, "        ]\n");
+		}
+
+		fprintf(summary->fp, "      }\n");
+		fprintf(summary->fp, "    }");
+
+		report = report->next;
+	}
+
+	fprintf(summary->fp, "\n");
+	fprintf(summary->fp, "  ]\n");
+	fprintf(summary->fp, "}\n");
+
+	if (fclose(summary->fp) != 0)
+		goto on_error;
+
+	printf("written summary file to %s\n", summary->filename);
+
+	free(summary);
+	return 0;
+
+on_error:
+	fclose(summary->fp);
+	free(summary);
+	return -1;
+}
+
+/* indirection between protocol output selection */
+
+#define SUMMARY(FN, ...) do { \
+		switch (_clar.summary_format) { \
+			case CL_SUMMARY_JUNIT: \
+				return clar_summary_junit_##FN (__VA_ARGS__); \
+				break; \
+			case CL_SUMMARY_JSON: \
+				return clar_summary_json_##FN (__VA_ARGS__); \
+				break; \
+			default: \
+				abort(); \
+		} \
+	} while(0)
+
+struct clar_summary *clar_summary_init(const char *filename)
+{
+	SUMMARY(init, filename);
+}
+
+int clar_summary_shutdown(struct clar_summary *summary)
+{
+	SUMMARY(shutdown, summary);
+}

deps/clar/clar_libgit2.c

@@ -172,13 +172,42 @@ static git_repository *_cl_repo = NULL;
 
 git_repository *cl_git_sandbox_init(const char *sandbox)
 {
-	/* Get the name of the sandbox folder which will be created */
-	const char *basename = cl_fixture_basename(sandbox);
+	const char *basename;
+	char *ref_format;
 
-	/* Copy the whole sandbox folder from our fixtures to our test sandbox
-	 * area.  After this it can be accessed with `./sandbox`
-	 */
-	cl_fixture_sandbox(sandbox);
+	/* Get the name of the sandbox folder which will be created */
+	basename = cl_fixture_basename(sandbox);
+
+	ref_format = cl_getenv("CLAR_REF_FORMAT");
+	if (!ref_format || !strcmp(ref_format, "files")) {
+		/*
+		 * Copy the whole sandbox folder from our fixtures to our
+		 * test sandbox area. After this it can be accessed with
+		 * `./sandbox`
+		 */
+		cl_fixture_sandbox(sandbox);
+	} else if (!strcmp(ref_format, "reftable")) {
+		struct git_str reftable_sandbox = GIT_STR_INIT;
+
+		cl_git_pass(git_str_joinpath(&reftable_sandbox, "reftable", sandbox));
+		if (!git_fs_path_isdir(cl_fixture(reftable_sandbox.ptr))) {
+			git_str_dispose(&reftable_sandbox);
+			_cl_sandbox = NULL;
+			_cl_repo = NULL;
+			cl_fail("no seed for reftable repository");
+		}
+
+		/*
+		 * Copy over the sandbox and rename it so that the basename
+		 * matches the originally requested basename.
+		 */
+		cl_fixture_sandbox(reftable_sandbox.ptr);
+		git_str_dispose(&reftable_sandbox);
+	} else {
+		cl_fail("Unexpected ref format");
+	}
+
+	git__free(ref_format);
 	_cl_sandbox = sandbox;
 
 	cl_git_pass(p_chdir(basename));
@@ -504,6 +533,26 @@ void cl_repo_set_string(git_repository *repo, const char *cfg, const char *value
 	git_config_free(config);
 }
 
+int cl_repo_has_ref_format(git_repository *repo, const char *format)
+{
+	const char *configured_format;
+	git_config *config;
+	int result;
+
+	cl_git_pass(git_repository_config_snapshot(&config, repo));
+	result = git_config_get_string(&configured_format, config,
+				       "extensions.refStorage");
+	if (result < 0 && result != GIT_ENOTFOUND)
+		cl_fail("cannot read extensions.refStorage");
+	if (result < 0)
+		configured_format = "files";
+
+	result = !strcmp(configured_format, format);
+
+	git_config_free(config);
+	return result;
+}
+
 /* this is essentially the code from git__unescape modified slightly */
 static size_t strip_cr_from_buf(char *start, size_t len)
 {

deps/clar/clar_libgit2.h

@@ -254,6 +254,8 @@ int cl_repo_get_int(git_repository *repo, const char *cfg);
 
 void cl_repo_set_string(git_repository *repo, const char *cfg, const char *value);
 
+int cl_repo_has_ref_format(git_repository *repo, const char *format);
+
 /*
  * set up a fake "home" directory -- automatically configures cleanup
  * function to restore the home directory, although you can call it

deps/clar/generate.py

@@ -17,8 +17,13 @@ class Module(object):
 
         def _render_callback(self, cb):
             if not cb:
-                return '    { NULL, NULL }'
-            return '    { "%s", &%s }' % (cb['short_name'], cb['symbol'])
+                return '    { NULL, NULL, 0, NULL }'
+
+            return '    { "%s", %s, %d, &%s }' % \
+                (cb['short_name'], \
+                 '"' + cb['description'] + '"' if cb['description'] != None else "NULL", \
+                 cb['runs'], \
+                 cb['symbol'])
 
     class DeclarationTemplate(Template):
         def render(self):
@@ -27,6 +32,9 @@ class Module(object):
             for initializer in self.module.initializers:
                 out += "extern %s;\n" % initializer['declaration']
 
+            if self.module.reset:
+                out += "extern %s;\n" % self.module.reset['declaration']
+
             if self.module.cleanup:
                 out += "extern %s;\n" % self.module.cleanup['declaration']
 
@@ -34,7 +42,7 @@ class Module(object):
 
     class CallbacksTemplate(Template):
         def render(self):
-            out = "static const struct clar_func _clar_cb_%s[] = {\n" % self.module.name
+            out = "static const struct %s_func _%s_cb_%s[] = {\n" % (self.module.app_name, self.module.app_name, self.module.name)
             out += ",\n".join(self._render_callback(cb) for cb in self.module.callbacks)
             out += "\n};\n"
             return out
@@ -58,14 +66,16 @@ class Module(object):
     {
         "${clean_name}",
     ${initialize},
+    ${reset},
     ${cleanup},
         ${cb_ptr}, ${cb_count}, ${enabled}
     }"""
                 ).substitute(
                     clean_name = name,
                     initialize = self._render_callback(initializer),
+                    reset = self._render_callback(self.module.reset),
                     cleanup = self._render_callback(self.module.cleanup),
-                    cb_ptr = "_clar_cb_%s" % self.module.name,
+                    cb_ptr = "_%s_cb_%s" % (self.module.app_name, self.module.name),
                     cb_count = len(self.module.callbacks),
                     enabled = int(self.module.enabled)
                 )
@@ -73,10 +83,12 @@ class Module(object):
 
             return ','.join(templates)
 
-    def __init__(self, name):
+    def __init__(self, name, app_name, prefix):
         self.name = name
+        self.app_name = app_name
+        self.prefix = prefix
 
-        self.mtime = 0
+        self.mtime = None
         self.enabled = True
         self.modified = False
 
@@ -85,7 +97,7 @@ class Module(object):
 
     def _skip_comments(self, text):
         SKIP_COMMENTS_REGEX = re.compile(
-            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
+            r'//.*?$|/\*(?!\s*\[clar\]:).*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
             re.DOTALL | re.MULTILINE)
 
         def _replacer(match):
@@ -95,24 +107,63 @@ class Module(object):
         return re.sub(SKIP_COMMENTS_REGEX, _replacer, text)
 
     def parse(self, contents):
-        TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\s*\(\s*void\s*\))\s*\{"
+        TEST_FUNC_REGEX = r"^(void\s+(%s_%s__(\w+))\s*\(\s*void\s*\))(?:\s*/\*\s*\[clar\]:\s*(.*?)\s*\*/)?\s*\{"
 
         contents = self._skip_comments(contents)
-        regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE)
+        regex = re.compile(TEST_FUNC_REGEX % (self.prefix, self.name), re.MULTILINE)
 
         self.callbacks = []
         self.initializers = []
+        self.reset = None
         self.cleanup = None
 
-        for (declaration, symbol, short_name) in regex.findall(contents):
+        for (declaration, symbol, short_name, options) in regex.findall(contents):
+            runs = 0
+            description = None
+
+            while options != '':
+                match = re.search(r'^([a-zA-Z0-9]+)=(\"[^"]*\"|[a-zA-Z0-9_\-]+|\d+)(?:,\s*|\Z)(.*)', options)
+
+                if match == None:
+                    print("Invalid options: '%s' for '%s'" % (options, symbol))
+                    sys.exit(1)
+
+                key = match.group(1)
+                value = match.group(2)
+                options = match.group(3)
+
+                match = re.search(r'^\"(.*)\"$', value)
+                if match != None:
+                    value = match.group(1)
+
+                match = re.search(r'([^a-zA-Z0-9 _\-,\.])', value)
+                if match != None:
+                    print("Invalid character '%s' in %s for '%s'" % (match.group(1), key, symbol))
+                    sys.exit(1)
+
+                if key == "description":
+                    description = value
+                elif key == "runs":
+                    if not value.isnumeric():
+                        print("Invalid option: '%s' in runs for '%s'" % (option, symbol))
+                        sys.exit(1)
+                    runs = int(value)
+                else:
+                    print("Invalid option: '%s' for '%s'" % (key, symbol))
+                    sys.exit(1)
+
             data = {
                 "short_name" : short_name,
                 "declaration" : declaration,
-                "symbol" : symbol
+                "symbol" : symbol,
+                "description" : description,
+                "runs" : runs
             }
 
             if short_name.startswith('initialize'):
                 self.initializers.append(data)
+            elif short_name == 'reset':
+                self.reset = data
             elif short_name == 'cleanup':
                 self.cleanup = data
             else:
@@ -158,22 +209,29 @@ class TestSuite(object):
 
     def find_modules(self):
         modules = []
-        for root, _, files in os.walk(self.path):
-            module_root = root[len(self.path):]
-            module_root = [c for c in module_root.split(os.sep) if c]
 
-            tests_in_module = fnmatch.filter(files, "*.c")
+        if os.path.isfile(self.path):
+            full_path = os.path.abspath(self.path)
+            module_name = os.path.basename(self.path)
+            module_name = os.path.splitext(module_name)[0]
+            modules.append((full_path, module_name))
+        else:
+            for root, _, files in os.walk(self.path):
+                module_root = root[len(self.path):]
+                module_root = [c for c in module_root.split(os.sep) if c]
 
-            for test_file in tests_in_module:
-                full_path = os.path.join(root, test_file)
-                module_name = "_".join(module_root + [test_file[:-2]]).replace("-", "_")
+                tests_in_module = fnmatch.filter(files, "*.c")
 
-                modules.append((full_path, module_name))
+                for test_file in tests_in_module:
+                    full_path = os.path.join(root, test_file)
+                    module_name = "_".join(module_root + [test_file[:-2]]).replace("-", "_")
+
+                    modules.append((full_path, module_name))
 
         return modules
 
-    def load_cache(self):
-        path = os.path.join(self.output, '.clarcache')
+    def load_cache(self, app_name):
+        path = os.path.join(self.output, ".%scache" % app_name)
         cache = {}
 
         try:
@@ -185,18 +243,18 @@ class TestSuite(object):
 
         return cache
 
-    def save_cache(self):
-        path = os.path.join(self.output, '.clarcache')
+    def save_cache(self, app_name):
+        path = os.path.join(self.output, ".%scache" % app_name)
         with open(path, 'wb') as cache:
             pickle.dump(self.modules, cache)
 
-    def load(self, force = False):
+    def load(self, app_name, prefix, force = False):
         module_data = self.find_modules()
-        self.modules = {} if force else self.load_cache()
+        self.modules = {} if force else self.load_cache(app_name)
 
         for path, name in module_data:
             if name not in self.modules:
-                self.modules[name] = Module(name)
+                self.modules[name] = Module(name, app_name, prefix)
 
             if not self.modules[name].refresh(path):
                 del self.modules[name]
@@ -215,12 +273,15 @@ class TestSuite(object):
     def callback_count(self):
         return sum(len(module.callbacks) for module in self.modules.values())
 
-    def write(self):
-        wrote_suite = self.write_suite()
-        wrote_header = self.write_header()
+    def write(self, name):
+        if not os.path.exists(self.output):
+            os.makedirs(self.output)
+
+        wrote_suite = self.write_suite(name)
+        wrote_header = self.write_header(name)
 
         if wrote_suite or wrote_header:
-            self.save_cache()
+            self.save_cache(name)
             return True
 
         return False
@@ -247,8 +308,8 @@ class TestSuite(object):
 
         return True
 
-    def write_suite(self):
-        suite_fn = os.path.join(self.output, 'clar.suite')
+    def write_suite(self, name):
+        suite_fn = os.path.join(self.output, '%s.suite' % name)
 
         with io.StringIO() as suite_file:
             modules = sorted(self.modules.values(), key=lambda module: module.name)
@@ -261,25 +322,26 @@ class TestSuite(object):
                 t = Module.CallbacksTemplate(module)
                 suite_file.write(t.render())
 
-            suites = "static struct clar_suite _clar_suites[] = {" + ','.join(
+            suites = "static struct %s_suite _%s_suites[] = {" % (name, name)
+            suites += ','.join(
                 Module.InfoTemplate(module).render() for module in modules
             ) + "\n};\n"
 
             suite_file.write(suites)
 
-            suite_file.write(u"static const size_t _clar_suite_count = %d;\n" % self.suite_count())
-            suite_file.write(u"static const size_t _clar_callback_count = %d;\n" % self.callback_count())
+            suite_file.write(u"static const size_t _%s_suite_count = %d;\n" % (name, self.suite_count()))
+            suite_file.write(u"static const size_t _%s_callback_count = %d;\n" % (name, self.callback_count()))
 
             return self.write_output(suite_fn, suite_file.getvalue())
 
         return False
 
-    def write_header(self):
-        header_fn = os.path.join(self.output, 'clar_suite.h')
+    def write_header(self, name):
+        header_fn = os.path.join(self.output, '%s_suite.h' % name)
 
         with io.StringIO() as header_file:
-            header_file.write(u"#ifndef _____clar_suite_h_____\n")
-            header_file.write(u"#define _____clar_suite_h_____\n")
+            header_file.write(u"#ifndef _____%s_suite_h_____\n" % name)
+            header_file.write(u"#define _____%s_suite_h_____\n" % name)
 
             modules = sorted(self.modules.values(), key=lambda module: module.name)
 
@@ -300,6 +362,8 @@ if __name__ == '__main__':
     parser.add_option('-f', '--force', action="store_true", dest='force', default=False)
     parser.add_option('-x', '--exclude', dest='excluded', action='append', default=[])
     parser.add_option('-o', '--output', dest='output')
+    parser.add_option('-n', '--name', dest='name', default='clar')
+    parser.add_option('-p', '--prefix', dest='prefix', default='test')
 
     options, args = parser.parse_args()
     if len(args) > 1:
@@ -307,10 +371,14 @@ if __name__ == '__main__':
         sys.exit(1)
 
     path = args.pop() if args else '.'
+    if os.path.isfile(path) and not options.output:
+        print("Must provide --output when specifying a file")
+        sys.exit(1)
     output = options.output or path
-    suite = TestSuite(path, output)
-    suite.load(options.force)
-    suite.disable(options.excluded)
-    if suite.write():
-        print("Written `clar.suite`, `clar_suite.h` (%d tests in %d suites)" % (suite.callback_count(), suite.suite_count()))
 
+    suite = TestSuite(path, output)
+    suite.load(options.name, options.prefix, options.force)
+    suite.disable(options.excluded)
+
+    if suite.write(options.name):
+        print("Written `%s.suite`, `%s_suite.h` (%d tests in %d suites)" % (options.name, options.name, suite.callback_count(), suite.suite_count()))

deps/pcre/CMakeLists.txt

@@ -23,6 +23,7 @@ check_type_size("unsigned long long"    UNSIGNED_LONG_LONG)
 disable_warnings(unused-function)
 disable_warnings(implicit-fallthrough)
 disable_warnings(unused-but-set-variable)
+disable_warnings(dangling-pointer)
 
 # User-configurable options
 
@@ -127,7 +128,7 @@ add_definitions(-DHAVE_CONFIG_H)
 
 if(MSVC)
         add_definitions(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS)
-endif(MSVC)
+endif()
 
 set(CMAKE_INCLUDE_CURRENT_DIR 1)
 
@@ -137,5 +138,3 @@ set(targets)
 # pcre
 include_directories(${PROJECT_BINARY_DIR}/src/pcre)
 add_library(pcre OBJECT ${PCRE_HEADERS} ${PCRE_SOURCES} ${PCREPOSIX_SOURCES})
-
-# end CMakeLists.txt

deps/reftable/CMakeLists.txt

@@ -0,0 +1,29 @@
+add_library(reftable OBJECT ${SRC_REFTABLE})
+
+disable_warnings(implicit-fallthrough)
+
+file(GLOB SRC_REFTABLE "*.c" "*.h")
+list(SORT SRC_REFTABLE)
+
+set_property(TARGET reftable PROPERTY C_STANDARD 99)
+target_sources(reftable PRIVATE ${SRC_REFTABLE})
+target_include_directories(reftable PRIVATE
+	include
+	"${PROJECT_BINARY_DIR}/src/util"
+	"${PROJECT_BINARY_DIR}/include"
+	"${PROJECT_SOURCE_DIR}/src/util"
+	"${PROJECT_SOURCE_DIR}/include"
+	${LIBGIT2_DEPENDENCY_INCLUDES}
+	${LIBGIT2_SYSTEM_INCLUDES}
+)
+
+# The reftable library is not warning-free, so we disable turning warnings into
+# errors.
+if(MSVC)
+        set_source_files_properties(block.c PROPERTIES COMPILE_FLAGS -WX-)
+        set_source_files_properties(blocksource.c PROPERTIES COMPILE_FLAGS -WX-)
+        set_source_files_properties(record.c PROPERTIES COMPILE_FLAGS -WX-)
+        set_source_files_properties(stack.c PROPERTIES COMPILE_FLAGS -WX-)
+        set_source_files_properties(table.c PROPERTIES COMPILE_FLAGS -WX-)
+        set_source_files_properties(writer.c PROPERTIES COMPILE_FLAGS -WX-)
+endif()

deps/reftable/LICENSE

@@ -0,0 +1,31 @@
+BSD License
+
+Copyright (c) 2020, Google LLC
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+* Neither the name of Google LLC nor the names of its contributors may
+be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

deps/reftable/basics.c

@@ -0,0 +1,278 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#define REFTABLE_ALLOW_BANNED_ALLOCATORS
+#include "basics.h"
+#include "reftable-basics.h"
+#include "reftable-error.h"
+
+static void *(*reftable_malloc_ptr)(size_t sz);
+static void *(*reftable_realloc_ptr)(void *, size_t);
+static void (*reftable_free_ptr)(void *);
+
+void *reftable_malloc(size_t sz)
+{
+	if (!sz)
+		return NULL;
+	if (reftable_malloc_ptr)
+		return (*reftable_malloc_ptr)(sz);
+	return malloc(sz);
+}
+
+void *reftable_realloc(void *p, size_t sz)
+{
+	if (!sz) {
+		reftable_free(p);
+		return NULL;
+	}
+
+	if (reftable_realloc_ptr)
+		return (*reftable_realloc_ptr)(p, sz);
+	return realloc(p, sz);
+}
+
+void reftable_free(void *p)
+{
+	if (reftable_free_ptr)
+		reftable_free_ptr(p);
+	else
+		free(p);
+}
+
+void *reftable_calloc(size_t nelem, size_t elsize)
+{
+	void *p;
+
+	if (nelem && elsize > SIZE_MAX / nelem)
+		return NULL;
+
+	p = reftable_malloc(nelem * elsize);
+	if (!p)
+		return NULL;
+
+	memset(p, 0, nelem * elsize);
+	return p;
+}
+
+char *reftable_strdup(const char *str)
+{
+	size_t len = strlen(str);
+	char *result = reftable_malloc(len + 1);
+	if (!result)
+		return NULL;
+	memcpy(result, str, len + 1);
+	return result;
+}
+
+void reftable_set_alloc(void *(*malloc)(size_t),
+			void *(*realloc)(void *, size_t), void (*free)(void *))
+{
+	reftable_malloc_ptr = malloc;
+	reftable_realloc_ptr = realloc;
+	reftable_free_ptr = free;
+}
+
+void reftable_buf_init(struct reftable_buf *buf)
+{
+	struct reftable_buf empty = REFTABLE_BUF_INIT;
+	*buf = empty;
+}
+
+void reftable_buf_release(struct reftable_buf *buf)
+{
+	reftable_free(buf->buf);
+	reftable_buf_init(buf);
+}
+
+void reftable_buf_reset(struct reftable_buf *buf)
+{
+	if (buf->alloc) {
+		buf->len = 0;
+		buf->buf[0] = '\0';
+	}
+}
+
+int reftable_buf_setlen(struct reftable_buf *buf, size_t len)
+{
+	if (len > buf->len)
+		return -1;
+	if (len == buf->len)
+		return 0;
+	buf->buf[len] = '\0';
+	buf->len = len;
+	return 0;
+}
+
+int reftable_buf_cmp(const struct reftable_buf *a, const struct reftable_buf *b)
+{
+	size_t len = a->len < b->len ? a->len : b->len;
+	if (len) {
+		int cmp = memcmp(a->buf, b->buf, len);
+		if (cmp)
+			return cmp;
+	}
+	return a->len < b->len ? -1 : a->len != b->len;
+}
+
+int reftable_buf_add(struct reftable_buf *buf, const void *data, size_t len)
+{
+	size_t newlen = buf->len + len;
+
+	if (newlen + 1 > buf->alloc) {
+		if (REFTABLE_ALLOC_GROW(buf->buf, newlen + 1, buf->alloc))
+			return REFTABLE_OUT_OF_MEMORY_ERROR;
+	}
+
+	memcpy(buf->buf + buf->len, data, len);
+	buf->buf[newlen] = '\0';
+	buf->len = newlen;
+
+	return 0;
+}
+
+int reftable_buf_addstr(struct reftable_buf *buf, const char *s)
+{
+	return reftable_buf_add(buf, s, strlen(s));
+}
+
+char *reftable_buf_detach(struct reftable_buf *buf)
+{
+	char *result = buf->buf;
+	reftable_buf_init(buf);
+	return result;
+}
+
+size_t binsearch(size_t sz, int (*f)(size_t k, void *args), void *args)
+{
+	size_t lo = 0;
+	size_t hi = sz;
+
+	/* Invariants:
+	 *
+	 *  (hi == sz) || f(hi) == true
+	 *  (lo == 0 && f(0) == true) || fi(lo) == false
+	 */
+	while (hi - lo > 1) {
+		size_t mid = lo + (hi - lo) / 2;
+		int ret = f(mid, args);
+		if (ret < 0)
+			return sz;
+
+		if (ret > 0)
+			hi = mid;
+		else
+			lo = mid;
+	}
+
+	if (lo)
+		return hi;
+
+	return f(0, args) ? 0 : 1;
+}
+
+void free_names(char **a)
+{
+	char **p;
+	if (!a) {
+		return;
+	}
+	for (p = a; *p; p++) {
+		reftable_free(*p);
+	}
+	reftable_free(a);
+}
+
+size_t names_length(const char **names)
+{
+	const char **p = names;
+	while (*p)
+		p++;
+	return p - names;
+}
+
+int parse_names(char *buf, int size, char ***out)
+{
+	char **names = NULL;
+	size_t names_cap = 0;
+	size_t names_len = 0;
+	char *p = buf;
+	char *end = buf + size;
+	int err = 0;
+
+	while (p < end) {
+		char *next = strchr(p, '\n');
+		if (!next) {
+			err = REFTABLE_FORMAT_ERROR;
+			goto done;
+		} else if (next < end) {
+			*next = '\0';
+		} else {
+			next = end;
+		}
+
+		if (p < next) {
+			if (REFTABLE_ALLOC_GROW(names, names_len + 1,
+						names_cap)) {
+				err = REFTABLE_OUT_OF_MEMORY_ERROR;
+				goto done;
+			}
+
+			names[names_len] = reftable_strdup(p);
+			if (!names[names_len++]) {
+				err = REFTABLE_OUT_OF_MEMORY_ERROR;
+				goto done;
+			}
+		}
+		p = next + 1;
+	}
+
+	if (REFTABLE_ALLOC_GROW(names, names_len + 1, names_cap)) {
+		err = REFTABLE_OUT_OF_MEMORY_ERROR;
+		goto done;
+	}
+	names[names_len] = NULL;
+
+	*out = names;
+	return 0;
+done:
+	for (size_t i = 0; i < names_len; i++)
+		reftable_free(names[i]);
+	reftable_free(names);
+	return err;
+}
+
+int names_equal(const char **a, const char **b)
+{
+	size_t i = 0;
+	for (; a[i] && b[i]; i++)
+		if (strcmp(a[i], b[i]))
+			return 0;
+	return a[i] == b[i];
+}
+
+size_t common_prefix_size(struct reftable_buf *a, struct reftable_buf *b)
+{
+	size_t p = 0;
+	for (; p < a->len && p < b->len; p++)
+		if (a->buf[p] != b->buf[p])
+			break;
+	return p;
+}
+
+uint32_t hash_size(enum reftable_hash id)
+{
+	if (!id)
+		return REFTABLE_HASH_SIZE_SHA1;
+	switch (id) {
+	case REFTABLE_HASH_SHA1:
+		return REFTABLE_HASH_SIZE_SHA1;
+	case REFTABLE_HASH_SHA256:
+		return REFTABLE_HASH_SIZE_SHA256;
+	}
+	abort();
+}

deps/reftable/basics.h

@@ -0,0 +1,291 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef BASICS_H
+#define BASICS_H
+
+/*
+ * miscellaneous utilities that are not provided by Git.
+ */
+
+#include "system.h"
+#include "reftable-basics.h"
+
+#ifdef __GNUC__
+#define REFTABLE_UNUSED __attribute__((__unused__))
+#else
+#define REFTABLE_UNUSED
+#endif
+
+/*
+ * Initialize the buffer such that it is ready for use. This is equivalent to
+ * using REFTABLE_BUF_INIT for stack-allocated variables.
+ */
+void reftable_buf_init(struct reftable_buf *buf);
+
+/*
+ * Release memory associated with the buffer. The buffer is reinitialized such
+ * that it can be reused for subsequent operations.
+ */
+void reftable_buf_release(struct reftable_buf *buf);
+
+/*
+ * Reset the buffer such that it is effectively empty, without releasing the
+ * memory that this structure holds on to. This is equivalent to calling
+ * `reftable_buf_setlen(buf, 0)`.
+ */
+void reftable_buf_reset(struct reftable_buf *buf);
+
+/*
+ * Trim the buffer to a shorter length by updating the `len` member and writing
+ * a NUL byte to `buf[len]`. Returns 0 on success, -1 when `len` points outside
+ * of the array.
+ */
+int reftable_buf_setlen(struct reftable_buf *buf, size_t len);
+
+/*
+ * Lexicographically compare the two buffers. Returns 0 when both buffers have
+ * the same contents, -1 when `a` is lexicographically smaller than `b`, and 1
+ * otherwise.
+ */
+int reftable_buf_cmp(const struct reftable_buf *a, const struct reftable_buf *b);
+
+/*
+ * Append `len` bytes from `data` to the buffer. This function works with
+ * arbitrary byte sequences, including ones that contain embedded NUL
+ * characters. As such, we use `void *` as input type. Returns 0 on success,
+ * REFTABLE_OUT_OF_MEMORY_ERROR on allocation failure.
+ */
+int reftable_buf_add(struct reftable_buf *buf, const void *data, size_t len);
+
+/* Equivalent to `reftable_buf_add(buf, s, strlen(s))`. */
+int reftable_buf_addstr(struct reftable_buf *buf, const char *s);
+
+/*
+ * Detach the buffer from the structure such that the underlying memory is now
+ * owned by the caller. The buffer is reinitialized such that it can be reused
+ * for subsequent operations.
+ */
+char *reftable_buf_detach(struct reftable_buf *buf);
+
+/* Bigendian en/decoding of integers */
+
+static inline void reftable_put_be16(void *out, uint16_t i)
+{
+	unsigned char *p = out;
+	p[0] = (uint8_t)((i >> 8) & 0xff);
+	p[1] = (uint8_t)((i >> 0) & 0xff);
+}
+
+static inline void reftable_put_be24(void *out, uint32_t i)
+{
+	unsigned char *p = out;
+	p[0] = (uint8_t)((i >> 16) & 0xff);
+	p[1] = (uint8_t)((i >>  8) & 0xff);
+	p[2] = (uint8_t)((i >>  0) & 0xff);
+}
+
+static inline void reftable_put_be32(void *out, uint32_t i)
+{
+	unsigned char *p = out;
+	p[0] = (uint8_t)((i >> 24) & 0xff);
+	p[1] = (uint8_t)((i >> 16) & 0xff);
+	p[2] = (uint8_t)((i >>  8) & 0xff);
+	p[3] = (uint8_t)((i >>  0) & 0xff);
+}
+
+static inline void reftable_put_be64(void *out, uint64_t i)
+{
+	unsigned char *p = out;
+	p[0] = (uint8_t)((i >> 56) & 0xff);
+	p[1] = (uint8_t)((i >> 48) & 0xff);
+	p[2] = (uint8_t)((i >> 40) & 0xff);
+	p[3] = (uint8_t)((i >> 32) & 0xff);
+	p[4] = (uint8_t)((i >> 24) & 0xff);
+	p[5] = (uint8_t)((i >> 16) & 0xff);
+	p[6] = (uint8_t)((i >>  8) & 0xff);
+	p[7] = (uint8_t)((i >>  0) & 0xff);
+}
+
+static inline uint16_t reftable_get_be16(const void *in)
+{
+	const unsigned char *p = in;
+	return (uint16_t)(p[0]) << 8 |
+	       (uint16_t)(p[1]) << 0;
+}
+
+static inline uint32_t reftable_get_be24(const void *in)
+{
+	const unsigned char *p = in;
+	return (uint32_t)(p[0]) << 16 |
+	       (uint32_t)(p[1]) << 8 |
+	       (uint32_t)(p[2]) << 0;
+}
+
+static inline uint32_t reftable_get_be32(const void *in)
+{
+	const unsigned char *p = in;
+	return (uint32_t)(p[0]) << 24 |
+	       (uint32_t)(p[1]) << 16 |
+	       (uint32_t)(p[2]) <<  8|
+	       (uint32_t)(p[3]) <<  0;
+}
+
+static inline uint64_t reftable_get_be64(const void *in)
+{
+	const unsigned char *p = in;
+	return (uint64_t)(p[0]) << 56 |
+	       (uint64_t)(p[1]) << 48 |
+	       (uint64_t)(p[2]) << 40 |
+	       (uint64_t)(p[3]) << 32 |
+	       (uint64_t)(p[4]) << 24 |
+	       (uint64_t)(p[5]) << 16 |
+	       (uint64_t)(p[6]) <<  8 |
+	       (uint64_t)(p[7]) <<  0;
+}
+
+/*
+ * find smallest index i in [0, sz) at which `f(i) > 0`, assuming that f is
+ * ascending. Return sz if `f(i) == 0` for all indices. The search is aborted
+ * and `sz` is returned in case `f(i) < 0`.
+ *
+ * Contrary to bsearch(3), this returns something useful if the argument is not
+ * found.
+ */
+size_t binsearch(size_t sz, int (*f)(size_t k, void *args), void *args);
+
+/*
+ * Frees a NULL terminated array of malloced strings. The array itself is also
+ * freed.
+ */
+void free_names(char **a);
+
+/*
+ * Parse a newline separated list of names. `size` is the length of the buffer,
+ * without terminating '\0'. Empty names are discarded.
+ *
+ * Returns 0 on success, a reftable error code on error.
+ */
+int parse_names(char *buf, int size, char ***out);
+
+/* compares two NULL-terminated arrays of strings. */
+int names_equal(const char **a, const char **b);
+
+/* returns the array size of a NULL-terminated array of strings. */
+size_t names_length(const char **names);
+
+/* Allocation routines; they invoke the functions set through
+ * reftable_set_alloc() */
+void *reftable_malloc(size_t sz);
+void *reftable_realloc(void *p, size_t sz);
+void reftable_free(void *p);
+void *reftable_calloc(size_t nelem, size_t elsize);
+char *reftable_strdup(const char *str);
+
+static inline int reftable_alloc_size(size_t nelem, size_t elsize, size_t *out)
+{
+	if (nelem && elsize > SIZE_MAX / nelem)
+		return -1;
+	*out = nelem * elsize;
+	return 0;
+}
+
+#define REFTABLE_ALLOC_ARRAY(x, alloc) do { \
+		size_t alloc_size; \
+		if (reftable_alloc_size(sizeof(*(x)), (alloc), &alloc_size) < 0) { \
+			errno = ENOMEM; \
+			(x) = NULL; \
+		} else { \
+			(x) = reftable_malloc(alloc_size); \
+		} \
+	} while (0)
+#define REFTABLE_CALLOC_ARRAY(x, alloc) (x) = reftable_calloc((alloc), sizeof(*(x)))
+#define REFTABLE_REALLOC_ARRAY(x, alloc) do { \
+		size_t alloc_size; \
+		if (reftable_alloc_size(sizeof(*(x)), (alloc), &alloc_size) < 0) { \
+			errno = ENOMEM; \
+			(x) = NULL; \
+		} else { \
+			(x) = reftable_realloc((x), alloc_size); \
+		} \
+	} while (0)
+
+static inline void *reftable_alloc_grow(void *p, size_t nelem, size_t elsize,
+					size_t *allocp)
+{
+	void *new_p;
+	size_t alloc = *allocp * 2 + 1, alloc_bytes;
+	if (alloc < nelem)
+		alloc = nelem;
+	if (reftable_alloc_size(elsize, alloc, &alloc_bytes) < 0) {
+		errno = ENOMEM;
+		return p;
+	}
+	new_p = reftable_realloc(p, alloc_bytes);
+	if (!new_p)
+		return p;
+	*allocp = alloc;
+	return new_p;
+}
+
+#define REFTABLE_ALLOC_GROW(x, nr, alloc) ( \
+	(nr) > (alloc) && ( \
+		(x) = reftable_alloc_grow((x), (nr), sizeof(*(x)), &(alloc)), \
+		(nr) > (alloc) \
+	) \
+)
+
+#define REFTABLE_ALLOC_GROW_OR_NULL(x, nr, alloc) do { \
+	size_t reftable_alloc_grow_or_null_alloc = alloc; \
+	if (REFTABLE_ALLOC_GROW((x), (nr), reftable_alloc_grow_or_null_alloc)) { \
+		REFTABLE_FREE_AND_NULL(x); \
+		alloc = 0; \
+	} else { \
+		alloc = reftable_alloc_grow_or_null_alloc; \
+	} \
+} while (0)
+
+#define REFTABLE_FREE_AND_NULL(p) do { reftable_free(p); (p) = NULL; } while (0)
+
+#ifndef REFTABLE_ALLOW_BANNED_ALLOCATORS
+# define REFTABLE_BANNED(func) use_reftable_##func##_instead
+# undef malloc
+# define malloc(sz) REFTABLE_BANNED(malloc)
+# undef realloc
+# define realloc(ptr, sz) REFTABLE_BANNED(realloc)
+# undef free
+# define free(ptr) REFTABLE_BANNED(free)
+# undef calloc
+# define calloc(nelem, elsize) REFTABLE_BANNED(calloc)
+# undef strdup
+# define strdup(str) REFTABLE_BANNED(strdup)
+#endif
+
+#define REFTABLE_SWAP(a, b) do {								\
+	void *_swap_a_ptr = &(a);								\
+	void *_swap_b_ptr = &(b);								\
+	unsigned char _swap_buffer[sizeof(a) - 2 * sizeof(a) * (sizeof(a) != sizeof(b))];	\
+	memcpy(_swap_buffer, _swap_a_ptr, sizeof(a));						\
+	memcpy(_swap_a_ptr, _swap_b_ptr, sizeof(a));						\
+	memcpy(_swap_b_ptr, _swap_buffer, sizeof(a));						\
+} while (0)
+
+/* Find the longest shared prefix size of `a` and `b` */
+size_t common_prefix_size(struct reftable_buf *a, struct reftable_buf *b);
+
+uint32_t hash_size(enum reftable_hash id);
+
+/*
+ * Format IDs that identify the hash function used by a reftable. Note that
+ * these constants end up on disk and thus mustn't change. The format IDs are
+ * "sha1" and "s256" in big endian, respectively.
+ */
+#define REFTABLE_FORMAT_ID_SHA1   ((uint32_t) 0x73686131)
+#define REFTABLE_FORMAT_ID_SHA256 ((uint32_t) 0x73323536)
+
+#endif

deps/reftable/block.c

@@ -0,0 +1,655 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#include "block.h"
+
+#include "blocksource.h"
+#include "constants.h"
+#include "iter.h"
+#include "record.h"
+#include "reftable-error.h"
+#include "system.h"
+
+size_t header_size(int version)
+{
+	switch (version) {
+	case 1:
+		return 24;
+	case 2:
+		return 28;
+	}
+	abort();
+}
+
+size_t footer_size(int version)
+{
+	switch (version) {
+	case 1:
+		return 68;
+	case 2:
+		return 72;
+	}
+	abort();
+}
+
+static int block_writer_register_restart(struct block_writer *w, int n,
+					 int is_restart, struct reftable_buf *key)
+{
+	uint32_t rlen;
+	int err;
+
+	rlen = w->restart_len;
+	if (rlen >= MAX_RESTARTS)
+		is_restart = 0;
+
+	if (is_restart)
+		rlen++;
+	if (2 + 3 * rlen + n > w->block_size - w->next)
+		return REFTABLE_ENTRY_TOO_BIG_ERROR;
+	if (is_restart) {
+		REFTABLE_ALLOC_GROW_OR_NULL(w->restarts, w->restart_len + 1,
+					    w->restart_cap);
+		if (!w->restarts)
+			return REFTABLE_OUT_OF_MEMORY_ERROR;
+		w->restarts[w->restart_len++] = w->next;
+	}
+
+	w->next += n;
+
+	reftable_buf_reset(&w->last_key);
+	err = reftable_buf_add(&w->last_key, key->buf, key->len);
+	if (err < 0)
+		return err;
+
+	w->entries++;
+	return 0;
+}
+
+int block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *block,
+		      uint32_t block_size, uint32_t header_off, uint32_t hash_size)
+{
+	bw->block = block;
+	bw->hash_size = hash_size;
+	bw->block_size = block_size;
+	bw->header_off = header_off;
+	bw->block[header_off] = typ;
+	bw->next = header_off + 4;
+	bw->restart_interval = 16;
+	bw->entries = 0;
+	bw->restart_len = 0;
+	bw->last_key.len = 0;
+	if (!bw->zstream) {
+		REFTABLE_CALLOC_ARRAY(bw->zstream, 1);
+		if (!bw->zstream)
+			return REFTABLE_OUT_OF_MEMORY_ERROR;
+		deflateInit(bw->zstream, 9);
+	}
+
+	return 0;
+}
+
+uint8_t block_writer_type(struct block_writer *bw)
+{
+	return bw->block[bw->header_off];
+}
+
+/*
+ * Adds the reftable_record to the block. Returns 0 on success and
+ * appropriate error codes on failure.
+ */
+int block_writer_add(struct block_writer *w, struct reftable_record *rec)
+{
+	struct reftable_buf empty = REFTABLE_BUF_INIT;
+	struct reftable_buf last =
+		w->entries % w->restart_interval == 0 ? empty : w->last_key;
+	struct string_view out = {
+		.buf = w->block + w->next,
+		.len = w->block_size - w->next,
+	};
+	struct string_view start = out;
+	int is_restart = 0;
+	int n = 0;
+	int err;
+
+	err = reftable_record_key(rec, &w->scratch);
+	if (err < 0)
+		goto done;
+
+	if (!w->scratch.len) {
+		err = REFTABLE_API_ERROR;
+		goto done;
+	}
+
+	n = reftable_encode_key(&is_restart, out, last, w->scratch,
+				reftable_record_val_type(rec));
+	if (n < 0) {
+		err = n;
+		goto done;
+	}
+	string_view_consume(&out, n);
+
+	n = reftable_record_encode(rec, out, w->hash_size);
+	if (n < 0) {
+		err = n;
+		goto done;
+	}
+	string_view_consume(&out, n);
+
+	err = block_writer_register_restart(w, start.len - out.len, is_restart,
+					    &w->scratch);
+done:
+	return err;
+}
+
+int block_writer_finish(struct block_writer *w)
+{
+	for (uint32_t i = 0; i < w->restart_len; i++) {
+		reftable_put_be24(w->block + w->next, w->restarts[i]);
+		w->next += 3;
+	}
+
+	reftable_put_be16(w->block + w->next, w->restart_len);
+	w->next += 2;
+	reftable_put_be24(w->block + 1 + w->header_off, w->next);
+
+	/*
+	 * Log records are stored zlib-compressed. Note that the compression
+	 * also spans over the restart points we have just written.
+	 */
+	if (block_writer_type(w) == REFTABLE_BLOCK_TYPE_LOG) {
+		int block_header_skip = 4 + w->header_off;
+		uLongf src_len = w->next - block_header_skip, compressed_len;
+		int ret;
+
+		ret = deflateReset(w->zstream);
+		if (ret != Z_OK)
+			return REFTABLE_ZLIB_ERROR;
+
+		/*
+		 * Precompute the upper bound of how many bytes the compressed
+		 * data may end up with. Combined with `Z_FINISH`, `deflate()`
+		 * is guaranteed to return `Z_STREAM_END`.
+		 */
+		compressed_len = deflateBound(w->zstream, src_len);
+		REFTABLE_ALLOC_GROW_OR_NULL(w->compressed, compressed_len,
+					    w->compressed_cap);
+		if (!w->compressed) {
+			ret = REFTABLE_OUT_OF_MEMORY_ERROR;
+			return ret;
+		}
+
+		w->zstream->next_out = w->compressed;
+		w->zstream->avail_out = compressed_len;
+		w->zstream->next_in = w->block + block_header_skip;
+		w->zstream->avail_in = src_len;
+
+		/*
+		 * We want to perform all decompression in a single step, which
+		 * is why we can pass Z_FINISH here. As we have precomputed the
+		 * deflated buffer's size via `deflateBound()` this function is
+		 * guaranteed to succeed according to the zlib documentation.
+		 */
+		ret = deflate(w->zstream, Z_FINISH);
+		if (ret != Z_STREAM_END)
+			return REFTABLE_ZLIB_ERROR;
+
+		/*
+		 * Overwrite the uncompressed data we have already written and
+		 * adjust the `next` pointer to point right after the
+		 * compressed data.
+		 */
+		memcpy(w->block + block_header_skip, w->compressed,
+		       w->zstream->total_out);
+		w->next = w->zstream->total_out + block_header_skip;
+	}
+
+	return w->next;
+}
+
+static int read_block(struct reftable_block_source *source,
+		      struct reftable_block_data *dest, uint64_t off,
+		      uint32_t sz)
+{
+	size_t size = block_source_size(source);
+	block_source_release_data(dest);
+	if (off >= size)
+		return 0;
+	if (off + sz > size)
+		sz = size - off;
+	return block_source_read_data(source, dest, off, sz);
+}
+
+int reftable_block_init(struct reftable_block *block,
+			struct reftable_block_source *source,
+			uint32_t offset, uint32_t header_size,
+			uint32_t table_block_size, uint32_t hash_size,
+			uint8_t want_type)
+{
+	uint32_t guess_block_size = table_block_size ?
+		table_block_size : DEFAULT_BLOCK_SIZE;
+	uint32_t full_block_size = table_block_size;
+	uint16_t restart_count;
+	uint32_t restart_off;
+	uint32_t block_size;
+	uint8_t block_type;
+	int err;
+
+	err = read_block(source, &block->block_data, offset, guess_block_size);
+	if (err < 0)
+		goto done;
+
+	block_type = block->block_data.data[header_size];
+	if (!reftable_is_block_type(block_type)) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+	if (want_type != REFTABLE_BLOCK_TYPE_ANY && block_type != want_type) {
+		err = 1;
+		goto done;
+	}
+
+	block_size = reftable_get_be24(block->block_data.data + header_size + 1);
+	if (block_size > guess_block_size) {
+		err = read_block(source, &block->block_data, offset, block_size);
+		if (err < 0)
+			goto done;
+	}
+
+	if (block_type == REFTABLE_BLOCK_TYPE_LOG) {
+		uint32_t block_header_skip = 4 + header_size;
+		uLong dst_len = block_size - block_header_skip;
+		uLong src_len = block->block_data.len - block_header_skip;
+
+		/* Log blocks specify the *uncompressed* size in their header. */
+		REFTABLE_ALLOC_GROW_OR_NULL(block->uncompressed_data, block_size,
+					    block->uncompressed_cap);
+		if (!block->uncompressed_data) {
+			err = REFTABLE_OUT_OF_MEMORY_ERROR;
+			goto done;
+		}
+
+		/* Copy over the block header verbatim. It's not compressed. */
+		memcpy(block->uncompressed_data, block->block_data.data, block_header_skip);
+
+		if (!block->zstream) {
+			REFTABLE_CALLOC_ARRAY(block->zstream, 1);
+			if (!block->zstream) {
+				err = REFTABLE_OUT_OF_MEMORY_ERROR;
+				goto done;
+			}
+
+			err = inflateInit(block->zstream);
+		} else {
+			err = inflateReset(block->zstream);
+		}
+		if (err != Z_OK) {
+			err = REFTABLE_ZLIB_ERROR;
+			goto done;
+		}
+
+		block->zstream->next_in = block->block_data.data + block_header_skip;
+		block->zstream->avail_in = src_len;
+		block->zstream->next_out = block->uncompressed_data + block_header_skip;
+		block->zstream->avail_out = dst_len;
+
+		/*
+		 * We know both input as well as output size, and we know that
+		 * the sizes should never be bigger than `uInt_MAX` because
+		 * blocks can at most be 16MB large. We can thus use `Z_FINISH`
+		 * here to instruct zlib to inflate the data in one go, which
+		 * is more efficient than using `Z_NO_FLUSH`.
+		 */
+		err = inflate(block->zstream, Z_FINISH);
+		if (err != Z_STREAM_END) {
+			err = REFTABLE_ZLIB_ERROR;
+			goto done;
+		}
+		err = 0;
+
+		if (block->zstream->total_out + block_header_skip != block_size) {
+			err = REFTABLE_FORMAT_ERROR;
+			goto done;
+		}
+
+		/* We're done with the input data. */
+		block_source_release_data(&block->block_data);
+		block->block_data.data = block->uncompressed_data;
+		block->block_data.len = block_size;
+		full_block_size = src_len + block_header_skip - block->zstream->avail_in;
+	} else if (full_block_size == 0) {
+		full_block_size = block_size;
+	} else if (block_size < full_block_size && block_size < block->block_data.len &&
+		   block->block_data.data[block_size] != 0) {
+		/* If the block is smaller than the full block size, it is
+		   padded (data followed by '\0') or the next block is
+		   unaligned. */
+		full_block_size = block_size;
+	}
+
+	restart_count = reftable_get_be16(block->block_data.data + block_size - 2);
+	restart_off = block_size - 2 - 3 * restart_count;
+
+	block->block_type = block_type;
+	block->hash_size = hash_size;
+	block->restart_off = restart_off;
+	block->full_block_size = full_block_size;
+	block->header_off = header_size;
+	block->restart_count = restart_count;
+
+	err = 0;
+
+done:
+	if (err < 0)
+		reftable_block_release(block);
+	return err;
+}
+
+void reftable_block_release(struct reftable_block *block)
+{
+	inflateEnd(block->zstream);
+	reftable_free(block->zstream);
+	reftable_free(block->uncompressed_data);
+	block_source_release_data(&block->block_data);
+	memset(block, 0, sizeof(*block));
+}
+
+uint8_t reftable_block_type(const struct reftable_block *b)
+{
+	return b->block_data.data[b->header_off];
+}
+
+int reftable_block_first_key(const struct reftable_block *block, struct reftable_buf *key)
+{
+	int off = block->header_off + 4, n;
+	struct string_view in = {
+		.buf = block->block_data.data + off,
+		.len = block->restart_off - off,
+	};
+	uint8_t extra = 0;
+
+	reftable_buf_reset(key);
+
+	n = reftable_decode_key(key, &extra, in);
+	if (n < 0)
+		return n;
+	if (!key->len)
+		return REFTABLE_FORMAT_ERROR;
+
+	return 0;
+}
+
+static uint32_t block_restart_offset(const struct reftable_block *b, size_t idx)
+{
+	return reftable_get_be24(b->block_data.data + b->restart_off + 3 * idx);
+}
+
+void block_iter_init(struct block_iter *it, const struct reftable_block *block)
+{
+	it->block = block;
+	block_iter_seek_start(it);
+}
+
+void block_iter_seek_start(struct block_iter *it)
+{
+	reftable_buf_reset(&it->last_key);
+	it->next_off = it->block->header_off + 4;
+}
+
+struct restart_needle_less_args {
+	int error;
+	struct reftable_buf needle;
+	const struct reftable_block *block;
+};
+
+static int restart_needle_less(size_t idx, void *_args)
+{
+	struct restart_needle_less_args *args = _args;
+	uint32_t off = block_restart_offset(args->block, idx);
+	struct string_view in = {
+		.buf = args->block->block_data.data + off,
+		.len = args->block->restart_off - off,
+	};
+	uint64_t prefix_len, suffix_len;
+	uint8_t extra;
+	int n;
+
+	/*
+	 * Records at restart points are stored without prefix compression, so
+	 * there is no need to fully decode the record key here. This removes
+	 * the need for allocating memory.
+	 */
+	n = reftable_decode_keylen(in, &prefix_len, &suffix_len, &extra);
+	if (n < 0 || prefix_len) {
+		args->error = 1;
+		return -1;
+	}
+
+	string_view_consume(&in, n);
+	if (suffix_len > in.len) {
+		args->error = 1;
+		return -1;
+	}
+
+	n = memcmp(args->needle.buf, in.buf,
+		   args->needle.len < suffix_len ? args->needle.len : suffix_len);
+	if (n)
+		return n < 0;
+	return args->needle.len < suffix_len;
+}
+
+int block_iter_next(struct block_iter *it, struct reftable_record *rec)
+{
+	struct string_view in = {
+		.buf = (unsigned char *) it->block->block_data.data + it->next_off,
+		.len = it->block->restart_off - it->next_off,
+	};
+	struct string_view start = in;
+	uint8_t extra = 0;
+	int n = 0;
+
+	if (it->next_off >= it->block->restart_off)
+		return 1;
+
+	n = reftable_decode_key(&it->last_key, &extra, in);
+	if (n < 0)
+		return -1;
+	if (!it->last_key.len)
+		return REFTABLE_FORMAT_ERROR;
+
+	string_view_consume(&in, n);
+	n = reftable_record_decode(rec, it->last_key, extra, in, it->block->hash_size,
+				   &it->scratch);
+	if (n < 0)
+		return -1;
+	string_view_consume(&in, n);
+
+	it->next_off += start.len - in.len;
+	return 0;
+}
+
+void block_iter_reset(struct block_iter *it)
+{
+	reftable_buf_reset(&it->last_key);
+	it->next_off = 0;
+	it->block = NULL;
+}
+
+void block_iter_close(struct block_iter *it)
+{
+	reftable_buf_release(&it->last_key);
+	reftable_buf_release(&it->scratch);
+}
+
+int block_iter_seek_key(struct block_iter *it, struct reftable_buf *want)
+{
+	struct restart_needle_less_args args = {
+		.needle = *want,
+		.block = it->block,
+	};
+	struct reftable_record rec;
+	int err = 0;
+	size_t i;
+
+	/*
+	 * Perform a binary search over the block's restart points, which
+	 * avoids doing a linear scan over the whole block. Like this, we
+	 * identify the section of the block that should contain our key.
+	 *
+	 * Note that we explicitly search for the first restart point _greater_
+	 * than the sought-after record, not _greater or equal_ to it. In case
+	 * the sought-after record is located directly at the restart point we
+	 * would otherwise start doing the linear search at the preceding
+	 * restart point. While that works alright, we would end up scanning
+	 * too many record.
+	 */
+	i = binsearch(it->block->restart_count, &restart_needle_less, &args);
+	if (args.error) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+
+	/*
+	 * Now there are multiple cases:
+	 *
+	 *   - `i == 0`: The wanted record is smaller than the record found at
+	 *     the first restart point. As the first restart point is the first
+	 *     record in the block, our wanted record cannot be located in this
+	 *     block at all. We still need to position the iterator so that the
+	 *     next call to `block_iter_next()` will yield an end-of-iterator
+	 *     signal.
+	 *
+	 *   - `i == restart_count`: The wanted record was not found at any of
+	 *     the restart points. As there is no restart point at the end of
+	 *     the section the record may thus be contained in the last block.
+	 *
+	 *   - `i > 0`: The wanted record must be contained in the section
+	 *     before the found restart point. We thus do a linear search
+	 *     starting from the preceding restart point.
+	 */
+	if (i > 0)
+		it->next_off = block_restart_offset(it->block, i - 1);
+	else
+		it->next_off = it->block->header_off + 4;
+
+	err = reftable_record_init(&rec, reftable_block_type(it->block));
+	if (err < 0)
+		goto done;
+
+	/*
+	 * We're looking for the last entry less than the wanted key so that
+	 * the next call to `block_reader_next()` would yield the wanted
+	 * record. We thus don't want to position our iterator at the sought
+	 * after record, but one before. To do so, we have to go one entry too
+	 * far and then back up.
+	 */
+	while (1) {
+		size_t prev_off = it->next_off;
+
+		err = block_iter_next(it, &rec);
+		if (err < 0)
+			goto done;
+		if (err > 0) {
+			it->next_off = prev_off;
+			err = 0;
+			goto done;
+		}
+
+		err = reftable_record_key(&rec, &it->last_key);
+		if (err < 0)
+			goto done;
+
+		/*
+		 * Check whether the current key is greater or equal to the
+		 * sought-after key. In case it is greater we know that the
+		 * record does not exist in the block and can thus abort early.
+		 * In case it is equal to the sought-after key we have found
+		 * the desired record.
+		 *
+		 * Note that we store the next record's key record directly in
+		 * `last_key` without restoring the key of the preceding record
+		 * in case we need to go one record back. This is safe to do as
+		 * `block_iter_next()` would return the ref whose key is equal
+		 * to `last_key` now, and naturally all keys share a prefix
+		 * with themselves.
+		 */
+		if (reftable_buf_cmp(&it->last_key, want) >= 0) {
+			it->next_off = prev_off;
+			goto done;
+		}
+	}
+
+done:
+	reftable_record_release(&rec);
+	return err;
+}
+
+static int block_iter_seek_void(void *it, struct reftable_record *want)
+{
+	struct reftable_buf buf = REFTABLE_BUF_INIT;
+	struct block_iter *bi = it;
+	int err;
+
+	if (bi->block->block_type != want->type)
+		return REFTABLE_API_ERROR;
+
+	err = reftable_record_key(want, &buf);
+	if (err < 0)
+		goto out;
+
+	err = block_iter_seek_key(it, &buf);
+	if (err < 0)
+		goto out;
+
+	err = 0;
+
+out:
+	reftable_buf_release(&buf);
+	return err;
+}
+
+static int block_iter_next_void(void *it, struct reftable_record *rec)
+{
+	return block_iter_next(it, rec);
+}
+
+static void block_iter_close_void(void *it)
+{
+	block_iter_close(it);
+}
+
+static struct reftable_iterator_vtable block_iter_vtable = {
+	.seek = &block_iter_seek_void,
+	.next = &block_iter_next_void,
+	.close = &block_iter_close_void,
+};
+
+int reftable_block_init_iterator(const struct reftable_block *b,
+				 struct reftable_iterator *it)
+{
+	struct block_iter *bi;
+
+	REFTABLE_CALLOC_ARRAY(bi, 1);
+	block_iter_init(bi, b);
+
+	assert(!it->ops);
+	it->iter_arg = bi;
+	it->ops = &block_iter_vtable;
+
+	return 0;
+}
+
+void block_writer_release(struct block_writer *bw)
+{
+	deflateEnd(bw->zstream);
+	REFTABLE_FREE_AND_NULL(bw->zstream);
+	REFTABLE_FREE_AND_NULL(bw->restarts);
+	REFTABLE_FREE_AND_NULL(bw->compressed);
+	reftable_buf_release(&bw->scratch);
+	reftable_buf_release(&bw->last_key);
+	/* the block is not owned. */
+}

deps/reftable/block.h

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef BLOCK_H
+#define BLOCK_H
+
+#include "basics.h"
+#include "record.h"
+#include "reftable-block.h"
+#include "reftable-blocksource.h"
+
+/*
+ * Writes reftable blocks. The block_writer is reused across blocks to minimize
+ * allocation overhead.
+ */
+struct block_writer {
+	struct z_stream_s *zstream;
+	unsigned char *compressed;
+	size_t compressed_cap;
+
+	uint8_t *block;
+	uint32_t block_size;
+
+	/* Offset of the global header. Nonzero in the first block only. */
+	uint32_t header_off;
+
+	/* How often to restart keys. */
+	uint16_t restart_interval;
+	uint32_t hash_size;
+
+	/* Offset of next uint8_t to write. */
+	uint32_t next;
+	uint32_t *restarts;
+	uint32_t restart_len;
+	uint32_t restart_cap;
+
+	struct reftable_buf last_key;
+	/* Scratch buffer used to avoid allocations. */
+	struct reftable_buf scratch;
+	int entries;
+};
+
+/*
+ * initializes the blockwriter to write `typ` entries, using `block` as temporary
+ * storage. `block` is not owned by the block_writer. */
+int block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *block,
+		      uint32_t block_size, uint32_t header_off, uint32_t hash_size);
+
+/* returns the block type (eg. 'r' for ref records. */
+uint8_t block_writer_type(struct block_writer *bw);
+
+/* Attempts to append the record. Returns 0 on success or error code on failure. */
+int block_writer_add(struct block_writer *w, struct reftable_record *rec);
+
+/* appends the key restarts, and compress the block if necessary. */
+int block_writer_finish(struct block_writer *w);
+
+/* clears out internally allocated block_writer members. */
+void block_writer_release(struct block_writer *bw);
+
+/* Iterator for records contained in a single block. */
+struct block_iter {
+	/* offset within the block of the next entry to read. */
+	uint32_t next_off;
+	const struct reftable_block *block;
+
+	/* key for last entry we read. */
+	struct reftable_buf last_key;
+	struct reftable_buf scratch;
+};
+
+#define BLOCK_ITER_INIT { \
+	.last_key = REFTABLE_BUF_INIT, \
+	.scratch = REFTABLE_BUF_INIT, \
+}
+
+/*
+ * Initialize the block iterator with the given block. The iterator will be
+ * positioned at the first record contained in the block. The block must remain
+ * valid until the end of the iterator's lifetime. It is valid to re-initialize
+ * iterators multiple times.
+ */
+void block_iter_init(struct block_iter *it, const struct reftable_block *block);
+
+/* Position the initialized iterator at the first record of its block. */
+void block_iter_seek_start(struct block_iter *it);
+
+/*
+ * Position the initialized iterator at the desired record key. It is not an
+ * error in case the record cannot be found. If so, a subsequent call to
+ * `block_iter_next()` will indicate that the iterator is exhausted.
+ */
+int block_iter_seek_key(struct block_iter *it, struct reftable_buf *want);
+
+/* return < 0 for error, 0 for OK, > 0 for EOF. */
+int block_iter_next(struct block_iter *it, struct reftable_record *rec);
+
+/* Reset the block iterator to pristine state without releasing its memory. */
+void block_iter_reset(struct block_iter *it);
+
+/* deallocate memory for `it`. The block reader and its block is left intact. */
+void block_iter_close(struct block_iter *it);
+
+/* size of file header, depending on format version */
+size_t header_size(int version);
+
+/* size of file footer, depending on format version */
+size_t footer_size(int version);
+
+#endif