From 1424585fb64cbfdb86ff1637830634ec7e9247e0 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 10 Jan 2026 14:05:19 +0000 Subject: [PATCH 1/8] tests: move clar to `deps` Clean up the `tests` folder to only contain _actual tests_. Since clar is now a reasonably external project, move it to `deps`. --- {tests => deps}/clar/clar.c | 0 {tests => deps}/clar/clar.h | 0 {tests => deps}/clar/clar/fixtures.h | 0 {tests => deps}/clar/clar/fs.h | 0 {tests => deps}/clar/clar/print.h | 0 {tests => deps}/clar/clar/sandbox.h | 0 {tests => deps}/clar/clar/summary.h | 0 {tests => deps}/clar/clar_libgit2.c | 0 {tests => deps}/clar/clar_libgit2.h | 0 {tests => deps}/clar/clar_libgit2_alloc.c | 0 {tests => deps}/clar/clar_libgit2_alloc.h | 0 {tests => deps}/clar/clar_libgit2_timer.c | 0 {tests => deps}/clar/clar_libgit2_timer.h | 0 {tests => deps}/clar/clar_libgit2_trace.c | 0 {tests => deps}/clar/clar_libgit2_trace.h | 0 {tests => deps}/clar/generate.py | 0 {tests => deps}/clar/main.c | 0 tests/README.md | 2 -- tests/libgit2/CMakeLists.txt | 2 +- tests/util/CMakeLists.txt | 2 +- 20 files changed, 2 insertions(+), 4 deletions(-) rename {tests => deps}/clar/clar.c (100%) rename {tests => deps}/clar/clar.h (100%) rename {tests => deps}/clar/clar/fixtures.h (100%) rename {tests => deps}/clar/clar/fs.h (100%) rename {tests => deps}/clar/clar/print.h (100%) rename {tests => deps}/clar/clar/sandbox.h (100%) rename {tests => deps}/clar/clar/summary.h (100%) rename {tests => deps}/clar/clar_libgit2.c (100%) rename {tests => deps}/clar/clar_libgit2.h (100%) rename {tests => deps}/clar/clar_libgit2_alloc.c (100%) rename {tests => deps}/clar/clar_libgit2_alloc.h (100%) rename {tests => deps}/clar/clar_libgit2_timer.c (100%) rename {tests => deps}/clar/clar_libgit2_timer.h (100%) rename {tests => deps}/clar/clar_libgit2_trace.c (100%) rename {tests => deps}/clar/clar_libgit2_trace.h (100%) rename {tests => deps}/clar/generate.py (100%) rename {tests => deps}/clar/main.c (100%) diff --git a/tests/clar/clar.c b/deps/clar/clar.c similarity index 100% rename from tests/clar/clar.c rename to deps/clar/clar.c diff --git a/tests/clar/clar.h b/deps/clar/clar.h similarity index 100% rename from tests/clar/clar.h rename to deps/clar/clar.h diff --git a/tests/clar/clar/fixtures.h b/deps/clar/clar/fixtures.h similarity index 100% rename from tests/clar/clar/fixtures.h rename to deps/clar/clar/fixtures.h diff --git a/tests/clar/clar/fs.h b/deps/clar/clar/fs.h similarity index 100% rename from tests/clar/clar/fs.h rename to deps/clar/clar/fs.h diff --git a/tests/clar/clar/print.h b/deps/clar/clar/print.h similarity index 100% rename from tests/clar/clar/print.h rename to deps/clar/clar/print.h diff --git a/tests/clar/clar/sandbox.h b/deps/clar/clar/sandbox.h similarity index 100% rename from tests/clar/clar/sandbox.h rename to deps/clar/clar/sandbox.h diff --git a/tests/clar/clar/summary.h b/deps/clar/clar/summary.h similarity index 100% rename from tests/clar/clar/summary.h rename to deps/clar/clar/summary.h diff --git a/tests/clar/clar_libgit2.c b/deps/clar/clar_libgit2.c similarity index 100% rename from tests/clar/clar_libgit2.c rename to deps/clar/clar_libgit2.c diff --git a/tests/clar/clar_libgit2.h b/deps/clar/clar_libgit2.h similarity index 100% rename from tests/clar/clar_libgit2.h rename to deps/clar/clar_libgit2.h diff --git a/tests/clar/clar_libgit2_alloc.c b/deps/clar/clar_libgit2_alloc.c similarity index 100% rename from tests/clar/clar_libgit2_alloc.c rename to deps/clar/clar_libgit2_alloc.c diff --git a/tests/clar/clar_libgit2_alloc.h b/deps/clar/clar_libgit2_alloc.h similarity index 100% rename from tests/clar/clar_libgit2_alloc.h rename to deps/clar/clar_libgit2_alloc.h diff --git a/tests/clar/clar_libgit2_timer.c b/deps/clar/clar_libgit2_timer.c similarity index 100% rename from tests/clar/clar_libgit2_timer.c rename to deps/clar/clar_libgit2_timer.c diff --git a/tests/clar/clar_libgit2_timer.h b/deps/clar/clar_libgit2_timer.h similarity index 100% rename from tests/clar/clar_libgit2_timer.h rename to deps/clar/clar_libgit2_timer.h diff --git a/tests/clar/clar_libgit2_trace.c b/deps/clar/clar_libgit2_trace.c similarity index 100% rename from tests/clar/clar_libgit2_trace.c rename to deps/clar/clar_libgit2_trace.c diff --git a/tests/clar/clar_libgit2_trace.h b/deps/clar/clar_libgit2_trace.h similarity index 100% rename from tests/clar/clar_libgit2_trace.h rename to deps/clar/clar_libgit2_trace.h diff --git a/tests/clar/generate.py b/deps/clar/generate.py similarity index 100% rename from tests/clar/generate.py rename to deps/clar/generate.py diff --git a/tests/clar/main.c b/deps/clar/main.c similarity index 100% rename from tests/clar/main.c rename to deps/clar/main.c diff --git a/tests/README.md b/tests/README.md index 460e045e3..4c1a5890c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,8 +4,6 @@ These are the unit and integration tests for the libgit2 projects. * `benchmarks` These are benchmark tests that excercise the CLI. -* `clar` - This is [clar](https://github.com/clar-test/clar) the common test framework. * `headertest` This is a simple project that ensures that our public headers are compatible with extremely strict compilation options. diff --git a/tests/libgit2/CMakeLists.txt b/tests/libgit2/CMakeLists.txt index 7a4bb476c..f8b80eb32 100644 --- a/tests/libgit2/CMakeLists.txt +++ b/tests/libgit2/CMakeLists.txt @@ -12,7 +12,7 @@ if(NOT PYTHONINTERP_FOUND) "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests") ENDIF() -set(CLAR_PATH "${PROJECT_SOURCE_DIR}/tests/clar") +set(CLAR_PATH "${PROJECT_SOURCE_DIR}/deps/clar") set(CLAR_FIXTURES "${PROJECT_SOURCE_DIR}/tests/resources/") set(TEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}") add_definitions(-DCLAR_FIXTURE_PATH=\"${CLAR_FIXTURES}\") diff --git a/tests/util/CMakeLists.txt b/tests/util/CMakeLists.txt index 425c27dcd..6b44b85fa 100644 --- a/tests/util/CMakeLists.txt +++ b/tests/util/CMakeLists.txt @@ -12,7 +12,7 @@ if(NOT PYTHONINTERP_FOUND) "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests") ENDIF() -set(CLAR_PATH "${libgit2_SOURCE_DIR}/tests/clar") +set(CLAR_PATH "${libgit2_SOURCE_DIR}/deps/clar") set(CLAR_FIXTURES "${libgit2_SOURCE_DIR}/tests/resources/") set(TEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}") add_definitions(-DCLAR_FIXTURE_PATH=\"${CLAR_FIXTURES}\") From 8fda867e468093a2caf36c38bfb58c7bf883643e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 10 Jan 2026 14:14:27 +0000 Subject: [PATCH 2/8] benchmarks: move them to their own folder Benchmarks aren't really tests, don't keep them in the `tests` folder. --- .github/workflows/benchmark.yml | 2 +- {tests/benchmarks => benchmarks/cli}/README.md | 0 .../cli}/_script/flamegraph/README.md | 0 .../cli}/_script/flamegraph/aix-perf.pl | 0 .../cli}/_script/flamegraph/difffolded.pl | 0 .../_script/flamegraph/example-dtrace-stacks.txt | 0 .../cli}/_script/flamegraph/example-dtrace.svg | 0 .../_script/flamegraph/example-perf-stacks.txt.gz | Bin .../cli}/_script/flamegraph/example-perf.svg | 0 .../cli}/_script/flamegraph/files.pl | 0 .../cli}/_script/flamegraph/flamegraph.pl | 0 .../cli}/_script/flamegraph/jmaps | 0 .../cli}/_script/flamegraph/pkgsplit-perf.pl | 0 .../cli}/_script/flamegraph/range-perf.pl | 0 .../cli}/_script/flamegraph/record-test.sh | 0 .../cli}/_script/flamegraph/stackcollapse-aix.pl | 0 .../_script/flamegraph/stackcollapse-bpftrace.pl | 0 .../flamegraph/stackcollapse-chrome-tracing.py | 0 .../_script/flamegraph/stackcollapse-elfutils.pl | 0 .../flamegraph/stackcollapse-faulthandler.pl | 0 .../cli}/_script/flamegraph/stackcollapse-gdb.pl | 0 .../cli}/_script/flamegraph/stackcollapse-go.pl | 0 .../_script/flamegraph/stackcollapse-ibmjava.pl | 0 .../_script/flamegraph/stackcollapse-instruments.pl | 0 .../flamegraph/stackcollapse-java-exceptions.pl | 0 .../cli}/_script/flamegraph/stackcollapse-jstack.pl | 0 .../cli}/_script/flamegraph/stackcollapse-ljp.awk | 0 .../_script/flamegraph/stackcollapse-perf-sched.awk | 0 .../cli}/_script/flamegraph/stackcollapse-perf.pl | 0 .../cli}/_script/flamegraph/stackcollapse-pmc.pl | 0 .../_script/flamegraph/stackcollapse-recursive.pl | 0 .../_script/flamegraph/stackcollapse-sample.awk | 0 .../cli}/_script/flamegraph/stackcollapse-stap.pl | 0 .../cli}/_script/flamegraph/stackcollapse-vsprof.pl | 0 .../_script/flamegraph/stackcollapse-vtune-mc.pl | 0 .../cli}/_script/flamegraph/stackcollapse-vtune.pl | 0 .../cli}/_script/flamegraph/stackcollapse-wcp.pl | 0 .../_script/flamegraph/stackcollapse-xdebug.php | 0 .../cli}/_script/flamegraph/stackcollapse.pl | 0 .../cli}/_script/flamegraph/test.sh | 0 {tests/benchmarks => benchmarks/cli}/benchmark.sh | 0 .../cli}/benchmark_helpers.sh | 2 +- {tests/benchmarks => benchmarks/cli}/blame__git | 0 {tests/benchmarks => benchmarks/cli}/blame__linux | 0 {tests/benchmarks => benchmarks/cli}/blame__simple | 0 .../cli}/hash-object__text_100kb | 0 .../cli}/hash-object__text_10mb | 0 .../cli}/hash-object__text_1kb | 0 .../cli}/hash-object__text_nocache_100kb | 0 .../cli}/hash-object__text_nocache_10mb | 0 .../cli}/hash-object__text_nocache_1kb | 0 .../cli}/hash-object__write_text_100kb | 0 .../cli}/hash-object__write_text_10mb | 0 .../cli}/hash-object__write_text_1kb | 0 .../cli}/hash-object__write_text_nocache_100kb | 0 .../cli}/hash-object__write_text_nocache_10mb | 0 .../cli}/hash-object__write_text_nocache_1kb | 0 .../benchmarks => benchmarks/cli}/indexpack__250mb | 0 tests/README.md | 2 -- 59 files changed, 2 insertions(+), 4 deletions(-) rename {tests/benchmarks => benchmarks/cli}/README.md (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/README.md (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/aix-perf.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/difffolded.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/example-dtrace-stacks.txt (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/example-dtrace.svg (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/example-perf-stacks.txt.gz (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/example-perf.svg (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/files.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/flamegraph.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/jmaps (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/pkgsplit-perf.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/range-perf.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/record-test.sh (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-aix.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-bpftrace.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-chrome-tracing.py (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-elfutils.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-faulthandler.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-gdb.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-go.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-ibmjava.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-instruments.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-java-exceptions.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-jstack.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-ljp.awk (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-perf-sched.awk (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-perf.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-pmc.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-recursive.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-sample.awk (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-stap.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-vsprof.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-vtune-mc.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-vtune.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-wcp.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse-xdebug.php (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/stackcollapse.pl (100%) rename {tests/benchmarks => benchmarks/cli}/_script/flamegraph/test.sh (100%) rename {tests/benchmarks => benchmarks/cli}/benchmark.sh (100%) rename {tests/benchmarks => benchmarks/cli}/benchmark_helpers.sh (99%) rename {tests/benchmarks => benchmarks/cli}/blame__git (100%) rename {tests/benchmarks => benchmarks/cli}/blame__linux (100%) rename {tests/benchmarks => benchmarks/cli}/blame__simple (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__text_100kb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__text_10mb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__text_1kb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__text_nocache_100kb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__text_nocache_10mb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__text_nocache_1kb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__write_text_100kb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__write_text_10mb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__write_text_1kb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__write_text_nocache_100kb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__write_text_nocache_10mb (100%) rename {tests/benchmarks => benchmarks/cli}/hash-object__write_text_nocache_1kb (100%) rename {tests/benchmarks => benchmarks/cli}/indexpack__250mb (100%) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 103f4bcd0..1ff7178e9 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -115,7 +115,7 @@ jobs: fi mkdir benchmark && cd benchmark - ../source/tests/benchmarks/benchmark.sh \ + ../source/benchmarks/cli/benchmark.sh \ ${SUITE_FLAG} ${DEBUG_FLAG} \ --admin \ --baseline-cli "git" --cli "${GIT2_CLI}" --name libgit2 \ diff --git a/tests/benchmarks/README.md b/benchmarks/cli/README.md similarity index 100% rename from tests/benchmarks/README.md rename to benchmarks/cli/README.md diff --git a/tests/benchmarks/_script/flamegraph/README.md b/benchmarks/cli/_script/flamegraph/README.md similarity index 100% rename from tests/benchmarks/_script/flamegraph/README.md rename to benchmarks/cli/_script/flamegraph/README.md diff --git a/tests/benchmarks/_script/flamegraph/aix-perf.pl b/benchmarks/cli/_script/flamegraph/aix-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/aix-perf.pl rename to benchmarks/cli/_script/flamegraph/aix-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/difffolded.pl b/benchmarks/cli/_script/flamegraph/difffolded.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/difffolded.pl rename to benchmarks/cli/_script/flamegraph/difffolded.pl diff --git a/tests/benchmarks/_script/flamegraph/example-dtrace-stacks.txt b/benchmarks/cli/_script/flamegraph/example-dtrace-stacks.txt similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-dtrace-stacks.txt rename to benchmarks/cli/_script/flamegraph/example-dtrace-stacks.txt diff --git a/tests/benchmarks/_script/flamegraph/example-dtrace.svg b/benchmarks/cli/_script/flamegraph/example-dtrace.svg similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-dtrace.svg rename to benchmarks/cli/_script/flamegraph/example-dtrace.svg diff --git a/tests/benchmarks/_script/flamegraph/example-perf-stacks.txt.gz b/benchmarks/cli/_script/flamegraph/example-perf-stacks.txt.gz similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-perf-stacks.txt.gz rename to benchmarks/cli/_script/flamegraph/example-perf-stacks.txt.gz diff --git a/tests/benchmarks/_script/flamegraph/example-perf.svg b/benchmarks/cli/_script/flamegraph/example-perf.svg similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-perf.svg rename to benchmarks/cli/_script/flamegraph/example-perf.svg diff --git a/tests/benchmarks/_script/flamegraph/files.pl b/benchmarks/cli/_script/flamegraph/files.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/files.pl rename to benchmarks/cli/_script/flamegraph/files.pl diff --git a/tests/benchmarks/_script/flamegraph/flamegraph.pl b/benchmarks/cli/_script/flamegraph/flamegraph.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/flamegraph.pl rename to benchmarks/cli/_script/flamegraph/flamegraph.pl diff --git a/tests/benchmarks/_script/flamegraph/jmaps b/benchmarks/cli/_script/flamegraph/jmaps similarity index 100% rename from tests/benchmarks/_script/flamegraph/jmaps rename to benchmarks/cli/_script/flamegraph/jmaps diff --git a/tests/benchmarks/_script/flamegraph/pkgsplit-perf.pl b/benchmarks/cli/_script/flamegraph/pkgsplit-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/pkgsplit-perf.pl rename to benchmarks/cli/_script/flamegraph/pkgsplit-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/range-perf.pl b/benchmarks/cli/_script/flamegraph/range-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/range-perf.pl rename to benchmarks/cli/_script/flamegraph/range-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/record-test.sh b/benchmarks/cli/_script/flamegraph/record-test.sh similarity index 100% rename from tests/benchmarks/_script/flamegraph/record-test.sh rename to benchmarks/cli/_script/flamegraph/record-test.sh diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-aix.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-aix.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-aix.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-aix.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-bpftrace.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-bpftrace.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-bpftrace.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-bpftrace.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-chrome-tracing.py b/benchmarks/cli/_script/flamegraph/stackcollapse-chrome-tracing.py similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-chrome-tracing.py rename to benchmarks/cli/_script/flamegraph/stackcollapse-chrome-tracing.py diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-elfutils.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-elfutils.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-elfutils.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-elfutils.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-faulthandler.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-faulthandler.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-faulthandler.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-faulthandler.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-gdb.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-gdb.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-gdb.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-gdb.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-go.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-go.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-go.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-go.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-ibmjava.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-ibmjava.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-ibmjava.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-ibmjava.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-instruments.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-instruments.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-instruments.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-instruments.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-java-exceptions.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-java-exceptions.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-java-exceptions.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-java-exceptions.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-jstack.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-jstack.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-jstack.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-jstack.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-ljp.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-ljp.awk similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-ljp.awk rename to benchmarks/cli/_script/flamegraph/stackcollapse-ljp.awk diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-perf-sched.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-perf-sched.awk similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-perf-sched.awk rename to benchmarks/cli/_script/flamegraph/stackcollapse-perf-sched.awk diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-perf.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-perf.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-pmc.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-pmc.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-pmc.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-pmc.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-recursive.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-recursive.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-recursive.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-recursive.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-sample.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-sample.awk similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-sample.awk rename to benchmarks/cli/_script/flamegraph/stackcollapse-sample.awk diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-stap.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-stap.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-stap.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-stap.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-vsprof.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vsprof.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-vsprof.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-vsprof.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-vtune-mc.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vtune-mc.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-vtune-mc.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-vtune-mc.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-vtune.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vtune.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-vtune.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-vtune.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-wcp.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-wcp.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-wcp.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-wcp.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-xdebug.php b/benchmarks/cli/_script/flamegraph/stackcollapse-xdebug.php similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-xdebug.php rename to benchmarks/cli/_script/flamegraph/stackcollapse-xdebug.php diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse.pl b/benchmarks/cli/_script/flamegraph/stackcollapse.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse.pl diff --git a/tests/benchmarks/_script/flamegraph/test.sh b/benchmarks/cli/_script/flamegraph/test.sh similarity index 100% rename from tests/benchmarks/_script/flamegraph/test.sh rename to benchmarks/cli/_script/flamegraph/test.sh diff --git a/tests/benchmarks/benchmark.sh b/benchmarks/cli/benchmark.sh similarity index 100% rename from tests/benchmarks/benchmark.sh rename to benchmarks/cli/benchmark.sh diff --git a/tests/benchmarks/benchmark_helpers.sh b/benchmarks/cli/benchmark_helpers.sh similarity index 99% rename from tests/benchmarks/benchmark_helpers.sh rename to benchmarks/cli/benchmark_helpers.sh index cf0cd5121..f78aabc09 100644 --- a/tests/benchmarks/benchmark_helpers.sh +++ b/benchmarks/cli/benchmark_helpers.sh @@ -105,7 +105,7 @@ fullpath() { } resources_dir() { - cd "$(dirname "$0")/../resources" && pwd + cd "$(dirname "$0")/../../tests/resources" && pwd } temp_dir() { diff --git a/tests/benchmarks/blame__git b/benchmarks/cli/blame__git similarity index 100% rename from tests/benchmarks/blame__git rename to benchmarks/cli/blame__git diff --git a/tests/benchmarks/blame__linux b/benchmarks/cli/blame__linux similarity index 100% rename from tests/benchmarks/blame__linux rename to benchmarks/cli/blame__linux diff --git a/tests/benchmarks/blame__simple b/benchmarks/cli/blame__simple similarity index 100% rename from tests/benchmarks/blame__simple rename to benchmarks/cli/blame__simple diff --git a/tests/benchmarks/hash-object__text_100kb b/benchmarks/cli/hash-object__text_100kb similarity index 100% rename from tests/benchmarks/hash-object__text_100kb rename to benchmarks/cli/hash-object__text_100kb diff --git a/tests/benchmarks/hash-object__text_10mb b/benchmarks/cli/hash-object__text_10mb similarity index 100% rename from tests/benchmarks/hash-object__text_10mb rename to benchmarks/cli/hash-object__text_10mb diff --git a/tests/benchmarks/hash-object__text_1kb b/benchmarks/cli/hash-object__text_1kb similarity index 100% rename from tests/benchmarks/hash-object__text_1kb rename to benchmarks/cli/hash-object__text_1kb diff --git a/tests/benchmarks/hash-object__text_nocache_100kb b/benchmarks/cli/hash-object__text_nocache_100kb similarity index 100% rename from tests/benchmarks/hash-object__text_nocache_100kb rename to benchmarks/cli/hash-object__text_nocache_100kb diff --git a/tests/benchmarks/hash-object__text_nocache_10mb b/benchmarks/cli/hash-object__text_nocache_10mb similarity index 100% rename from tests/benchmarks/hash-object__text_nocache_10mb rename to benchmarks/cli/hash-object__text_nocache_10mb diff --git a/tests/benchmarks/hash-object__text_nocache_1kb b/benchmarks/cli/hash-object__text_nocache_1kb similarity index 100% rename from tests/benchmarks/hash-object__text_nocache_1kb rename to benchmarks/cli/hash-object__text_nocache_1kb diff --git a/tests/benchmarks/hash-object__write_text_100kb b/benchmarks/cli/hash-object__write_text_100kb similarity index 100% rename from tests/benchmarks/hash-object__write_text_100kb rename to benchmarks/cli/hash-object__write_text_100kb diff --git a/tests/benchmarks/hash-object__write_text_10mb b/benchmarks/cli/hash-object__write_text_10mb similarity index 100% rename from tests/benchmarks/hash-object__write_text_10mb rename to benchmarks/cli/hash-object__write_text_10mb diff --git a/tests/benchmarks/hash-object__write_text_1kb b/benchmarks/cli/hash-object__write_text_1kb similarity index 100% rename from tests/benchmarks/hash-object__write_text_1kb rename to benchmarks/cli/hash-object__write_text_1kb diff --git a/tests/benchmarks/hash-object__write_text_nocache_100kb b/benchmarks/cli/hash-object__write_text_nocache_100kb similarity index 100% rename from tests/benchmarks/hash-object__write_text_nocache_100kb rename to benchmarks/cli/hash-object__write_text_nocache_100kb diff --git a/tests/benchmarks/hash-object__write_text_nocache_10mb b/benchmarks/cli/hash-object__write_text_nocache_10mb similarity index 100% rename from tests/benchmarks/hash-object__write_text_nocache_10mb rename to benchmarks/cli/hash-object__write_text_nocache_10mb diff --git a/tests/benchmarks/hash-object__write_text_nocache_1kb b/benchmarks/cli/hash-object__write_text_nocache_1kb similarity index 100% rename from tests/benchmarks/hash-object__write_text_nocache_1kb rename to benchmarks/cli/hash-object__write_text_nocache_1kb diff --git a/tests/benchmarks/indexpack__250mb b/benchmarks/cli/indexpack__250mb similarity index 100% rename from tests/benchmarks/indexpack__250mb rename to benchmarks/cli/indexpack__250mb diff --git a/tests/README.md b/tests/README.md index 4c1a5890c..78e625242 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,8 +2,6 @@ These are the unit and integration tests for the libgit2 projects. -* `benchmarks` - These are benchmark tests that excercise the CLI. * `headertest` This is a simple project that ensures that our public headers are compatible with extremely strict compilation options. From 1a0bbabe30057c98701b10541af166d103761a52 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 10 Jan 2026 22:23:20 +0000 Subject: [PATCH 3/8] clar update for benchmarks --- deps/clar/clar.c | 187 ++++++++++++++++++++++++--------- deps/clar/clar.h | 22 ++++ deps/clar/clar/counter.h | 167 +++++++++++++++++++++++++++++ deps/clar/clar/print.h | 174 ++++++++++++++++++++++++++---- deps/clar/clar/summary.h | 221 ++++++++++++++++++++++++++++++++++----- deps/clar/generate.py | 121 +++++++++++++++------ 6 files changed, 765 insertions(+), 127 deletions(-) create mode 100644 deps/clar/clar/counter.h diff --git a/deps/clar/clar.c b/deps/clar/clar.c index e959a5ae0..1a206fcd2 100644 --- a/deps/clar/clar.c +++ b/deps/clar/clar.c @@ -100,6 +100,7 @@ typedef struct stat STAT_T; #endif +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #include "clar.h" @@ -113,11 +114,11 @@ fixture_path(const char *base, const char *fixture_name); #endif struct clar_error { - const char *file; - const char *function; - uintmax_t line_number; - const char *error_msg; + const char *message; char *description; + const char *function; + const char *file; + uintmax_t line_number; struct clar_error *next; }; @@ -130,13 +131,22 @@ struct clar_explicit { }; struct clar_report { - const char *test; - int test_number; const char *suite; + const char *test; + const char *description; + int test_number; + + int runs; enum cl_test_status status; time_t start; - double elapsed; + + double *times; + double time_min; + double time_max; + double time_mean; + double time_stddev; + double time_total; struct clar_error *errors; struct clar_error *last_error; @@ -150,10 +160,12 @@ struct clar_summary { }; static struct { + enum cl_test_mode test_mode; enum cl_test_status test_status; - const char *active_test; const char *active_suite; + const char *active_test; + const char *active_description; int total_skipped; int total_errors; @@ -162,6 +174,7 @@ static struct { int suites_ran; enum cl_output_format output_format; + enum cl_summary_format summary_format; int exit_on_error; int verbosity; @@ -193,28 +206,32 @@ static struct { struct clar_func { const char *name; + const char *description; + int runs; void (*ptr)(void); }; struct clar_suite { const char *name; struct clar_func initialize; + struct clar_func reset; struct clar_func cleanup; const struct clar_func *tests; size_t test_count; int enabled; }; -/* From clar_print_*.c */ +/* From print.h */ static void clar_print_init(int test_count, int suite_count); static void clar_print_shutdown(int test_count, int suite_count, int error_count); static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error); -static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status failed); -static void clar_print_onsuite(const char *suite_name, int suite_index); +static void clar_print_suite_start(const char *suite_name, int suite_index); +static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number); +static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report); static void clar_print_onabortv(const char *msg, va_list argp); static void clar_print_onabort(const char *msg, ...); -/* From clar_sandbox.c */ +/* From sandbox.c */ static void clar_tempdir_init(void); static void clar_tempdir_shutdown(void); static int clar_sandbox_create(const char *suite_name, const char *test_name); @@ -224,6 +241,8 @@ static int clar_sandbox_cleanup(void); static struct clar_summary *clar_summary_init(const char *filename); static int clar_summary_shutdown(struct clar_summary *fp); +#include "clar/counter.h" + /* Load the declarations for the test suite */ #include "clar.suite" @@ -280,70 +299,121 @@ clar_report_all(void) } } -#ifdef WIN32 -# define clar_time DWORD - -static void clar_time_now(clar_time *out) +static void +compute_times(void) { - *out = GetTickCount(); -} + double total_squares = 0; + int i; -static double clar_time_diff(clar_time *start, clar_time *end) -{ - return ((double)*end - (double)*start) / 1000; -} -#else -# include + _clar.last_report->time_min = _clar.last_report->times[0]; + _clar.last_report->time_max = _clar.last_report->times[0]; + _clar.last_report->time_total = _clar.last_report->times[0]; -# define clar_time struct timeval + for (i = 1; i < _clar.last_report->runs; i++) { + if (_clar.last_report->times[i] < _clar.last_report->time_min) + _clar.last_report->time_min = _clar.last_report->times[i]; -static void clar_time_now(clar_time *out) -{ - gettimeofday(out, NULL); -} + if (_clar.last_report->times[i] > _clar.last_report->time_max) + _clar.last_report->time_max = _clar.last_report->times[i]; -static double clar_time_diff(clar_time *start, clar_time *end) -{ - return ((double)end->tv_sec + (double)end->tv_usec / 1.0E6) - - ((double)start->tv_sec + (double)start->tv_usec / 1.0E6); + _clar.last_report->time_total += _clar.last_report->times[i]; + } + + if (_clar.last_report->runs <= 1) { + _clar.last_report->time_stddev = 0; + } else { + _clar.last_report->time_mean = _clar.last_report->time_total / _clar.last_report->runs; + + for (i = 0; i < _clar.last_report->runs; i++) { + double dev = (_clar.last_report->times[i] > _clar.last_report->time_mean) ? + _clar.last_report->times[i] - _clar.last_report->time_mean : + _clar.last_report->time_mean - _clar.last_report->times[i]; + + total_squares += (dev * dev); + } + + _clar.last_report->time_stddev = sqrt(total_squares / _clar.last_report->runs); + } } -#endif static void clar_run_test( const struct clar_suite *suite, const struct clar_func *test, const struct clar_func *initialize, + const struct clar_func *reset, const struct clar_func *cleanup) { clar_time start, end; - _clar.trampoline_enabled = 1; + _clar.last_report->start = time(NULL); + _clar.last_report->times = &_clar.last_report->time_mean; CL_TRACE(CL_TRACE__TEST__BEGIN); clar_sandbox_create(suite->name, test->name); - _clar.last_report->start = time(NULL); - clar_time_now(&start); + clar_print_test_start(suite->name, test->name, _clar.tests_ran); + + _clar.trampoline_enabled = 1; if (setjmp(_clar.trampoline) == 0) { if (initialize->ptr != NULL) initialize->ptr(); CL_TRACE(CL_TRACE__TEST__RUN_BEGIN); - test->ptr(); + + do { + struct clar_counter start, end; + double elapsed; + + if (i > 0 && reset->ptr != NULL) { + reset->ptr(); + } else if (i > 0) { + if (_clar.local_cleanup != NULL) + _clar.local_cleanup(_clar.local_cleanup_payload); + if (cleanup->ptr != NULL) + cleanup->ptr(); + if (initialize->ptr != NULL) + initialize->ptr(); + } + + clar_counter_now(&start); + test->ptr(); + clar_counter_now(&end); + + elapsed = clar_counter_diff(&start, &end); + + /* + * unless the number of runs was explicitly given + * in benchmark mode, use the first run as a sample + * to determine how many runs we should attempt + */ + if (_clar.test_mode == CL_TEST_BENCHMARK && !runs) { + runs = MAX(CLAR_BENCHMARK_RUN_MIN, (int)(CLAR_BENCHMARK_RUN_TIME / elapsed)); + runs = MIN(CLAR_BENCHMARK_RUN_MAX, runs); + } + + if (i == 0 && runs > 1) { + _clar.last_report->times = calloc(runs, sizeof(double)); + + if (_clar.last_report->times == NULL) + clar_abort("Failed to allocate report times.\n"); + } + + _clar.last_report->runs++; + _clar.last_report->times[i] = elapsed; + } while(++i < runs); + CL_TRACE(CL_TRACE__TEST__RUN_END); } - clar_time_now(&end); - _clar.trampoline_enabled = 0; if (_clar.last_report->status == CL_TEST_NOTRUN) _clar.last_report->status = CL_TEST_OK; - _clar.last_report->elapsed = clar_time_diff(&start, &end); + compute_times(); if (_clar.local_cleanup != NULL) _clar.local_cleanup(_clar.local_cleanup_payload); @@ -363,7 +433,7 @@ clar_run_test( _clar.local_cleanup = NULL; _clar.local_cleanup_payload = NULL; - clar_print_ontest(suite->name, test->name, _clar.tests_ran, _clar.last_report->status); + clar_print_test_finish(suite->name, test->name, _clar.tests_ran, _clar.last_report); } static void @@ -380,10 +450,11 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) if (_clar.exit_on_error && _clar.total_errors) return; - clar_print_onsuite(suite->name, ++_clar.suites_ran); + clar_print_suite_start(suite->name, ++_clar.suites_ran); _clar.active_suite = suite->name; _clar.active_test = NULL; + _clar.active_description = NULL; CL_TRACE(CL_TRACE__SUITE_BEGIN); if (filter) { @@ -412,11 +483,13 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) continue; _clar.active_test = test[i].name; + _clar.active_description = test[i].description; if ((report = calloc(1, sizeof(*report))) == NULL) clar_abort("Failed to allocate report.\n"); report->suite = _clar.active_suite; report->test = _clar.active_test; + report->description = _clar.active_description; report->test_number = _clar.tests_ran; report->status = CL_TEST_NOTRUN; @@ -428,13 +501,14 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) _clar.last_report = report; - clar_run_test(suite, &test[i], &suite->initialize, &suite->cleanup); + clar_run_test(suite, &test[i], &suite->initialize, &suite->reset, &suite->cleanup); if (_clar.exit_on_error && _clar.total_errors) return; } _clar.active_test = NULL; + _clar.active_description = NULL; CL_TRACE(CL_TRACE__SUITE_END); } @@ -600,6 +674,14 @@ clar_test_init(int argc, char **argv) { const char *summary_env; + if (_clar.test_mode == CL_TEST_BENCHMARK) { + _clar.output_format = CL_OUTPUT_TIMING; + _clar.summary_format = CL_SUMMARY_JSON; + } else { + _clar.output_format = CL_OUTPUT_CLAP; + _clar.summary_format = CL_SUMMARY_JUNIT; + } + if (argc > 1) clar_parse_args(argc, argv); @@ -622,6 +704,12 @@ clar_test_init(int argc, char **argv) clar_tempdir_init(); } +void +clar_test_set_mode(enum cl_test_mode mode) +{ + _clar.test_mode = mode; +} + int clar_test_run(void) { @@ -671,6 +759,9 @@ clar_test_shutdown(void) free(error); } + if (report->times != &report->time_mean) + free(report->times); + report_next = report->next; free(report); } @@ -735,7 +826,7 @@ static void clar__failv( error->file = _clar.invoke_file ? _clar.invoke_file : file; error->function = _clar.invoke_func ? _clar.invoke_func : function; error->line_number = _clar.invoke_line ? _clar.invoke_line : line; - error->error_msg = error_msg; + error->message = error_msg; if (description != NULL) { va_list args_copy; @@ -791,14 +882,14 @@ void clar__assert( const char *file, const char *function, size_t line, - const char *error_msg, - const char *description, + const char *error_message, + const char *error_description, int should_abort) { if (condition) return; - clar__fail(file, function, line, error_msg, description, should_abort); + clar__fail(file, function, line, error_message, error_description, should_abort); } void clar__assert_equal( diff --git a/deps/clar/clar.h b/deps/clar/clar.h index 9ea91d3d0..47609df3a 100644 --- a/deps/clar/clar.h +++ b/deps/clar/clar.h @@ -21,6 +21,16 @@ # define CLAR_MAX_PATH 4096 #endif +/* + * In benchmark mode, by default, clar will run the test repeatedly for + * approximately `CLAR_BENCHMARK_RUN_TIME` seconds, and at least + * `CLAR_BENCHMARK_RUN_MIN` iterations. + */ + +#define CLAR_BENCHMARK_RUN_TIME 3.0 +#define CLAR_BENCHMARK_RUN_MIN 10 +#define CLAR_BENCHMARK_RUN_MAX 30000000 + #ifndef CLAR_SELFTEST # define CLAR_CURRENT_FILE __FILE__ # define CLAR_CURRENT_LINE __LINE__ @@ -31,6 +41,11 @@ # define CLAR_CURRENT_FUNC "func" #endif +enum cl_test_mode { + CL_TEST_STANDARD, + CL_TEST_BENCHMARK, +}; + enum cl_test_status { CL_TEST_OK, CL_TEST_FAILURE, @@ -41,10 +56,17 @@ enum cl_test_status { enum cl_output_format { CL_OUTPUT_CLAP, CL_OUTPUT_TAP, + CL_OUTPUT_TIMING, +}; + +enum cl_summary_format { + CL_SUMMARY_JUNIT, + CL_SUMMARY_JSON, }; /** Setup clar environment */ void clar_test_init(int argc, char *argv[]); +void clar_test_set_mode(enum cl_test_mode mode); int clar_test_run(void); void clar_test_shutdown(void); diff --git a/deps/clar/clar/counter.h b/deps/clar/clar/counter.h new file mode 100644 index 000000000..329f7481e --- /dev/null +++ b/deps/clar/clar/counter.h @@ -0,0 +1,167 @@ +#define CLAR_COUNTER_TV_DIFF(out_sec, out_usec, start_sec, start_usec, end_sec, end_usec) \ + if (start_usec > end_usec) { \ + out_sec = (end_sec - 1) - start_sec; \ + out_usec = (end_usec + 1000000) - start_usec; \ + } else { \ + out_sec = end_sec - start_sec; \ + out_usec = end_usec - start_usec; \ + } + +#ifdef _WIN32 + +struct clar_counter { + LARGE_INTEGER value; +}; + +static void clar_counter_now(struct clar_counter *out) +{ + QueryPerformanceCounter(&out->value); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + LARGE_INTEGER freq; + + QueryPerformanceFrequency(&freq); + return (double)(end->value.QuadPart - start->value.QuadPart)/(double)freq.QuadPart; +} + +#elif __APPLE__ + +#include +#include + +static double clar_counter_scaling_factor = -1; + +struct clar_counter { + union { + uint64_t absolute_time; + struct timeval tv; + } val; +}; + +static void clar_counter_now(struct clar_counter *out) +{ + if (clar_counter_scaling_factor == 0) { + mach_timebase_info_data_t info; + + clar_counter_scaling_factor = + mach_timebase_info(&info) == KERN_SUCCESS ? + ((double)info.numer / (double)info.denom) / 1.0E6 : + -1; + } + + /* mach_timebase_info failed; fall back to gettimeofday */ + if (clar_counter_scaling_factor < 0) + gettimeofday(&out->val.tv, NULL); + else + out->val.absolute_time = mach_absolute_time(); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + if (clar_counter_scaling_factor < 0) { + time_t sec; + suseconds_t usec; + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->val.tv.tv_sec, start->val.tv.tv_usec, + end->val.tv.tv_sec, end->val.tv.tv_usec); + + return (double)sec + ((double)usec / 1000000.0); + } else { + return (double)(end->val.absolute_time - start->val.absolute_time) * + clar_counter_scaling_factor; + } +} + +#elif defined(__amigaos4__) + +#include + +struct clar_counter { + struct TimeVal tv; +} + +static void clar_counter_now(struct clar_counter *out) +{ + ITimer->GetUpTime(&out->tv); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + uint32_t sec, usec; + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->tv.Seconds, start->tv.Microseconds, + end->tv.Seconds, end->tv.Microseconds); + + return (double)sec + ((double)usec / 1000000.0); +} + +#else + +#include + +struct clar_counter { + int type; + union { +#ifdef CLOCK_MONOTONIC + struct timespec tp; +#endif + struct timeval tv; + } val; +}; + +static void clar_counter_now(struct clar_counter *out) +{ +#ifdef CLOCK_MONOTONIC + if (clock_gettime(CLOCK_MONOTONIC, &out->val.tp) == 0) { + out->type = 0; + return; + } +#endif + + /* Fall back to using gettimeofday */ + out->type = 1; + gettimeofday(&out->val.tv, NULL); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + time_t sec; + suseconds_t usec; + +#ifdef CLOCK_MONOTONIC + if (start->type == 0) { + time_t sec; + long nsec; + + if (start->val.tp.tv_sec > end->val.tp.tv_sec) { + sec = (end->val.tp.tv_sec - 1) - start->val.tp.tv_sec; + nsec = (end->val.tp.tv_nsec + 1000000000) - start->val.tp.tv_nsec; + } else { + sec = end->val.tp.tv_sec - start->val.tp.tv_sec; + nsec = end->val.tp.tv_nsec - start->val.tp.tv_nsec; + } + + return (double)sec + ((double)nsec / 1000000000.0); + } +#endif + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->val.tv.tv_sec, start->val.tv.tv_usec, + end->val.tv.tv_sec, end->val.tv.tv_usec); + + return (double)sec + ((double)usec / 1000000.0); +} + +#endif diff --git a/deps/clar/clar/print.h b/deps/clar/clar/print.h index 59b7dc14a..55aff5f4f 100644 --- a/deps/clar/clar/print.h +++ b/deps/clar/clar/print.h @@ -48,7 +48,7 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con error->file, error->line_number); - clar_print_indented(error->error_msg, 2); + clar_print_indented(error->message, 2); if (error->description != NULL) clar_print_indented(error->description, 2); @@ -57,9 +57,8 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con fflush(stdout); } -static void clar_print_clap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_clap_test_start(const char *suite_name, const char *test_name, int test_number) { - (void)test_name; (void)test_number; if (_clar.verbosity < 0) @@ -67,15 +66,18 @@ static void clar_print_clap_ontest(const char *suite_name, const char *test_name if (_clar.verbosity > 1) { printf("%s::%s: ", suite_name, test_name); + fflush(stdout); + } +} - switch (status) { - case CL_TEST_OK: printf("ok\n"); break; - case CL_TEST_FAILURE: printf("fail\n"); break; - case CL_TEST_SKIP: printf("skipped\n"); break; - case CL_TEST_NOTRUN: printf("notrun\n"); break; - } - } else { - switch (status) { +static void clar_print_clap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) +{ + (void)suite_name; + (void)test_name; + (void)test_number; + + if (_clar.verbosity == 0) { + switch (report->status) { case CL_TEST_OK: printf("."); break; case CL_TEST_FAILURE: printf("F"); break; case CL_TEST_SKIP: printf("S"); break; @@ -83,10 +85,17 @@ static void clar_print_clap_ontest(const char *suite_name, const char *test_name } fflush(stdout); + } else if (_clar.verbosity > 1) { + switch (report->status) { + case CL_TEST_OK: printf("ok\n"); break; + case CL_TEST_FAILURE: printf("fail\n"); break; + case CL_TEST_SKIP: printf("skipped\n"); break; + case CL_TEST_NOTRUN: printf("notrun\n"); break; + } } } -static void clar_print_clap_onsuite(const char *suite_name, int suite_index) +static void clar_print_clap_suite_start(const char *suite_name, int suite_index) { if (_clar.verbosity < 0) return; @@ -127,7 +136,7 @@ static void clar_print_tap_error(int num, const struct clar_report *report, cons static void print_escaped(const char *str) { - const char *c; + char *c; while ((c = strchr(str, '\'')) != NULL) { printf("%.*s", (int)(c - str), str); @@ -138,14 +147,21 @@ static void print_escaped(const char *str) printf("%s", str); } -static void clar_print_tap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_tap_test_start(const char *suite_name, const char *test_name, int test_number) +{ + (void)suite_name; + (void)test_name; + (void)test_number; +} + +static void clar_print_tap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) { const struct clar_error *error = _clar.last_report->errors; (void)test_name; (void)test_number; - switch(status) { + switch(report->status) { case CL_TEST_OK: printf("ok %d - %s::%s\n", test_number, suite_name, test_name); break; @@ -155,7 +171,7 @@ static void clar_print_tap_ontest(const char *suite_name, const char *test_name, if (_clar.verbosity >= 0) { printf(" ---\n"); printf(" reason: |\n"); - clar_print_indented(error->error_msg, 6); + clar_print_indented(error->message, 6); if (error->description) clar_print_indented(error->description, 6); @@ -177,7 +193,7 @@ static void clar_print_tap_ontest(const char *suite_name, const char *test_name, fflush(stdout); } -static void clar_print_tap_onsuite(const char *suite_name, int suite_index) +static void clar_print_tap_suite_start(const char *suite_name, int suite_index) { if (_clar.verbosity < 0) return; @@ -191,6 +207,114 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg) fflush(stdout); } +/* timings format: useful for benchmarks */ + +static void clar_print_timing_init(int test_count, int suite_count) +{ + (void)test_count; + (void)suite_count; + + printf("Started benchmarks (mean time ± stddev / min time … max time):\n\n"); +} + +static void clar_print_timing_shutdown(int test_count, int suite_count, int error_count) +{ + (void)test_count; + (void)suite_count; + (void)error_count; +} + +static void clar_print_timing_error(int num, const struct clar_report *report, const struct clar_error *error) +{ + (void)num; + (void)report; + (void)error; +} + +static void clar_print_timing_test_start(const char *suite_name, const char *test_name, int test_number) +{ + (void)test_number; + + printf("%s::%s: ", suite_name, test_name); + fflush(stdout); +} + +static void clar_print_timing_time(double t) +{ + static const char *units[] = { "sec", "ms", "μs", "ns" }; + static const int units_len = sizeof(units) / sizeof(units[0]); + int unit = 0, exponent = 0, digits; + + while (t < 1.0 && unit < units_len - 1) { + t *= 1000.0; + unit++; + } + + while (t > 0.0 && t < 1.0 && exponent < 10) { + t *= 10.0; + exponent++; + } + + digits = (t < 10.0) ? 3 : ((t < 100.0) ? 2 : 1); + + printf("%.*f", digits, t); + + if (exponent > 0) + printf("e-%d", exponent); + + printf(" %s", units[unit]); +} + +static void clar_print_timing_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) +{ + const struct clar_error *error = _clar.last_report->errors; + + (void)suite_name; + (void)test_name; + (void)test_number; + + switch(report->status) { + case CL_TEST_OK: + clar_print_timing_time(report->time_mean); + + if (report->runs > 1) { + printf(" ± "); + clar_print_timing_time(report->time_stddev); + + printf(" / range: "); + clar_print_timing_time(report->time_min); + printf(" … "); + clar_print_timing_time(report->time_max); + printf(" (%d runs)", report->runs); + } + + printf("\n"); + break; + case CL_TEST_FAILURE: + printf("failed: %s\n", error->message); + break; + case CL_TEST_SKIP: + case CL_TEST_NOTRUN: + printf("skipped\n"); + break; + } + + fflush(stdout); +} + +static void clar_print_timing_suite_start(const char *suite_name, int suite_index) +{ + if (_clar.verbosity == 1) + printf("\n%s", suite_name); + + (void)suite_index; +} + +static void clar_print_timing_onabort(const char *fmt, va_list arg) +{ + vfprintf(stderr, fmt, arg); +} + /* indirection between protocol output selection */ #define PRINT(FN, ...) do { \ @@ -201,6 +325,9 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg) case CL_OUTPUT_TAP: \ clar_print_tap_##FN (__VA_ARGS__); \ break; \ + case CL_OUTPUT_TIMING: \ + clar_print_timing_##FN (__VA_ARGS__); \ + break; \ default: \ abort(); \ } \ @@ -221,14 +348,19 @@ static void clar_print_error(int num, const struct clar_report *report, const st PRINT(error, num, report, error); } -static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number) { - PRINT(ontest, suite_name, test_name, test_number, status); + PRINT(test_start, suite_name, test_name, test_number); } -static void clar_print_onsuite(const char *suite_name, int suite_index) +static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) { - PRINT(onsuite, suite_name, suite_index); + PRINT(test_finish, suite_name, test_name, test_number, report); +} + +static void clar_print_suite_start(const char *suite_name, int suite_index) +{ + PRINT(suite_start, suite_name, suite_index); } static void clar_print_onabortv(const char *msg, va_list argp) diff --git a/deps/clar/clar/summary.h b/deps/clar/clar/summary.h index 7b85f162d..f2be3c13a 100644 --- a/deps/clar/clar/summary.h +++ b/deps/clar/clar/summary.h @@ -1,8 +1,24 @@ - #include #include -static int clar_summary_close_tag( +static int clar_summary_time_digits(double t) +{ + int digits = 3; + + if (t >= 100.0) + return 1; + else if (t >= 10.0) + return 2; + + while (t > 0.0 && t < 1.0 && digits < 10) { + t *= 10.0; + digits++; + } + + return digits; +} + +static int clar_summary_junit_close_tag( struct clar_summary *summary, const char *tag, int indent) { const char *indt; @@ -14,12 +30,12 @@ static int clar_summary_close_tag( return fprintf(summary->fp, "%s\n", indt, tag); } -static int clar_summary_testsuites(struct clar_summary *summary) +static int clar_summary_junit_testsuites(struct clar_summary *summary) { return fprintf(summary->fp, "\n"); } -static int clar_summary_testsuite(struct clar_summary *summary, +static int clar_summary_junit_testsuite(struct clar_summary *summary, int idn, const char *name, time_t timestamp, int test_count, int fail_count, int error_count) { @@ -41,15 +57,15 @@ static int clar_summary_testsuite(struct clar_summary *summary, idn, name, iso_dt, test_count, fail_count, error_count); } -static int clar_summary_testcase(struct clar_summary *summary, +static int clar_summary_junit_testcase(struct clar_summary *summary, const char *name, const char *classname, double elapsed) { return fprintf(summary->fp, - "\t\t\n", - name, classname, elapsed); + "\t\t\n", + name, classname, clar_summary_time_digits(elapsed), elapsed); } -static int clar_summary_failure(struct clar_summary *summary, +static int clar_summary_junit_failure(struct clar_summary *summary, const char *type, const char *message, const char *desc) { return fprintf(summary->fp, @@ -57,22 +73,26 @@ static int clar_summary_failure(struct clar_summary *summary, type, message, desc); } -static int clar_summary_skipped(struct clar_summary *summary) +static int clar_summary_junit_skipped(struct clar_summary *summary) { return fprintf(summary->fp, "\t\t\t\n"); } -struct clar_summary *clar_summary_init(const char *filename) +static struct clar_summary *clar_summary_junit_init(const char *filename) { struct clar_summary *summary; FILE *fp; - if ((fp = fopen(filename, "w")) == NULL) - clar_abort("Failed to open the summary file '%s': %s.\n", - filename, strerror(errno)); + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); + return NULL; + } - if ((summary = malloc(sizeof(struct clar_summary))) == NULL) - clar_abort("Failed to allocate summary.\n"); + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); + fclose(fp); + return NULL; + } summary->filename = filename; summary->fp = fp; @@ -80,12 +100,12 @@ struct clar_summary *clar_summary_init(const char *filename) return summary; } -int clar_summary_shutdown(struct clar_summary *summary) +static int clar_summary_junit_shutdown(struct clar_summary *summary) { struct clar_report *report; const char *last_suite = NULL; - if (clar_summary_testsuites(summary) < 0) + if (clar_summary_junit_testsuites(summary) < 0) goto on_error; report = _clar.reports; @@ -93,38 +113,38 @@ int clar_summary_shutdown(struct clar_summary *summary) struct clar_error *error = report->errors; if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_testsuite(summary, 0, report->suite, + if (clar_summary_junit_testsuite(summary, 0, report->suite, report->start, _clar.tests_ran, _clar.total_errors, 0) < 0) goto on_error; } last_suite = report->suite; - clar_summary_testcase(summary, report->test, report->suite, report->elapsed); + clar_summary_junit_testcase(summary, report->test, report->suite, report->time_total); while (error != NULL) { - if (clar_summary_failure(summary, "assert", - error->error_msg, error->description) < 0) + if (clar_summary_junit_failure(summary, "assert", + error->message, error->description) < 0) goto on_error; error = error->next; } if (report->status == CL_TEST_SKIP) - clar_summary_skipped(summary); + clar_summary_junit_skipped(summary); - if (clar_summary_close_tag(summary, "testcase", 2) < 0) + if (clar_summary_junit_close_tag(summary, "testcase", 2) < 0) goto on_error; report = report->next; if (!report || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_close_tag(summary, "testsuite", 1) < 0) + if (clar_summary_junit_close_tag(summary, "testsuite", 1) < 0) goto on_error; } } - if (clar_summary_close_tag(summary, "testsuites", 0) < 0 || + if (clar_summary_junit_close_tag(summary, "testsuites", 0) < 0 || fclose(summary->fp) != 0) goto on_error; @@ -138,3 +158,154 @@ on_error: free(summary); return -1; } + +static struct clar_summary *clar_summary_json_init(const char *filename) +{ + struct clar_summary *summary; + FILE *fp; + + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); + return NULL; + } + + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); + fclose(fp); + return NULL; + } + + summary->filename = filename; + summary->fp = fp; + + return summary; +} + +static int clar_summary_json_shutdown(struct clar_summary *summary) +{ + struct clar_report *report; + int i; + + fprintf(summary->fp, "{\n"); + fprintf(summary->fp, " \"tests\": [\n"); + + report = _clar.reports; + while (report != NULL) { + struct clar_error *error = report->errors; + + if (report != _clar.reports) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " {\n"); + fprintf(summary->fp, " \"name\": \"%s::%s\",\n", report->suite, report->test); + + if (report->description) + fprintf(summary->fp, " \"description\": \"%s\",\n", report->description); + + fprintf(summary->fp, " \"results\": {\n"); + + fprintf(summary->fp, " \"status\": "); + if (report->status == CL_TEST_OK) + fprintf(summary->fp, "\"ok\",\n"); + else if (report->status == CL_TEST_FAILURE) + fprintf(summary->fp, "\"failed\",\n"); + else if (report->status == CL_TEST_SKIP) + fprintf(summary->fp, "\"skipped\"\n"); + else + clar_abort("unknown test status %d", report->status); + + if (report->status == CL_TEST_OK) { + fprintf(summary->fp, " \"mean\": %.*f,\n", + clar_summary_time_digits(report->time_mean), report->time_mean); + fprintf(summary->fp, " \"stddev\": %.*f,\n", + clar_summary_time_digits(report->time_stddev), report->time_stddev); + fprintf(summary->fp, " \"min\": %.*f,\n", + clar_summary_time_digits(report->time_min), report->time_min); + fprintf(summary->fp, " \"max\": %.*f,\n", + clar_summary_time_digits(report->time_max), report->time_max); + fprintf(summary->fp, " \"times\": [\n"); + + for (i = 0; i < report->runs; i++) { + if (i > 0) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " %.*f", + clar_summary_time_digits(report->times[i]), report->times[i]); + } + + fprintf(summary->fp, "\n ]\n"); + } + + if (report->status == CL_TEST_FAILURE) { + fprintf(summary->fp, " \"errors\": [\n"); + + while (error != NULL) { + if (error != report->errors) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " {\n"); + fprintf(summary->fp, " \"message\": \"%s\",\n", error->message); + + if (error->description) + fprintf(summary->fp, " \"description\": \"%s\",\n", error->description); + + fprintf(summary->fp, " \"function\": \"%s\",\n", error->function); + fprintf(summary->fp, " \"file\": \"%s\",\n", error->file); + fprintf(summary->fp, " \"line\": %" PRIuMAX "\n", error->line_number); + fprintf(summary->fp, " }"); + + error = error->next; + } + + fprintf(summary->fp, "\n"); + fprintf(summary->fp, " ]\n"); + } + + fprintf(summary->fp, " }\n"); + fprintf(summary->fp, " }"); + + report = report->next; + } + + fprintf(summary->fp, "\n"); + fprintf(summary->fp, " ]\n"); + fprintf(summary->fp, "}\n"); + + if (fclose(summary->fp) != 0) + goto on_error; + + printf("written summary file to %s\n", summary->filename); + + free(summary); + return 0; + +on_error: + fclose(summary->fp); + free(summary); + return -1; +} + +/* indirection between protocol output selection */ + +#define SUMMARY(FN, ...) do { \ + switch (_clar.summary_format) { \ + case CL_SUMMARY_JUNIT: \ + return clar_summary_junit_##FN (__VA_ARGS__); \ + break; \ + case CL_SUMMARY_JSON: \ + return clar_summary_json_##FN (__VA_ARGS__); \ + break; \ + default: \ + abort(); \ + } \ + } while(0) + +struct clar_summary *clar_summary_init(const char *filename) +{ + SUMMARY(init, filename); +} + +int clar_summary_shutdown(struct clar_summary *summary) +{ + SUMMARY(shutdown, summary); +} diff --git a/deps/clar/generate.py b/deps/clar/generate.py index 2357b2d6d..2e4eaa0a9 100644 --- a/deps/clar/generate.py +++ b/deps/clar/generate.py @@ -17,8 +17,13 @@ class Module(object): def _render_callback(self, cb): if not cb: - return ' { NULL, NULL }' - return ' { "%s", &%s }' % (cb['short_name'], cb['symbol']) + return ' { NULL, NULL, 0, NULL }' + + return ' { "%s", %s, %d, &%s }' % \ + (cb['short_name'], \ + '"' + cb['description'] + '"' if cb['description'] != None else "NULL", \ + cb['runs'], \ + cb['symbol']) class DeclarationTemplate(Template): def render(self): @@ -27,6 +32,9 @@ class Module(object): for initializer in self.module.initializers: out += "extern %s;\n" % initializer['declaration'] + if self.module.reset: + out += "extern %s;\n" % self.module.reset['declaration'] + if self.module.cleanup: out += "extern %s;\n" % self.module.cleanup['declaration'] @@ -34,7 +42,7 @@ class Module(object): class CallbacksTemplate(Template): def render(self): - out = "static const struct clar_func _clar_cb_%s[] = {\n" % self.module.name + out = "static const struct %s_func _%s_cb_%s[] = {\n" % (self.module.app_name, self.module.app_name, self.module.name) out += ",\n".join(self._render_callback(cb) for cb in self.module.callbacks) out += "\n};\n" return out @@ -58,14 +66,16 @@ class Module(object): { "${clean_name}", ${initialize}, + ${reset}, ${cleanup}, ${cb_ptr}, ${cb_count}, ${enabled} }""" ).substitute( clean_name = name, initialize = self._render_callback(initializer), + reset = self._render_callback(self.module.reset), cleanup = self._render_callback(self.module.cleanup), - cb_ptr = "_clar_cb_%s" % self.module.name, + cb_ptr = "_%s_cb_%s" % (self.module.app_name, self.module.name), cb_count = len(self.module.callbacks), enabled = int(self.module.enabled) ) @@ -73,8 +83,10 @@ class Module(object): return ','.join(templates) - def __init__(self, name): + def __init__(self, name, app_name, prefix): self.name = name + self.app_name = app_name + self.prefix = prefix self.mtime = None self.enabled = True @@ -85,7 +97,7 @@ class Module(object): def _skip_comments(self, text): SKIP_COMMENTS_REGEX = re.compile( - r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', + r'//.*?$|/\*(?!\s*\[clar\]:).*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE) def _replacer(match): @@ -95,24 +107,63 @@ class Module(object): return re.sub(SKIP_COMMENTS_REGEX, _replacer, text) def parse(self, contents): - TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\s*\(\s*void\s*\))\s*\{" + TEST_FUNC_REGEX = r"^(void\s+(%s_%s__(\w+))\s*\(\s*void\s*\))(?:\s*/\*\s*\[clar\]:\s*(.*?)\s*\*/)?\s*\{" contents = self._skip_comments(contents) - regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE) + regex = re.compile(TEST_FUNC_REGEX % (self.prefix, self.name), re.MULTILINE) self.callbacks = [] self.initializers = [] + self.reset = None self.cleanup = None - for (declaration, symbol, short_name) in regex.findall(contents): + for (declaration, symbol, short_name, options) in regex.findall(contents): + runs = 0 + description = None + + while options != '': + match = re.search(r'^([a-zA-Z0-9]+)=(\"[^"]*\"|[a-zA-Z0-9_\-]+|\d+)(?:,\s*|\Z)(.*)', options) + + if match == None: + print("Invalid options: '%s' for '%s'" % (options, symbol)) + sys.exit(1) + + key = match.group(1) + value = match.group(2) + options = match.group(3) + + match = re.search(r'^\"(.*)\"$', value) + if match != None: + value = match.group(1) + + match = re.search(r'([^a-zA-Z0-9 _\-,\.])', value) + if match != None: + print("Invalid character '%s' in %s for '%s'" % (match.group(1), key, symbol)) + sys.exit(1) + + if key == "description": + description = value + elif key == "runs": + if not value.isnumeric(): + print("Invalid option: '%s' in runs for '%s'" % (option, symbol)) + sys.exit(1) + runs = int(value) + else: + print("Invalid option: '%s' for '%s'" % (key, symbol)) + sys.exit(1) + data = { "short_name" : short_name, "declaration" : declaration, - "symbol" : symbol + "symbol" : symbol, + "description" : description, + "runs" : runs } if short_name.startswith('initialize'): self.initializers.append(data) + elif short_name == 'reset': + self.reset = data elif short_name == 'cleanup': self.cleanup = data else: @@ -179,8 +230,8 @@ class TestSuite(object): return modules - def load_cache(self): - path = os.path.join(self.output, '.clarcache') + def load_cache(self, app_name): + path = os.path.join(self.output, ".%scache" % app_name) cache = {} try: @@ -192,18 +243,18 @@ class TestSuite(object): return cache - def save_cache(self): - path = os.path.join(self.output, '.clarcache') + def save_cache(self, app_name): + path = os.path.join(self.output, ".%scache" % app_name) with open(path, 'wb') as cache: pickle.dump(self.modules, cache) - def load(self, force = False): + def load(self, app_name, prefix, force = False): module_data = self.find_modules() - self.modules = {} if force else self.load_cache() + self.modules = {} if force else self.load_cache(app_name) for path, name in module_data: if name not in self.modules: - self.modules[name] = Module(name) + self.modules[name] = Module(name, app_name, prefix) if not self.modules[name].refresh(path): del self.modules[name] @@ -222,15 +273,15 @@ class TestSuite(object): def callback_count(self): return sum(len(module.callbacks) for module in self.modules.values()) - def write(self): + def write(self, name): if not os.path.exists(self.output): os.makedirs(self.output) - wrote_suite = self.write_suite() - wrote_header = self.write_header() + wrote_suite = self.write_suite(name) + wrote_header = self.write_header(name) if wrote_suite or wrote_header: - self.save_cache() + self.save_cache(name) return True return False @@ -257,8 +308,8 @@ class TestSuite(object): return True - def write_suite(self): - suite_fn = os.path.join(self.output, 'clar.suite') + def write_suite(self, name): + suite_fn = os.path.join(self.output, '%s.suite' % name) with io.StringIO() as suite_file: modules = sorted(self.modules.values(), key=lambda module: module.name) @@ -271,25 +322,26 @@ class TestSuite(object): t = Module.CallbacksTemplate(module) suite_file.write(t.render()) - suites = "static struct clar_suite _clar_suites[] = {" + ','.join( + suites = "static struct %s_suite _%s_suites[] = {" % (name, name) + suites += ','.join( Module.InfoTemplate(module).render() for module in modules ) + "\n};\n" suite_file.write(suites) - suite_file.write(u"static const size_t _clar_suite_count = %d;\n" % self.suite_count()) - suite_file.write(u"static const size_t _clar_callback_count = %d;\n" % self.callback_count()) + suite_file.write(u"static const size_t _%s_suite_count = %d;\n" % (name, self.suite_count())) + suite_file.write(u"static const size_t _%s_callback_count = %d;\n" % (name, self.callback_count())) return self.write_output(suite_fn, suite_file.getvalue()) return False - def write_header(self): - header_fn = os.path.join(self.output, 'clar_suite.h') + def write_header(self, name): + header_fn = os.path.join(self.output, '%s_suite.h' % name) with io.StringIO() as header_file: - header_file.write(u"#ifndef _____clar_suite_h_____\n") - header_file.write(u"#define _____clar_suite_h_____\n") + header_file.write(u"#ifndef _____%s_suite_h_____\n" % name) + header_file.write(u"#define _____%s_suite_h_____\n" % name) modules = sorted(self.modules.values(), key=lambda module: module.name) @@ -310,6 +362,8 @@ if __name__ == '__main__': parser.add_option('-f', '--force', action="store_true", dest='force', default=False) parser.add_option('-x', '--exclude', dest='excluded', action='append', default=[]) parser.add_option('-o', '--output', dest='output') + parser.add_option('-n', '--name', dest='name', default='clar') + parser.add_option('-p', '--prefix', dest='prefix', default='test') options, args = parser.parse_args() if len(args) > 1: @@ -323,7 +377,8 @@ if __name__ == '__main__': output = options.output or path suite = TestSuite(path, output) - suite.load(options.force) + suite.load(options.name, options.prefix, options.force) suite.disable(options.excluded) - if suite.write(): - print("Written `clar.suite`, `clar_suite.h` (%d tests in %d suites)" % (suite.callback_count(), suite.suite_count())) + + if suite.write(options.name): + print("Written `%s.suite`, `%s_suite.h` (%d tests in %d suites)" % (options.name, options.name, suite.callback_count(), suite.suite_count())) From 3ac6bf801c850d138b80e0c1740f3b2439314b5d Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 24 Jan 2026 14:16:25 +0000 Subject: [PATCH 4/8] clar: avoid iterator clobbered by longjmp If the iterator is moved to a register, it may be clobbered by the longjmp. Avoid this by treating it as volatile. --- deps/clar/clar.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deps/clar/clar.c b/deps/clar/clar.c index 1a206fcd2..9f627f343 100644 --- a/deps/clar/clar.c +++ b/deps/clar/clar.c @@ -345,6 +345,8 @@ clar_run_test( const struct clar_func *cleanup) { clar_time start, end; + int runs = test->runs; + volatile int i = 0; _clar.last_report->start = time(NULL); _clar.last_report->times = &_clar.last_report->time_mean; From d55ff3cd7020313cfe88091eaadd5cfbeb07f5ba Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sun, 25 Jan 2026 17:02:39 +0000 Subject: [PATCH 5/8] clar: fix remove unused --- deps/clar/clar.c | 1 - 1 file changed, 1 deletion(-) diff --git a/deps/clar/clar.c b/deps/clar/clar.c index 9f627f343..0d1ad2812 100644 --- a/deps/clar/clar.c +++ b/deps/clar/clar.c @@ -344,7 +344,6 @@ clar_run_test( const struct clar_func *reset, const struct clar_func *cleanup) { - clar_time start, end; int runs = test->runs; volatile int i = 0; From 928a81e8728f102adae3c270599537603ea19ff5 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 10 Jan 2026 13:53:35 +0000 Subject: [PATCH 6/8] microbenchmarks: benchmark the oid code At present, the library's oid manipulation functions are slower when built in SHA256 mode than when not. Add some microbenchmarks around the oid compare and copy functions to understand this better. --- CMakeLists.txt | 7 +- benchmarks/CMakeLists.txt | 1 + benchmarks/libgit2/CMakeLists.txt | 78 +++++++++++++++ benchmarks/libgit2/main.c | 31 ++++++ benchmarks/libgit2/oid.c | 151 ++++++++++++++++++++++++++++++ benchmarks/libgit2/precompiled.c | 1 + benchmarks/libgit2/precompiled.h | 2 + tests/libgit2/CMakeLists.txt | 6 +- tests/util/CMakeLists.txt | 6 +- 9 files changed, 280 insertions(+), 3 deletions(-) create mode 100644 benchmarks/CMakeLists.txt create mode 100644 benchmarks/libgit2/CMakeLists.txt create mode 100644 benchmarks/libgit2/main.c create mode 100644 benchmarks/libgit2/oid.c create mode 100644 benchmarks/libgit2/precompiled.c create mode 100644 benchmarks/libgit2/precompiled.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 335901d1f..3e9747575 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,8 @@ option(EXPERIMENTAL_SHA256 "Enable experimental SHA256 support (for R&D/test # Optional subsystems option(BUILD_SHARED_LIBS "Build Shared Library (OFF for Static)" ON) -option(BUILD_TESTS "Build Tests using the Clar suite" ON) +option(BUILD_TESTS "Build the test suite" ON) +option(BUILD_BENCHMARKS "Build the benchmark suite" OFF) option(BUILD_CLI "Build the command-line interface" ON) option(BUILD_EXAMPLES "Build library usage example apps" OFF) option(BUILD_FUZZERS "Build the fuzz targets" OFF) @@ -112,6 +113,10 @@ if(BUILD_TESTS) add_subdirectory(tests) endif() +if(BUILD_BENCHMARKS) + add_subdirectory(benchmarks) +endif() + if(BUILD_EXAMPLES) add_subdirectory(examples) endif() diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt new file mode 100644 index 000000000..1b31354e5 --- /dev/null +++ b/benchmarks/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(libgit2) diff --git a/benchmarks/libgit2/CMakeLists.txt b/benchmarks/libgit2/CMakeLists.txt new file mode 100644 index 000000000..e2668b25e --- /dev/null +++ b/benchmarks/libgit2/CMakeLists.txt @@ -0,0 +1,78 @@ +# util: the unit tests for libgit2's utility library + +if(NOT "${CMAKE_VERSION}" VERSION_LESS 3.27) + cmake_policy(SET CMP0148 OLD) +endif() + +set(Python_ADDITIONAL_VERSIONS 3 2.7) +find_package(PythonInterp) + +if(NOT PYTHONINTERP_FOUND) + message(FATAL_ERROR "Could not find a python interpeter, which is needed to build the tests. " + "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests") +endif() + +set(CLAR_PATH "${PROJECT_SOURCE_DIR}/deps/clar") +set(BENCHMARK_PATH "${CMAKE_CURRENT_SOURCE_DIR}") +add_definitions(-DCLAR_TMPDIR=\"libgit2_bench\") +add_definitions(-DCLAR_WIN32_LONGPATHS) +add_definitions(-DCLAR_HAS_REALPATH) +add_definitions(-D_FILE_OFFSET_BITS=64) + +file(GLOB BENCHMARK_SRC *.c *.h) +list(SORT BENCHMARK_SRC) + +set(CLAR_SRC + "${CLAR_PATH}/clar.c" + "${CLAR_PATH}/clar.h" + "${CLAR_PATH}/clar/fixtures.h" + "${CLAR_PATH}/clar/print.h" + "${CLAR_PATH}/clar/summary.h" + "${CLAR_PATH}/clar/sandbox.h" + "${CLAR_PATH}/clar/fs.h") + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/clar.suite ${CMAKE_CURRENT_BINARY_DIR}/clar_suite.h + COMMAND ${PYTHON_EXECUTABLE} ${CLAR_PATH}/generate.py -p benchmark -o "${CMAKE_CURRENT_BINARY_DIR}" -f . + DEPENDS ${BENCHMARK_SRC} + WORKING_DIRECTORY ${BENCHMARK_PATH} +) + +set_source_files_properties( + ${CLAR_PATH}/clar.c + PROPERTIES OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clar.suite) + +add_executable(libgit2_benchmarks ${CLAR_SRC} + ${BENCHMARK_SRC} + $ + ${LIBGIT2_DEPENDENCY_OBJECTS}) + +target_link_libraries(libgit2_benchmarks libgit2package ${LIBGIT2_SYSTEM_LIBS}) +if(NOT MSVC_IDE) + target_link_libraries(libgit2_benchmarks m) +endif() + +ide_split_sources(libgit2_benchmarks) + +target_include_directories(libgit2_benchmarks PRIVATE + "${CLAR_PATH}" + "${libgit2_BINARY_DIR}/src/util" + "${libgit2_BINARY_DIR}/include" + "${libgit2_SOURCE_DIR}/src/util" + "${libgit2_SOURCE_DIR}/include" + "${CMAKE_CURRENT_BINARY_DIR}" + "${LIBGIT2_DEPENDENCY_INCLUDES}" + "${LIBGIT2_SYSTEM_INCLUDES}") + +# +# Old versions of gcc require us to declare our test functions; don't do +# this on newer compilers to avoid unnecessary recompilation. +# +if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0) + target_compile_options(libgit2_benchmarks PRIVATE -include "clar_suite.h") +endif() + +if(MSVC_IDE) + set_target_properties(libgit2_benchmarks PROPERTIES COMPILE_FLAGS "/Yuprecompiled.h /FIprecompiled.h") + set_source_files_properties("precompiled.c" COMPILE_FLAGS "/Ycprecompiled.h") +endif() diff --git a/benchmarks/libgit2/main.c b/benchmarks/libgit2/main.c new file mode 100644 index 000000000..a8f680f6d --- /dev/null +++ b/benchmarks/libgit2/main.c @@ -0,0 +1,31 @@ +#include + +#include "clar.h" +#include + +#ifdef _WIN32 +int __cdecl main(int argc, char *argv[]) +#else +int main(int argc, char *argv[]) +#endif +{ + int res; + + clar_test_set_mode(CL_TEST_BENCHMARK); + clar_test_init(argc, argv); + + res = git_libgit2_init(); + if (res < 0) { + const git_error *err = git_error_last(); + const char *msg = err ? err->message : "unknown failure"; + fprintf(stderr, "failed to init libgit2: %s\n", msg); + return res; + } + + /* Run the test suite */ + res = clar_test_run(); + + clar_test_shutdown(); + + return res; +} diff --git a/benchmarks/libgit2/oid.c b/benchmarks/libgit2/oid.c new file mode 100644 index 000000000..8e5b157e6 --- /dev/null +++ b/benchmarks/libgit2/oid.c @@ -0,0 +1,151 @@ +#include "clar.h" + +#include +#include +#include + +#include + +#define BENCHMARK_OID_COUNT 256 + +static git_oid sha1_one[BENCHMARK_OID_COUNT]; +static git_oid *sha1_two; + +#ifdef GIT_EXPERIMENTAL_SHA256 +static git_oid sha256_one[BENCHMARK_OID_COUNT]; +static git_oid *sha256_two; +#endif + +static void update_data_to_val(git_oid *out, git_oid_t type, uint32_t val) +{ + unsigned char data[GIT_OID_MAX_SIZE] = {0}; + size_t size; + +#ifdef GIT_EXPERIMENTAL_SHA256 + size = (type == GIT_OID_SHA256) ? GIT_OID_SHA256_SIZE : GIT_OID_SHA1_SIZE; +#else + size = GIT_OID_SHA1_SIZE; + + ((void)(type)); +#endif + + memset(data, 0, GIT_OID_MAX_SIZE); + + data[size - 1] = (unsigned char)(val & 0x000000ff); + data[size - 2] = (unsigned char)((val & 0x0000ff00) >> 8); + data[size - 3] = (unsigned char)((val & 0x00ff0000) >> 16); + data[size - 4] = (unsigned char)((val & 0x00ff0000) >> 24); + +#ifdef GIT_EXPERIMENTAL_SHA256 + cl_assert(git_oid_from_raw(out, data, type) == 0); +#else + cl_assert(git_oid_fromraw(out, data) == 0); +#endif +} + +void benchmark_oid__initialize(void) +{ + uint32_t accum = 0; + size_t i; + + sha1_two = calloc(BENCHMARK_OID_COUNT, sizeof(git_oid)); + cl_assert(sha1_two != NULL); + +#ifdef GIT_EXPERIMENTAL_SHA256 + sha256_two = calloc(BENCHMARK_OID_COUNT, sizeof(git_oid)); + cl_assert(sha256_two != NULL); +#endif + + for (i = 0; i < BENCHMARK_OID_COUNT; i++) { + update_data_to_val(&sha1_one[i], GIT_OID_SHA1, accum++); + update_data_to_val(&sha1_two[i], GIT_OID_SHA1, accum++); + } + +#ifdef GIT_EXPERIMENTAL_SHA256 + for (i = 0; i < BENCHMARK_OID_COUNT; i++) { + update_data_to_val(&sha256_one[i], GIT_OID_SHA256, accum++); + update_data_to_val(&sha256_two[i], GIT_OID_SHA256, accum++); + } +#endif +} + +void benchmark_oid__reset(void) +{ +} + +void benchmark_oid__cleanup(void) +{ + free(sha1_two); + +#ifdef GIT_EXPERIMENTAL_SHA256 + free(sha256_two); +#endif +} + +void benchmark_oid__cmp_sha1(void) +{ + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_cmp(&sha1_one[j], &sha1_two[j]); +} + +void benchmark_oid__cmp_sha256(void) +{ +#ifdef GIT_EXPERIMENTAL_SHA256 + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_cmp(&sha256_one[j], &sha256_two[j]); +#else + clar__skip(); +#endif +} + +void benchmark_oid__cpy_sha1(void) +{ + git_oid dest; + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_cpy(&dest, &sha1_one[j]); +} + +void benchmark_oid__cpy_sha256(void) +{ +#ifdef GIT_EXPERIMENTAL_SHA256 + git_oid dest; + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_cpy(&dest, &sha256_one[j]); +#else + clar__skip(); +#endif +} + +void benchmark_oid__zero_sha1(void) +{ + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_is_zero(&sha1_one[j]); +} + +void benchmark_oid__zero_sha256(void) +{ +#ifdef GIT_EXPERIMENTAL_SHA256 + size_t i, j; + + for (i = 0; i < 1024 * 16; i++) + for (j = 0; j < BENCHMARK_OID_COUNT; j++) + git_oid_is_zero(&sha256_one[j]); +#else + clar__skip(); +#endif +} diff --git a/benchmarks/libgit2/precompiled.c b/benchmarks/libgit2/precompiled.c new file mode 100644 index 000000000..5f656a45d --- /dev/null +++ b/benchmarks/libgit2/precompiled.c @@ -0,0 +1 @@ +#include "precompiled.h" diff --git a/benchmarks/libgit2/precompiled.h b/benchmarks/libgit2/precompiled.h new file mode 100644 index 000000000..e6b34738d --- /dev/null +++ b/benchmarks/libgit2/precompiled.h @@ -0,0 +1,2 @@ +#include "git2.h" +#include "clar.h" diff --git a/tests/libgit2/CMakeLists.txt b/tests/libgit2/CMakeLists.txt index f8b80eb32..681635a35 100644 --- a/tests/libgit2/CMakeLists.txt +++ b/tests/libgit2/CMakeLists.txt @@ -10,7 +10,7 @@ find_package(PythonInterp) if(NOT PYTHONINTERP_FOUND) message(FATAL_ERROR "Could not find a python interpreter, which is needed to build the tests. " "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests") -ENDIF() +endif() set(CLAR_PATH "${PROJECT_SOURCE_DIR}/deps/clar") set(CLAR_FIXTURES "${PROJECT_SOURCE_DIR}/tests/resources/") @@ -48,7 +48,11 @@ add_executable(libgit2_tests ${SRC_CLAR} ${SRC_TEST} ${LIBGIT2_OBJECTS}) set_target_properties(libgit2_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) target_include_directories(libgit2_tests PRIVATE ${TEST_INCLUDES} ${LIBGIT2_INCLUDES} ${LIBGIT2_DEPENDENCY_INCLUDES}) target_include_directories(libgit2_tests SYSTEM PRIVATE ${LIBGIT2_SYSTEM_INCLUDES}) + target_link_libraries(libgit2_tests ${LIBGIT2_SYSTEM_LIBS}) +if(NOT MSVC_IDE) + target_link_libraries(libgit2_tests m) +endif() ide_split_sources(libgit2_tests) diff --git a/tests/util/CMakeLists.txt b/tests/util/CMakeLists.txt index 6b44b85fa..ba861aa1c 100644 --- a/tests/util/CMakeLists.txt +++ b/tests/util/CMakeLists.txt @@ -10,7 +10,7 @@ find_package(PythonInterp) if(NOT PYTHONINTERP_FOUND) message(FATAL_ERROR "Could not find a python interpeter, which is needed to build the tests. " "Make sure python is available, or pass -DBUILD_TESTS=OFF to skip building the tests") -ENDIF() +endif() set(CLAR_PATH "${libgit2_SOURCE_DIR}/deps/clar") set(CLAR_FIXTURES "${libgit2_SOURCE_DIR}/tests/resources/") @@ -48,7 +48,11 @@ set_target_properties(util_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${libgit2_B target_include_directories(util_tests PRIVATE ${TEST_INCLUDES} ${LIBGIT2_INCLUDES} ${LIBGIT2_DEPENDENCY_INCLUDES}) target_include_directories(util_tests SYSTEM PRIVATE ${LIBGIT2_SYSTEM_INCLUDES}) + target_link_libraries(util_tests ${LIBGIT2_SYSTEM_LIBS}) +if(NOT MSVC_IDE) + target_link_libraries(util_tests m) +endif() ide_split_sources(util_tests) From c0f5c9bd803da2431f7ef82d0b9dba8c45d0ce75 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 26 Jan 2026 21:51:54 +0000 Subject: [PATCH 7/8] ci: generate markdown results from benchmarks --- ci/compare-benchmarks.js | 99 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100755 ci/compare-benchmarks.js diff --git a/ci/compare-benchmarks.js b/ci/compare-benchmarks.js new file mode 100755 index 000000000..7646ed975 --- /dev/null +++ b/ci/compare-benchmarks.js @@ -0,0 +1,99 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const process = require('node:process'); + +const path = process.argv[2] || '.'; +const output = process.argv[3]; + +const dirs = fs.readdirSync(path); + +tests = [ ]; +results = { }; + +for (const dir of dirs) { + const name = dir.replace(/^results-/, ''); + + const control = JSON.parse(fs.readFileSync(`${path}/${dir}/control.json`)); + const candidate = JSON.parse(fs.readFileSync(`${path}/${dir}/candidate.json`)); + + results[name] = { }; + + const controlTests = control.tests.map((x) => x.name); + const candidateTests = candidate.tests.map((x) => x.name); + tests = [...new Set(tests.concat(controlTests).concat(candidateTests))]; + + for (const test of tests) { + const controlResults = control.tests.filter((x) => x.name === test)[0]; + const candidateResults = candidate.tests.filter((x) => x.name === test)[0]; + + let result; + + if (controlResults.results.status === 'ok' && + candidateResults.results.status === 'ok') { + const delta = (candidateResults.results.mean - controlResults.results.mean) / candidateResults.results.mean; + const deltaPercent = Math.round(Math.abs(delta) * 1000) / 10; + const direction = (delta > 0) ? 'slower' : 'faster'; + let highlight = ''; + + if (delta > 0.1) { + highlight = '**'; + } + + results[name][test] = `${highlight}${deltaPercent}% ${direction}${highlight}`; + } + else if (controlResults.results.status === 'ok') { + results[name][test] = `${candidateResults.results.stauts} in candidate`; + } + else if (candidateResults.results.status === 'ok') { + results[name][test] = `${controlResults.results.stauts} in control`; + } + else { + results[name][test] = controlResults.results.status; + } + } +} + +let markdown = '| Platform \\ Test |'; + +for (const test in Object.values(results)[0]) { + markdown += ` ${test} |`; +} +markdown += '\n'; + +markdown += `| --- |`; +for (const test in Object.values(results)[0]) { + markdown += ` --- |`; +} +markdown += '\n'; + +for (const name in results) { + markdown += `| ${name.replaceAll(/-/g, '‑')} |`; + + for (const test in results[name]) { + markdown += ` ${results[name][test]} |`; + } + + markdown += `\n`; +} + +if (output) { + fs.writeFileSync(output, markdown); +} +else { + console.log(markdown); +} + +function p(arr, q) { + const array = [...arr].sort(); + const pos = (array.length - 1) * q; + const base = Math.floor(pos); + const rest = pos - base; + + if (array[base + 1] !== undefined) { + return array[base] + rest * (array[base + 1] - array[base]); + } + else { + return array[base]; + } +} From 9935da4661bb7cf06ca3503a8f9dd08838b2543f Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 19 Jan 2026 13:09:36 +0000 Subject: [PATCH 8/8] ci: optionally run benchmarks --- .github/workflows/ab-perf.yml | 229 ++++++++++++++++++++++ .github/workflows/main.yml | 346 ++++++++++++++++------------------ 2 files changed, 388 insertions(+), 187 deletions(-) create mode 100644 .github/workflows/ab-perf.yml diff --git a/.github/workflows/ab-perf.yml b/.github/workflows/ab-perf.yml new file mode 100644 index 000000000..72a3eac3c --- /dev/null +++ b/.github/workflows/ab-perf.yml @@ -0,0 +1,229 @@ +# A/B testing with benchmarks to compare a control branch (main) to a +# candidate branch (a pull request). +name: A/B Performance Test + +on: + workflow_dispatch: + +env: + docker-registry: ghcr.io + docker-config-path: ci/docker + +jobs: + # Run our CI/CD builds. We build a matrix with the various build targets + # and their details. Then we build either in a docker container (Linux) + # or on the actual hosts (macOS, Windows). + build: + strategy: + fail-fast: false + matrix: + platform: + # All builds: core platforms + - name: "Linux (Noble, GCC, OpenSSL, libssh2)" + id: noble-gcc-openssl + os: ubuntu-latest + container: + name: noble + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)" + id: noble-clang-mbedtls + os: ubuntu-latest + container: + name: noble + env: + CC: clang + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + CMAKE_GENERATOR: Ninja + - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" + id: xenial-gcc-openssl + os: ubuntu-latest + container: + name: xenial + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" + id: xenial-gcc-mbedtls + os: ubuntu-latest + container: + name: xenial + env: + CC: clang + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "macOS" + id: macos + os: macos-14 + setup-script: osx + env: + CC: clang + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + CMAKE_GENERATOR: Ninja + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, Visual Studio, Schannel)" + id: windows-amd64-vs + os: windows-2022 + setup-script: win32 + build_prefix: RelWithDebInfo + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, Visual Studio, WinHTTP)" + id: windows-x86-vs + os: windows-2022 + setup-script: win32 + build_prefix: RelWithDebInfo + env: + ARCH: x86 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, mingw, WinHTTP)" + id: windows-amd64-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: amd64 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, mingw, Schannel)" + id: windows-x86-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: x86 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + env: ${{ matrix.platform.env }} + runs-on: ${{ matrix.platform.os }} + name: "A/B: ${{ matrix.platform.name }}" + steps: + - name: Check out control + uses: actions/checkout@v4 + with: + path: source/control + ref: main + - name: Check out candidate + uses: actions/checkout@v4 + with: + path: source/candidate + - name: Set up build environment + run: source/candidate/ci/setup-${{ matrix.platform.setup-script }}-build.sh + shell: bash + if: matrix.platform.setup-script != '' + - name: Setup QEMU + run: docker run --rm --privileged multiarch/qemu-user-static:register --reset + if: matrix.platform.container.qemu == true + - name: Set up container + uses: ./source/candidate/.github/actions/download-or-build-container + with: + registry: ${{ env.docker-registry }} + config-path: ${{ env.docker-config-path }} + container: ${{ matrix.platform.container.name }} + github_token: ${{ secrets.github_token }} + dockerfile: ${{ matrix.platform.container.dockerfile }} + if: matrix.platform.container.name != '' + - name: Prepare builds + run: | + mkdir build + mkdir build/control + mkdir build/candidate + - name: Build control + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ../../source/control/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Build candidate + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ../../source/candidate/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run control benchmarks + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run candidate benchmarks + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Organize results + run: | + mkdir results + mv ${BUILD_WORKSPACE:-.}/build/control/results.json ./results/control.json + mv ${BUILD_WORKSPACE:-.}/build/candidate/results.json ./results/candidate.json + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.platform.id }} + path: results + if: always() + + # Publish the results + publish: + name: Publish results + needs: [ build ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Download test results + uses: actions/download-artifact@v4 + with: + path: results + - name: Generate markdown + run: node ci/compare-benchmarks.js results results.md + - name: Update pull request + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const markdown = fs.readFileSync('results.md'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: markdown + }); diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4ca339806..b6ad39d06 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,28 +1,21 @@ -# Continuous integration and pull request validation builds for the -# main and maintenance branches. -name: CI Build +# A/B testing with benchmarks to compare a control branch (main) to a +# candidate branch (a pull request). +name: A/B Performance Test on: - push: - branches: [ main, maint/* ] - pull_request: - branches: [ main, maint/* ] workflow_dispatch: env: docker-registry: ghcr.io docker-config-path: ci/docker -permissions: - contents: write - packages: write - jobs: # Run our CI/CD builds. We build a matrix with the various build targets # and their details. Then we build either in a docker container (Linux) # or on the actual hosts (macOS, Windows). build: strategy: + fail-fast: false matrix: platform: # All builds: core platforms @@ -34,7 +27,8 @@ jobs: env: CC: gcc CMAKE_GENERATOR: Ninja - CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)" id: noble-clang-mbedtls os: ubuntu-latest @@ -42,167 +36,118 @@ jobs: name: noble env: CC: clang - CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo CMAKE_GENERATOR: Ninja - - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" - id: xenial-gcc-openssl - os: ubuntu-latest - container: - name: xenial - env: - CC: gcc - CMAKE_GENERATOR: Ninja - CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON - - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" - id: xenial-gcc-mbedtls - os: ubuntu-latest - container: - name: xenial - env: - CC: clang - CMAKE_GENERATOR: Ninja - CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 +# - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" +# id: xenial-gcc-openssl +# os: ubuntu-latest +# container: +# name: xenial +# env: +# CC: gcc +# CMAKE_GENERATOR: Ninja +# CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" +# id: xenial-gcc-mbedtls +# os: ubuntu-latest +# container: +# name: xenial +# env: +# CC: clang +# CMAKE_GENERATOR: Ninja +# CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo - name: "macOS" id: macos os: macos-14 setup-script: osx env: CC: clang - CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo CMAKE_GENERATOR: Ninja PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig SKIP_SSH_TESTS: true SKIP_NEGOTIATE_TESTS: true - - name: "Windows (amd64, Visual Studio, Schannel)" - id: windows-amd64-vs - os: windows-2022 - setup-script: win32 - env: - ARCH: amd64 - CMAKE_GENERATOR: Visual Studio 17 2022 - CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 - BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin - BUILD_TEMP: D:\Temp - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - name: "Windows (x86, Visual Studio, WinHTTP)" - id: windows-x86-vs - os: windows-2022 - setup-script: win32 - env: - ARCH: x86 - CMAKE_GENERATOR: Visual Studio 17 2022 - CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 - BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin - BUILD_TEMP: D:\Temp - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - name: "Windows (amd64, mingw, WinHTTP)" - id: windows-amd64-mingw - os: windows-2022 - setup-script: mingw - env: - ARCH: amd64 - CMAKE_GENERATOR: MinGW Makefiles - CMAKE_OPTIONS: -DDEPRECATE_HARD=ON - BUILD_TEMP: D:\Temp - BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - name: "Windows (x86, mingw, Schannel)" - id: windows-x86-mingw - os: windows-2022 - setup-script: mingw - env: - ARCH: x86 - CMAKE_GENERATOR: MinGW Makefiles - CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel - BUILD_TEMP: D:\Temp - BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - - # All builds: sanitizers - - name: "Sanitizer (Memory)" - id: sanitizer-memory - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=memory -fsanitize-memory-track-origins=2 -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_C_EXTENSIONS=ON -DCMAKE_PREFIX_PATH=/usr/local/msan -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - - name: "Sanitizer (Address)" - id: sanitizer-address - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=address -ggdb -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - - name: "Sanitizer (UndefinedBehavior)" - id: sanitizer-ub - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=undefined,nullability -fno-sanitize-recover=undefined,nullability -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - - name: "Sanitizer (Thread)" - id: sanitizer-thread - os: ubuntu-latest - setup-script: sanitizer - container: - name: noble - env: - CC: clang - CFLAGS: -fsanitize=thread -fno-optimize-sibling-calls -fno-omit-frame-pointer - CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON - CMAKE_GENERATOR: Ninja - SKIP_SSH_TESTS: true - SKIP_NEGOTIATE_TESTS: true - ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 - UBSAN_OPTIONS: print_stacktrace=1 - TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1 - fail-fast: false +# - name: "Windows (amd64, Visual Studio, Schannel)" +# id: windows-amd64-vs +# os: windows-2022 +# setup-script: win32 +# build_prefix: RelWithDebInfo +# env: +# ARCH: amd64 +# CMAKE_GENERATOR: Visual Studio 17 2022 +# CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin +# BUILD_TEMP: D:\Temp +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true +# - name: "Windows (x86, Visual Studio, WinHTTP)" +# id: windows-x86-vs +# os: windows-2022 +# setup-script: win32 +# build_prefix: RelWithDebInfo +# env: +# ARCH: x86 +# CMAKE_GENERATOR: Visual Studio 17 2022 +# CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin +# BUILD_TEMP: D:\Temp +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true +# - name: "Windows (amd64, mingw, WinHTTP)" +# id: windows-amd64-mingw +# os: windows-2022 +# setup-script: mingw +# env: +# ARCH: amd64 +# CMAKE_GENERATOR: MinGW Makefiles +# CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_TEMP: D:\Temp +# BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true +# - name: "Windows (x86, mingw, Schannel)" +# id: windows-x86-mingw +# os: windows-2022 +# setup-script: mingw +# env: +# ARCH: x86 +# CMAKE_GENERATOR: MinGW Makefiles +# CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +# CMAKE_BUILD_OPTIONS: --config RelWithDebInfo +# BUILD_TEMP: D:\Temp +# BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin +# SKIP_SSH_TESTS: true +# SKIP_NEGOTIATE_TESTS: true env: ${{ matrix.platform.env }} runs-on: ${{ matrix.platform.os }} - name: "Build: ${{ matrix.platform.name }}" + name: "A/B: ${{ matrix.platform.name }}" steps: - - name: Check out repository + - name: Check out control uses: actions/checkout@v4 with: - path: source - fetch-depth: 0 + path: source/control +# TODO: there are no control benchmarks in main yet +# ref: main + - name: Check out candidate + uses: actions/checkout@v4 + with: + path: source/candidate - name: Set up build environment - run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh + run: source/candidate/ci/setup-${{ matrix.platform.setup-script }}-build.sh shell: bash if: matrix.platform.setup-script != '' - name: Setup QEMU run: docker run --rm --privileged multiarch/qemu-user-static:register --reset if: matrix.platform.container.qemu == true - name: Set up container - uses: ./source/.github/actions/download-or-build-container + uses: ./source/candidate/.github/actions/download-or-build-container with: registry: ${{ env.docker-registry }} config-path: ${{ env.docker-config-path }} @@ -210,49 +155,76 @@ jobs: github_token: ${{ secrets.github_token }} dockerfile: ${{ matrix.platform.container.dockerfile }} if: matrix.platform.container.name != '' - - name: Prepare build - run: mkdir build - - name: Build - uses: ./source/.github/actions/run-build - with: - command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/build.sh - container: ${{ matrix.platform.container.name }} - container-version: ${{ env.docker-registry-container-sha }} - shell: ${{ matrix.platform.shell }} - - name: Test - uses: ./source/.github/actions/run-build - with: - command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/test.sh - container: ${{ matrix.platform.container.name }} - container-version: ${{ env.docker-registry-container-sha }} - shell: ${{ matrix.platform.shell }} - - name: Upload test results - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: test-results-${{ matrix.platform.id }} - path: build/results_*.xml - - documentation: - name: Validate documentation - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v4 - - name: Validate documentation + - name: Prepare builds run: | - (cd script/api-docs && npm install) - script/api-docs/api-generator.js --validate-only --strict --deprecate-hard . + mkdir build + mkdir build/control + mkdir build/candidate + - name: Build control + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ../../source/control/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Build candidate + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ../../source/candidate/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run control benchmarks + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run candidate benchmarks + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Organize results + run: | + mkdir results + mv ${BUILD_WORKSPACE:-.}/build/control/results.json ./results/control.json + mv ${BUILD_WORKSPACE:-.}/build/candidate/results.json ./results/candidate.json + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.platform.id }} + path: results + if: always() - test_results: - name: Test results + # Publish the results + publish: + name: Publish results needs: [ build ] if: always() runs-on: ubuntu-latest steps: + - name: Check out repository + uses: actions/checkout@v4 - name: Download test results uses: actions/download-artifact@v4 - - name: Generate test summary - uses: test-summary/action@v2 with: - paths: 'test-results-*/*.xml' + path: results + - name: Generate markdown + run: node ci/compare-benchmarks.js results results.md + - name: Update pull request + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const markdown = fs.readFileSync('results.md'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: markdown + });