Merge pull request #7043 from libgit2/ethomson/benchmark

Benchmarks: refactoring

.github/workflows/benchmark.yml

@@ -117,6 +117,7 @@ jobs:
         mkdir benchmark && cd benchmark
         ../source/tests/benchmarks/benchmark.sh \
             ${SUITE_FLAG} ${DEBUG_FLAG} \
+            --admin \
             --baseline-cli "git" --cli "${GIT2_CLI}" --name libgit2 \
             --json benchmarks.json --flamegraph --zip benchmarks.zip
       shell: bash

tests/benchmarks/benchmark.sh

@@ -6,7 +6,7 @@ set -eo pipefail
 # parse the command line
 #
 
-usage() { echo "usage: $(basename "$0") [--cli <path>] [--name <cli-name>] [--baseline-cli <path>] [--suite <suite>] [--json <path>] [--flamegraph] [--zip <path>] [--verbose] [--debug]"; }
+usage() { echo "usage: $(basename "$0") [--cli <path>] [--name <cli-name>] [--baseline-cli <path>] [--suite <suite>] [--admin] [--json <path>] [--flamegraph] [--zip <path>] [--verbose] [--debug]"; }
 
 TEST_CLI="git"
 TEST_CLI_NAME=
@@ -16,6 +16,7 @@ JSON_RESULT=
 FLAMEGRAPH=
 ZIP_RESULT=
 OUTPUT_DIR=
+ADMIN=
 VERBOSE=
 DEBUG=
 NEXT=
@@ -58,6 +59,8 @@ for a in "$@"; do
 		NEXT="suite"
 	elif [[ "${a}" == "-s"* ]]; then
 		SUITE="${a/-s/}"
+	elif [ "${a}" == "--admin" ]; then
+		ADMIN=1
 	elif [ "${a}" = "-v" ] || [ "${a}" == "--verbose" ]; then
 		VERBOSE=1
 	elif [ "${a}" == "--debug" ]; then
@@ -223,6 +226,10 @@ for TEST_PATH in "${BENCHMARK_DIR}"/*; do
 		SHOW_OUTPUT="--show-output"
 	fi
 
+	if [ "${ADMIN}" = "1" ]; then
+		ALLOW_ADMIN="--admin"
+	fi
+
 	OUTPUT_FILE="${OUTPUT_DIR}/${TEST_FILE}.out"
 	ERROR_FILE="${OUTPUT_DIR}/${TEST_FILE}.err"
 	JSON_FILE="${OUTPUT_DIR}/${TEST_FILE}.json"
@@ -230,7 +237,7 @@ for TEST_PATH in "${BENCHMARK_DIR}"/*; do
 
 	FAILED=
 	{
-	  ${TEST_PATH} --cli "${TEST_CLI}" --baseline-cli "${BASELINE_CLI}" --json "${JSON_FILE}" ${SHOW_OUTPUT} >"${OUTPUT_FILE}" 2>"${ERROR_FILE}";
+	  ${TEST_PATH} --cli "${TEST_CLI}" --baseline-cli "${BASELINE_CLI}" --json "${JSON_FILE}" ${ALLOW_ADMIN} ${SHOW_OUTPUT} >"${OUTPUT_FILE}" 2>"${ERROR_FILE}";
 	  FAILED=$?
 	} || true
 
@@ -311,6 +318,8 @@ for TEST_PATH in "${BENCHMARK_DIR}"/*; do
 				ANY_FAILED=1
 			fi
 		fi
+	else
+		echo ""
 	fi
 done
 

tests/benchmarks/benchmark_helpers.sh

@@ -7,11 +7,12 @@ set -eo pipefail
 # command-line parsing
 #
 
-usage() { echo "usage: $(basename "$0") [--cli <path>] [--baseline-cli <path>] [--output-style <style>] [--json <path>] [--profile] [--flamegraph <path>]"; }
+usage() { echo "usage: $(basename "$0") [--cli <path>] [--baseline-cli <path>] [--admin] [--output-style <style>] [--json <path>] [--show-output] [--output-style <style>] [--profile] [--flamegraph <path>]"; }
 
 NEXT=
 BASELINE_CLI=
 TEST_CLI="git"
+ADMIN=
 SHOW_OUTPUT=
 JSON=
 PROFILE=
@@ -69,6 +70,8 @@ for a in "$@"; do
                 JSON="${a/-j/}"
 	elif [ "${a}" = "-p" ] || [ "${a}" = "--profile" ]; then
 		PROFILE=1
+	elif [ "${a}" = "--admin" ]; then
+		ADMIN=1
 	elif [ "${a}" = "-F" ] || [ "${a}" = "--flamegraph" ]; then
 		NEXT="flamegraph"
 	elif [[ "${a}" == "-F"* ]]; then
@@ -335,6 +338,9 @@ parse_arguments() {
 	# this test should run `n` warmups
 	WARMUP=0
 
+	# this test requires administrative privileges
+	REQUIRES_ADMIN=0
+
 	if [ "$*" = "" ]; then
 		gitbench_usage 1>&2
 		exit 1
@@ -356,6 +362,8 @@ parse_arguments() {
 			NEXT="prepare"
 		elif [ "${a}" = "--chdir" ]; then
 			NEXT="chdir"
+		elif [ "${a}" = "--requires-admin" ]; then
+			REQUIRES_ADMIN=1
 		elif [[ "${a}" == "--" ]]; then
 			shift
 			break
@@ -379,6 +387,7 @@ parse_arguments() {
 	echo "PREPARE=\"${PREPARE}\""
 	echo "CHDIR=\"${CHDIR}\""
 	echo "WARMUP=\"${WARMUP}\""
+	echo "REQUIRES_ADMIN=\"${REQUIRES_ADMIN}\""
 
 	echo -n "GIT_ARGUMENTS=("
 
@@ -388,7 +397,7 @@ parse_arguments() {
 	echo " )"
 }
 
-gitbench_usage() { echo "usage: gitbench command..."; }
+gitbench_usage() { echo "usage: gitbench [--requires-admin] [--warmup <count>] [--prepare <command>] [--chdir <directory>] command..."; }
 
 exec_profiler() {
 	if [ "${BASELINE_CLI}" != "" ]; then
@@ -490,7 +499,8 @@ exec_hyperfine() {
 # invocation of hyperfine.
 #
 gitbench() {
-	eval $(parse_arguments "$@")
+	argument_data=$(parse_arguments "$@")
+	eval ${argument_data}
 
 	# sanity check
 
@@ -515,6 +525,17 @@ gitbench() {
 #	rm -rf "${SANDBOX_DIR:?}"
 }
 
+# helper script to ensure that --admin is specified
+needs_admin() {
+	if [ "${ADMIN}" != "1" ]; then
+		echo "$0: skipping administrator test" 1>&2
+		echo "" 1>&2
+		echo "This benchmark needs administrative (root) privileges. To run it," 1>&2
+		echo "specify '--admin' to the benchmark." 1>&2
+		exit 2
+	fi
+}
+
 # helper script to give useful error messages about configuration
 needs_repo() {
 	REPO="${1}"

tests/benchmarks/hash-object__text_100kb

@@ -2,6 +2,6 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
-gitbench --prepare "create_text_file text_100kb 102400 &&
-                    flush_disk_cache" \
+gitbench --prepare "create_text_file text_100kb 102400" \
+         --warmup 5 \
          hash-object "text_100kb"

tests/benchmarks/hash-object__text_10mb

@@ -2,6 +2,6 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
-gitbench --prepare "create_text_file text_10mb 10485760 &&
-                    flush_disk_cache" \
+gitbench --prepare "create_text_file text_10mb 10485760" \
+         --warmup 5 \
          hash-object "text_10mb"

tests/benchmarks/hash-object__text_1kb

@@ -2,6 +2,6 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
-gitbench --prepare "create_text_file text_1kb 1024 &&
-                    flush_disk_cache" \
+gitbench --prepare "create_text_file text_1kb 1024" \
+         --warmup 5 \
          hash-object "text_1kb"

tests/benchmarks/hash-object__text_cached_100kb

@@ -1,7 +0,0 @@
-#!/bin/bash -e
-
-. "$(dirname "$0")/benchmark_helpers.sh"
-
-gitbench --prepare "create_text_file text_100kb 102400" \
-         --warmup 5 \
-         hash-object "text_100kb"

tests/benchmarks/hash-object__text_cached_10mb

@@ -1,7 +0,0 @@
-#!/bin/bash -e
-
-. "$(dirname "$0")/benchmark_helpers.sh"
-
-gitbench --prepare "create_text_file text_10mb 10485760" \
-         --warmup 5 \
-         hash-object "text_10mb"

tests/benchmarks/hash-object__text_cached_1kb

@@ -1,7 +0,0 @@
-#!/bin/bash -e
-
-. "$(dirname "$0")/benchmark_helpers.sh"
-
-gitbench --prepare "create_text_file text_1kb 1024" \
-         --warmup 5 \
-         hash-object "text_1kb"

tests/benchmarks/hash-object__text_nocache_100kb

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+. "$(dirname "$0")/benchmark_helpers.sh"
+
+needs_admin
+
+gitbench --prepare "create_text_file text_100kb 102400 &&
+                    flush_disk_cache" \
+         hash-object "text_100kb"

tests/benchmarks/hash-object__text_nocache_10mb

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+. "$(dirname "$0")/benchmark_helpers.sh"
+
+needs_admin
+
+gitbench --prepare "create_text_file text_10mb 10485760 &&
+                    flush_disk_cache" \
+         hash-object "text_10mb"

tests/benchmarks/hash-object__text_nocache_1kb

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+. "$(dirname "$0")/benchmark_helpers.sh"
+
+needs_admin
+
+gitbench --prepare "create_text_file text_1kb 1024 &&
+                    flush_disk_cache" \
+         hash-object "text_1kb"

tests/benchmarks/hash-object__write_text_100kb

@@ -3,7 +3,7 @@
 . "$(dirname "$0")/benchmark_helpers.sh"
 
 gitbench --prepare "sandbox_repo empty_standard_repo &&
-                    create_text_file text_100kb 102400 &&
-                    flush_disk_cache" \
+                    create_text_file text_100kb 102400" \
+         --warmup 5 \
          --chdir "empty_standard_repo" \
-         hash-object -w "../text_100kb"
+         hash-object "../text_100kb"

tests/benchmarks/hash-object__write_text_10mb

@@ -3,7 +3,7 @@
 . "$(dirname "$0")/benchmark_helpers.sh"
 
 gitbench --prepare "sandbox_repo empty_standard_repo &&
-                    create_text_file text_10mb 10485760 &&
-                    flush_disk_cache" \
+                    create_text_file text_10mb 10485760" \
+         --warmup 5 \
          --chdir "empty_standard_repo" \
          hash-object "../text_10mb"

tests/benchmarks/hash-object__write_text_1kb

@@ -3,7 +3,7 @@
 . "$(dirname "$0")/benchmark_helpers.sh"
 
 gitbench --prepare "sandbox_repo empty_standard_repo &&
-                    create_text_file text_1kb 1024 &&
-                    flush_disk_cache" \
+                    create_text_file text_1kb 1024" \
+         --warmup 5 \
          --chdir "empty_standard_repo" \
          hash-object "../text_1kb"

tests/benchmarks/hash-object__write_text_nocache_100kb

@@ -2,8 +2,10 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
+needs_admin
+
 gitbench --prepare "sandbox_repo empty_standard_repo &&
-                    create_text_file text_100kb 102400" \
-         --warmup 5 \
+                    create_text_file text_100kb 102400 &&
+                    flush_disk_cache" \
          --chdir "empty_standard_repo" \
-         hash-object "../text_100kb"
+         hash-object -w "../text_100kb"

tests/benchmarks/hash-object__write_text_nocache_10mb

@@ -2,8 +2,10 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
+needs_admin
+
 gitbench --prepare "sandbox_repo empty_standard_repo &&
-                    create_text_file text_10mb 10485760" \
-         --warmup 5 \
+                    create_text_file text_10mb 10485760 &&
+                    flush_disk_cache" \
          --chdir "empty_standard_repo" \
          hash-object "../text_10mb"

tests/benchmarks/hash-object__write_text_nocache_1kb

@@ -2,8 +2,10 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
+needs_admin
+
 gitbench --prepare "sandbox_repo empty_standard_repo &&
-                    create_text_file text_1kb 1024" \
-         --warmup 5 \
+                    create_text_file text_1kb 1024 &&
+                    flush_disk_cache" \
          --chdir "empty_standard_repo" \
          hash-object "../text_1kb"