Merge pull request #5569 from lhchavez/ci-sanitizers

Add CI support for Memory and UndefinedBehavior Sanitizers

.github/workflows/main.yml

@@ -28,6 +28,7 @@ jobs:
         container:
         - xenial
         - bionic
+        - focal
         - docurium
     runs-on: ubuntu-latest
     steps:
@@ -86,6 +87,26 @@ jobs:
             CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON
             CMAKE_GENERATOR: Ninja
           os: ubuntu-latest
+        - # Focal, Clang 10, mbedTLS, MemorySanitizer
+          image: focal
+          env:
+            CC: clang-10
+            CFLAGS: -fsanitize=memory -fsanitize-memory-track-origins=2 -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer
+            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local/msan -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON
+            CMAKE_GENERATOR: Ninja
+            SKIP_SSH_TESTS: true
+            ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
+          os: ubuntu-latest
+        - # Focal, Clang 10, OpenSSL, UndefinedBehaviorSanitizer
+          image: focal
+          env:
+            CC: clang-10
+            CFLAGS: -fsanitize=undefined,nullability -fno-sanitize-recover=undefined,nullability -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer
+            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON
+            CMAKE_GENERATOR: Ninja
+            SKIP_SSH_TESTS: true
+            ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
+          os: ubuntu-latest
         - # macOS
           os: macos-10.15
           env:
@@ -161,7 +182,21 @@ jobs:
         export GITTEST_NEGOTIATE_PASSWORD="${{ secrets.GITTEST_NEGOTIATE_PASSWORD }}"
 
         if [ -n "${{ matrix.platform.image }}" ]; then
-          docker run -v $(pwd):/home/libgit2/source -w /home/libgit2/source -e CC -e CMAKE_GENERATOR -e CMAKE_OPTIONS -e PKG_CONFIG_PATH -e GITTEST_NEGOTIATE_PASSWORD -e SKIP_SSH_TESTS -e SKIP_NEGOTIATE_TESTS ${{ env.docker-registry-container-sha }} /bin/bash -c "mkdir build && cd build && ../azure-pipelines/build.sh && ../azure-pipelines/test.sh"
+          docker run \
+              --rm \
+              -v "$(pwd):/home/libgit2/source" \
+              -w /home/libgit2/source \
+              -e ASAN_SYMBOLIZER_PATH \
+              -e CC \
+              -e CFLAGS \
+              -e CMAKE_GENERATOR \
+              -e CMAKE_OPTIONS \
+              -e GITTEST_NEGOTIATE_PASSWORD \
+              -e PKG_CONFIG_PATH \
+              -e SKIP_NEGOTIATE_TESTS \
+              -e SKIP_SSH_TESTS \
+              ${{ env.docker-registry-container-sha }} \
+              /bin/bash -c "mkdir build && cd build && ../azure-pipelines/build.sh && ../azure-pipelines/test.sh"
         else
           mkdir build && cd build
           ../azure-pipelines/build.sh
@@ -189,7 +224,12 @@ jobs:
         git config user.email 'libgit2@users.noreply.github.com'
         git branch gh-pages origin/gh-pages
         docker login https://${{ env.docker-registry }} -u ${{ github.actor }} -p ${{ github.token }}
-        docker run --rm -v $(pwd):/home/libgit2/source -w /home/libgit2/source ${{ env.docker-registry }}/${{ github.repository }}/docurium:latest cm doc api.docurium
+        docker run \
+            --rm \
+            -v "$(pwd):/home/libgit2/source" \
+            -w /home/libgit2/source \
+            ${{ env.docker-registry }}/${{ github.repository }}/docurium:latest \
+            cm doc api.docurium
         git checkout gh-pages
         zip --exclude .git/\* --exclude .gitignore --exclude .gitattributes -r api-documentation.zip .
     - uses: actions/upload-artifact@v2

azure-pipelines/build.sh

@@ -37,11 +37,18 @@ echo "Kernel version:"
 uname -a 2>&1 | indent
 
 echo "CMake version:"
-env PATH="$BUILD_PATH" "$CMAKE" --version 2>&1 | indent
+env PATH="${BUILD_PATH}" "${CMAKE}" --version 2>&1 | indent
 
-if test -n "$CC"; then
+if test -n "${CC}"; then
 	echo "Compiler version:"
-	"$CC" --version 2>&1 | indent
+	"${CC}" --version 2>&1 | indent
+fi
+echo "Environment:"
+if test -n "${CC}"; then
+	echo "CC=${CC}" | indent
+fi
+if test -n "${CFLAGS}"; then
+	echo "CFLAGS=${CFLAGS}" | indent
 fi
 echo ""
 
@@ -49,12 +56,12 @@ echo "##########################################################################
 echo "## Configuring build environment"
 echo "##############################################################################"
 
-echo cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS}
+echo cmake -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS} -S \"${SOURCE_DIR}\"
-env PATH="$BUILD_PATH" "$CMAKE" ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G "${CMAKE_GENERATOR}" ${CMAKE_OPTIONS}
+env PATH="${BUILD_PATH}" "${CMAKE}" -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G "${CMAKE_GENERATOR}" ${CMAKE_OPTIONS} -S "${SOURCE_DIR}"
 
 echo ""
 echo "##############################################################################"
 echo "## Building libgit2"
 echo "##############################################################################"
 
-env PATH="$BUILD_PATH" "$CMAKE" --build .
+env PATH="${BUILD_PATH}" "${CMAKE}" --build .

azure-pipelines/docker/focal

@@ -0,0 +1,79 @@
+FROM ubuntu:focal AS apt
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        bzip2 \
+        clang-10 \
+        cmake \
+        curl \
+        gcc-10 \
+        git \
+        krb5-user \
+        libcurl4-gnutls-dev \
+        libgcrypt20-dev \
+        libkrb5-dev \
+        libpcre3-dev \
+        libssl-dev \
+        libz-dev \
+        llvm-10 \
+        make \
+        ninja-build \
+        openjdk-8-jre-headless \
+        openssh-server \
+        openssl \
+        pkgconf \
+        python \
+        sudo \
+        valgrind \
+        && \
+    rm -rf /var/lib/apt/lists/* && \
+    mkdir /usr/local/msan
+
+FROM apt AS mbedtls
+RUN cd /tmp && \
+    curl --location --silent --show-error https://tls.mbed.org/download/mbedtls-2.16.2-apache.tgz | \
+        tar -xz && \
+    cd mbedtls-2.16.2 && \
+    scripts/config.pl unset MBEDTLS_AESNI_C && \
+    scripts/config.pl set MBEDTLS_MD4_C 1 && \
+    mkdir build build-msan && \
+    cd build && \
+    CC=clang-10 CFLAGS="-fPIC" cmake -G Ninja -DENABLE_PROGRAMS=OFF -DENABLE_TESTING=OFF -DUSE_SHARED_MBEDTLS_LIBRARY=ON -DUSE_STATIC_MBEDTLS_LIBRARY=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local -DCMAKE_INSTALL_PREFIX=/usr/local .. && \
+    ninja install && \
+    cd ../build-msan && \
+    CC=clang-10 CFLAGS="-fPIC" cmake -G Ninja -DENABLE_PROGRAMS=OFF -DENABLE_TESTING=OFF -DUSE_SHARED_MBEDTLS_LIBRARY=ON -DUSE_STATIC_MBEDTLS_LIBRARY=OFF -DCMAKE_BUILD_TYPE=MemSanDbg -DCMAKE_INSTALL_PREFIX=/usr/local/msan .. && \
+    ninja install && \
+    cd .. && \
+    rm -rf mbedtls-2.16.2
+
+FROM mbedtls AS libssh2
+RUN cd /tmp && \
+    curl --insecure --location --silent --show-error https://www.libssh2.org/download/libssh2-1.8.2.tar.gz | \
+        tar -xz && \
+    cd libssh2-1.8.2 && \
+    mkdir build build-msan && \
+    cd build && \
+    CC=clang-10 CFLAGS="-fPIC" cmake -G Ninja -DBUILD_SHARED_LIBS=ON -DCRYPTO_BACKEND=Libgcrypt -DCMAKE_PREFIX_PATH=/usr/local -DCMAKE_INSTALL_PREFIX=/usr/local .. && \
+    ninja install && \
+    cd ../build-msan && \
+    CC=clang-10 CFLAGS="-fPIC -fsanitize=memory -fno-optimize-sibling-calls -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer" LDFLAGS="-fsanitize=memory" cmake -G Ninja -DBUILD_SHARED_LIBS=ON -DCRYPTO_BACKEND=mbedTLS -DCMAKE_PREFIX_PATH=/usr/local/msan -DCMAKE_INSTALL_PREFIX=/usr/local/msan .. && \
+    ninja install && \
+    cd .. && \
+    rm -rf libssh2-1.8.2
+
+FROM libssh2 AS valgrind
+RUN cd /tmp && \
+    curl --insecure --location --silent --show-error https://sourceware.org/pub/valgrind/valgrind-3.15.0.tar.bz2 | \
+        tar -xj && \
+    cd valgrind-3.15.0 && \
+    CC=clang-10 ./configure && \
+    make MAKEFLAGS="-j -l$(grep -c ^processor /proc/cpuinfo)" && \
+    make install && \
+    cd .. && \
+    rm -rf valgrind-3.15.0
+
+FROM valgrind AS configure
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod a+x /usr/local/bin/entrypoint.sh
+RUN mkdir /var/run/sshd
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

script/sanitizers.supp

@@ -0,0 +1,4 @@
+[undefined]
+# This library allows unaligned access on Intel-like processors. Prevent UBSan
+# from complaining about that.
+fun:sha1_compression_states