From 68bfacb1072ceb6396e193c66276d06dee4f4a9d Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 18 Feb 2020 15:17:17 +0100 Subject: [PATCH 1/5] azure: remove unused Linux setup script Since migrating to Docker containings for our build and test infrastructure, we do not use the "setup-linux.sh" script anymore. Remove it to avoid any confusion. --- azure-pipelines/setup-linux.sh | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100755 azure-pipelines/setup-linux.sh diff --git a/azure-pipelines/setup-linux.sh b/azure-pipelines/setup-linux.sh deleted file mode 100755 index c5ecb550b..000000000 --- a/azure-pipelines/setup-linux.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -set -e -set -x - -TMPDIR=${TMPDIR:-/tmp} - -if [ -z "$SKIP_APT" ]; then - apt-get update - apt-get -y install build-essential pkg-config clang cmake openssl libssl-dev libssh2-1-dev libcurl4-gnutls-dev openssh-server -fi - -mkdir -p /var/run/sshd - -if [ "$MBEDTLS" ]; then - MBEDTLS_DIR=${MBEDTLS_DIR:-$(mktemp -d ${TMPDIR}/mbedtls.XXXXXXXX)} - - git clone --depth 10 --single-branch --branch mbedtls-2.6.1 https://github.com/ARMmbed/mbedtls.git ${MBEDTLS_DIR} - cd ${MBEDTLS_DIR} - - CFLAGS=-fPIC cmake -DENABLE_PROGRAMS=OFF -DENABLE_TESTING=OFF -DUSE_SHARED_MBEDTLS_LIBRARY=OFF -DUSE_STATIC_MBEDTLS_LIBRARY=ON . - cmake --build . - - if [ -z "$SKIP_MBEDTLS_INSTALL" ]; then - make install - fi -fi From f9985688073cf5dd0d69eec01de0797f6987359a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 18 Feb 2020 15:17:45 +0100 Subject: [PATCH 2/5] azure: docker: detect errors when building images The build step for our Docker images currently succeeds even if building the Docker image fails due to missing && chains in the build script. Fix this by adding them in. --- azure-pipelines/docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines/docker.yml b/azure-pipelines/docker.yml index 0e1988c8a..0f0885770 100644 --- a/azure-pipelines/docker.yml +++ b/azure-pipelines/docker.yml @@ -13,9 +13,9 @@ steps: if [ -f /tmp/dockercache/${{parameters.docker.image}}.tar ]; then docker load < /tmp/dockercache/${{parameters.docker.image}}.tar; fi displayName: 'Load Docker cache' - script: | - cd $(Build.SourcesDirectory)/azure-pipelines/docker - docker build -t libgit2/${{parameters.docker.image}} --build-arg BASE=${{parameters.docker.base}} -f ${{parameters.docker.image}} . - if [ ! -d /tmp/dockercache ]; then mkdir /tmp/dockercache; fi + cd $(Build.SourcesDirectory)/azure-pipelines/docker && + docker build -t libgit2/${{parameters.docker.image}} --build-arg BASE=${{parameters.docker.base}} -f ${{parameters.docker.image}} . && + if [ ! -d /tmp/dockercache ]; then mkdir /tmp/dockercache; fi && docker save libgit2/${{parameters.docker.image}} $(docker history -q libgit2/${{parameters.docker.image}} | grep -v '') > /tmp/dockercache/${{parameters.docker.image}}.tar displayName: 'Build Docker image' - task: docker@0 From 76b49caf6a208e44d19c84caa6d42389f0de6194 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 18 Feb 2020 15:20:08 +0100 Subject: [PATCH 3/5] azure: docker: synchronize Xenial/Bionic build instructions Our two Docker build instructions for Xenial and Bionic have diverged a bit. Let's re-synchronize them with each other to make them as similar as possible. --- azure-pipelines/docker/bionic | 8 +++++--- azure-pipelines/docker/xenial | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/azure-pipelines/docker/bionic b/azure-pipelines/docker/bionic index 648bda704..179134f25 100644 --- a/azure-pipelines/docker/bionic +++ b/azure-pipelines/docker/bionic @@ -1,7 +1,7 @@ ARG BASE -FROM $BASE +FROM $BASE AS apt RUN apt-get update && \ - apt-get install -y --no-install-recommends \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ clang \ cmake \ curl \ @@ -22,8 +22,8 @@ RUN apt-get update && \ valgrind \ && \ rm -rf /var/lib/apt/lists/* -RUN mkdir /var/run/sshd +FROM apt AS mbedtls RUN cd /tmp && \ curl --location --silent https://tls.mbed.org/download/mbedtls-2.16.2-apache.tgz | \ tar -xz && \ @@ -34,7 +34,9 @@ RUN cd /tmp && \ cd .. && \ rm -rf mbedtls-2.16.2 +FROM mbedtls AS configure COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod a+x /usr/local/bin/entrypoint.sh +RUN mkdir /var/run/sshd ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/azure-pipelines/docker/xenial b/azure-pipelines/docker/xenial index cb5d4919a..bf68b5e69 100644 --- a/azure-pipelines/docker/xenial +++ b/azure-pipelines/docker/xenial @@ -23,7 +23,9 @@ RUN apt-get update && \ openssl \ pkgconf \ python \ - valgrind + valgrind \ + && \ + rm -rf /var/lib/apt/lists/* FROM apt AS mbedtls RUN cd /tmp && \ From 01a8340662749943f3917505dc8ca65006495bec Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 18 Feb 2020 15:20:43 +0100 Subject: [PATCH 4/5] azure: docker: fix ARM builds by replacing gosu(1) Our nightly builds are currently failing due to our ARM-based jobs. These jobs crash immediately when entering the Docker container with a exception thrown by Go's language runtime. As we're able to successfully builds the Docker images in previous steps, it's unlikely to be a bug in Docker itself. Instead, this exception is thrown by gosu(1), which is a Go-based utility to drop privileges and run by our entrypoint. Fix the issue by dropping gosu(1) in favor of sudo(1). --- azure-pipelines/docker/bionic | 2 +- azure-pipelines/docker/entrypoint.sh | 4 ++-- azure-pipelines/docker/xenial | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines/docker/bionic b/azure-pipelines/docker/bionic index 179134f25..ae9604de7 100644 --- a/azure-pipelines/docker/bionic +++ b/azure-pipelines/docker/bionic @@ -7,7 +7,6 @@ RUN apt-get update && \ curl \ gcc \ git \ - gosu \ libcurl4-openssl-dev \ libpcre3-dev \ libssh2-1-dev \ @@ -19,6 +18,7 @@ RUN apt-get update && \ openssl \ pkgconf \ python \ + sudo \ valgrind \ && \ rm -rf /var/lib/apt/lists/* diff --git a/azure-pipelines/docker/entrypoint.sh b/azure-pipelines/docker/entrypoint.sh index 38eedf02b..830df49e9 100644 --- a/azure-pipelines/docker/entrypoint.sh +++ b/azure-pipelines/docker/entrypoint.sh @@ -1,4 +1,4 @@ #!/bin/bash -e useradd --shell /bin/bash libgit2 -chown -R $(id -u libgit2) /home/libgit2 -exec gosu libgit2 "$@" +chown --recursive libgit2:libgit2 /home/libgit2 +exec sudo --preserve-env --user=libgit2 "$@" diff --git a/azure-pipelines/docker/xenial b/azure-pipelines/docker/xenial index bf68b5e69..8b2501de6 100644 --- a/azure-pipelines/docker/xenial +++ b/azure-pipelines/docker/xenial @@ -8,7 +8,6 @@ RUN apt-get update && \ curl \ gcc \ git \ - gosu \ krb5-user \ libcurl4-gnutls-dev \ libgcrypt20-dev \ @@ -23,6 +22,7 @@ RUN apt-get update && \ openssl \ pkgconf \ python \ + sudo \ valgrind \ && \ rm -rf /var/lib/apt/lists/* From eaa70c6c36d3a8061e6c3ec5f95cf137fe1222ab Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 18 Feb 2020 18:09:11 +0100 Subject: [PATCH 5/5] tests: object: decrease number of concurrent cache accesses In our test case object::cache::fast_thread_rush, we're creating 100 concurrent threads opening a repository and reading objects from it. This test actually fails on ARM32 with an out-of-memory error, which isn't entirely unexpected. Work around the issue by halving the number of threads. --- tests/object/cache.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/object/cache.c b/tests/object/cache.c index 3ccc325e4..08bf03648 100644 --- a/tests/object/cache.c +++ b/tests/object/cache.c @@ -244,15 +244,15 @@ static void *cache_quick(void *arg) void test_object_cache__fast_thread_rush(void) { - int try, th, data[THREADCOUNT*2]; + int try, th, data[THREADCOUNT]; #ifdef GIT_THREADS - git_thread t[THREADCOUNT*2]; + git_thread t[THREADCOUNT]; #endif for (try = 0; try < REPEAT; ++try) { cl_git_pass(git_repository_open(&g_repo, cl_fixture("testrepo.git"))); - for (th = 0; th < THREADCOUNT*2; ++th) { + for (th = 0; th < THREADCOUNT; ++th) { data[th] = th; #ifdef GIT_THREADS cl_git_pass( @@ -263,7 +263,7 @@ void test_object_cache__fast_thread_rush(void) } #ifdef GIT_THREADS - for (th = 0; th < THREADCOUNT*2; ++th) { + for (th = 0; th < THREADCOUNT; ++th) { void *rval; cl_git_pass(git_thread_join(&t[th], &rval)); cl_assert_equal_i(th, *((int *)rval));