From a3ec07d73774fb8f311a6ef4eb4f0d48a5addc1c Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Feb 2020 13:34:18 +0100 Subject: [PATCH 1/4] azure: docker: pipe downloaded archives into tar(1) directly When building dependencies for our Docker images, we first download the sources to disk first, unpack them and finally remove the archive again. This can be sped up by piping the downloading archive into tar(1) directly to parallelize both tasks. Furthermore, let's silence curl(1) to not print to status information to stderr, which tends to be interpreted as errors by Azure Pipelines. --- azure-pipelines/docker/bionic | 5 ++--- azure-pipelines/docker/xenial | 15 ++++++--------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/azure-pipelines/docker/bionic b/azure-pipelines/docker/bionic index 83d96abf3..648bda704 100644 --- a/azure-pipelines/docker/bionic +++ b/azure-pipelines/docker/bionic @@ -25,9 +25,8 @@ RUN apt-get update && \ RUN mkdir /var/run/sshd RUN cd /tmp && \ - curl -LO https://tls.mbed.org/download/mbedtls-2.16.2-apache.tgz && \ - tar -xf mbedtls-2.16.2-apache.tgz && \ - rm -f mbedtls-2.16.2-apache.tgz && \ + curl --location --silent https://tls.mbed.org/download/mbedtls-2.16.2-apache.tgz | \ + tar -xz && \ cd mbedtls-2.16.2 && \ scripts/config.pl set MBEDTLS_MD4_C 1 && \ CFLAGS=-fPIC cmake -G Ninja -DENABLE_PROGRAMS=OFF -DENABLE_TESTING=OFF -DUSE_SHARED_MBEDTLS_LIBRARY=OFF -DUSE_STATIC_MBEDTLS_LIBRARY=ON . && \ diff --git a/azure-pipelines/docker/xenial b/azure-pipelines/docker/xenial index 6e3a469ca..cb5d4919a 100644 --- a/azure-pipelines/docker/xenial +++ b/azure-pipelines/docker/xenial @@ -27,9 +27,8 @@ RUN apt-get update && \ FROM apt AS mbedtls RUN cd /tmp && \ - curl -LO https://tls.mbed.org/download/mbedtls-2.16.2-apache.tgz && \ - tar -xf mbedtls-2.16.2-apache.tgz && \ - rm -f mbedtls-2.16.2-apache.tgz && \ + curl --location --silent https://tls.mbed.org/download/mbedtls-2.16.2-apache.tgz | \ + tar -xz && \ cd mbedtls-2.16.2 && \ scripts/config.pl set MBEDTLS_MD4_C 1 && \ CFLAGS=-fPIC cmake -G Ninja -DENABLE_PROGRAMS=OFF -DENABLE_TESTING=OFF -DUSE_SHARED_MBEDTLS_LIBRARY=OFF -DUSE_STATIC_MBEDTLS_LIBRARY=ON . && \ @@ -39,9 +38,8 @@ RUN cd /tmp && \ FROM mbedtls AS libssh2 RUN cd /tmp && \ - curl -LO https://www.libssh2.org/download/libssh2-1.8.2.tar.gz && \ - tar -xf libssh2-1.8.2.tar.gz && \ - rm -f libssh2-1.8.2.tar.gz && \ + curl --location --silent https://www.libssh2.org/download/libssh2-1.8.2.tar.gz | \ + tar -xz && \ cd libssh2-1.8.2 && \ CFLAGS=-fPIC cmake -G Ninja -DBUILD_SHARED_LIBS=ON -DCRYPTO_BACKEND=Libgcrypt . && \ ninja install && \ @@ -50,9 +48,8 @@ RUN cd /tmp && \ FROM libssh2 AS valgrind RUN cd /tmp && \ - curl -LO https://sourceware.org/pub/valgrind/valgrind-3.15.0.tar.bz2 && \ - tar -xf valgrind-3.15.0.tar.bz2 && \ - rm -f valgrind-3.15.0.tar.bz2 && \ + curl --location --silent https://sourceware.org/pub/valgrind/valgrind-3.15.0.tar.bz2 | \ + tar -xj && \ cd valgrind-3.15.0 && \ ./configure && \ make && \ From 52cb4040f3b185746de3ed9fd1a5251239c5a9a9 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Feb 2020 13:52:07 +0100 Subject: [PATCH 2/4] azure: test: silence curl to not cause Azure to trop Without the "--silent" parameter, curl will print a progress meter to stderr. Azure has the nice feature of interpreting any output to stderr as errors with a big red warning towards the end of the build. Let's thus silence curl to not generate any misleading messages. --- azure-pipelines/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/test.sh b/azure-pipelines/test.sh index dfd9960ec..c6c89c6b7 100755 --- a/azure-pipelines/test.sh +++ b/azure-pipelines/test.sh @@ -84,7 +84,7 @@ if [ -z "$SKIP_GITDAEMON_TESTS" ]; then fi if [ -z "$SKIP_PROXY_TESTS" ]; then - curl -L https://github.com/ethomson/poxyproxy/releases/download/v0.7.0/poxyproxy-0.7.0.jar >poxyproxy.jar + curl --location --silent https://github.com/ethomson/poxyproxy/releases/download/v0.7.0/poxyproxy-0.7.0.jar >poxyproxy.jar echo "" echo "Starting HTTP proxy (Basic)..." @@ -96,7 +96,7 @@ if [ -z "$SKIP_PROXY_TESTS" ]; then fi if [ -z "$SKIP_NTLM_TESTS" ]; then - curl -L https://github.com/ethomson/poxygit/releases/download/v0.4.0/poxygit-0.4.0.jar >poxygit.jar + curl --location --silent https://github.com/ethomson/poxygit/releases/download/v0.4.0/poxygit-0.4.0.jar >poxygit.jar echo "" echo "Starting HTTP server..." From fb03f02aa144f45bff89923238888809fae314dc Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Feb 2020 13:56:36 +0100 Subject: [PATCH 3/4] azure: docker: avoid re-creating libgit2 home directory The Docker entrypoint currently creates the libgit2 user with "useradd --create-home". As we start the Docker container with two volumes pointing into "/home/libgit2/", the home directory will already exist. While useradd(1) copes with this just fine, it will print error messages to stderr which end up as failures in our Azure pipelines. Fix this by simply removing the "--create-home" parameter. --- azure-pipelines/docker/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/docker/entrypoint.sh b/azure-pipelines/docker/entrypoint.sh index 2118a2b5d..38eedf02b 100644 --- a/azure-pipelines/docker/entrypoint.sh +++ b/azure-pipelines/docker/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash -useradd --shell /bin/bash --create-home libgit2 +#!/bin/bash -e +useradd --shell /bin/bash libgit2 chown -R $(id -u libgit2) /home/libgit2 exec gosu libgit2 "$@" From 49bb4237cbe5af6c2ca5f155db3551ff9eb75e2e Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Feb 2020 14:04:07 +0100 Subject: [PATCH 4/4] azure: test: silence termination message when killing git-daemon(1) In order to properly tear down the test environment, we will kill git-daemon(1) if we've exercised it. As git-daemon(1) is spawned as a background process, it is still owned by the shell and thus killing it later on will print a termination message to the shell's stderr, causing Azure to report it as an error. Fix this by disowning the background process. --- azure-pipelines/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/test.sh b/azure-pipelines/test.sh index c6c89c6b7..96832d64c 100755 --- a/azure-pipelines/test.sh +++ b/azure-pipelines/test.sh @@ -81,6 +81,7 @@ if [ -z "$SKIP_GITDAEMON_TESTS" ]; then git init --bare "${GITDAEMON_DIR}/test.git" git daemon --listen=localhost --export-all --enable=receive-pack --base-path="${GITDAEMON_DIR}" "${GITDAEMON_DIR}" 2>/dev/null & GITDAEMON_PID=$! + disown $GITDAEMON_PID fi if [ -z "$SKIP_PROXY_TESTS" ]; then