├── .travis.yml ├── DESIGN.md ├── Dockerfile ├── LICENSE ├── README.md ├── VERSION ├── base-ubuntu-trusty ├── Dockerfile.gcc ├── build.sh ├── push.sh └── scripts │ ├── dockerfile_gcc.sh │ ├── install_common.sh │ ├── install_libidn2.sh │ ├── install_libpsl.sh │ └── install_nghttp2.sh ├── build_scripts ├── build_and_push.sh ├── cleanup.sh ├── common.sh ├── download_stable_curl.sh ├── run_normal.sh ├── stage1.sh ├── stage2.sh ├── stage3.sh └── stage4.sh ├── docker_hub_scripts └── remove_tag.py └── docker_scripts ├── create_gssapi_libssh2.sh ├── create_testuser.sh ├── install_base_tools.sh ├── install_clang7.sh ├── install_gcc8.sh ├── test_common.sh ├── test_normal.sh └── testuser_run.sh /.travis.yml: -------------------------------------------------------------------------------- 1 | language: minimal 2 | services: 3 | - docker 4 | jobs: 5 | include: 6 | # Pull requests don't have access to encrypted variables, so they cannot 7 | # benefit from the multistage support. 8 | 9 | # Stage 1: Build the base image 10 | - stage: stage1_base_image 11 | script: ./build_scripts/stage1.sh 12 | if: type != pull_request 13 | 14 | # Stage 2: GCC and Clang 15 | - stage: stage2_gcc_clang 16 | script: ./build_scripts/stage2.sh gcc8 17 | if: type != pull_request 18 | - script: ./build_scripts/stage2.sh clang7 19 | if: type != pull_request 20 | 21 | # Stage 3: Build remaining images 22 | - stage: stage3_build_images 23 | script: ./build_scripts/stage3.sh gssapi_libssh2 24 | if: type != pull_request 25 | 26 | # Stage 4: Test building curl 27 | - stage: stage4_test_images 28 | script: ./build_scripts/stage4.sh gssapi_libssh2 29 | if: type != pull_request 30 | 31 | # Cleanup: remove any temporary images 32 | - stage: remove_images 33 | script: ./build_scripts/cleanup.sh 34 | if: tag IS blank AND type != pull_request 35 | 36 | # ----- 37 | 38 | # For pull requests we still want to do testing, but we have to do it in 39 | # serial mode. 40 | - stage: all_stages_pull_request 41 | script: 42 | - ./build_scripts/stage1.sh 43 | - ./build_scripts/stage2.sh 44 | - ./build_scripts/stage3.sh 45 | - ./build_scripts/stage4.sh 46 | if: type = pull_request 47 | 48 | env: 49 | global: 50 | - secure: hZHhrJmUroDWlcXMs6oH9ckWpbKOd3/9z6EEceBmRy/X1J21N5vMCnnjeQT1uDRbN3IO/Ir56eCNQ8tGXYptH50/yxywQ4QtUk+vR/VrqUuI+m69lKG0WjDP+SzOCHaf17lbrXtyBPRsVZe+W34uNt0bMMcRpF3IG0WiNQFnrugwXtaYDyZPEJb8sb4s95lhHzsjfAaJQpUNncL7Tk4neg3ItZ0oLvjdBptMJqDOPrXtEFwBr8OMQ0WxkP7hdeDdj1xvTvvb1xmlI69tRO9w5xkAhLPD9YA7mcizCjnEPuLGFowiyafLF3MfQapII2opYaFyhoj1Z6gCUNNhi9ueDMOkSjq4TDgJ0SIGIHFJLf+9VMhubPFCMySGFjtI/unziSQjAnYOczM+crFPU3a4VxHti9289jTmiu9YmFxivb09PGo3OrWemHERySy3wtcKxbpruljk1F0gsFKrRFH4ovLLq4Oz0zfNyWnSZIdVVUc5TQKtJNlBQkLA1IAaSC6okIV+gTrdWHhPb9Fuxj4gb+qX0hNdLTwc7RDhPUF2ua/CnOi3JXOrE7Y9AZvN/9d6zlq+cfh4yYbOEEcpJELHSGhulaGgz1PQ3iSMbBh5L1c0CgNydbcoHRzVzdhZYNbm3C9t46pcb/EFjKH0eEDxjD/3CB9egLdXjnRoiMmr1/o= 51 | - secure: UVw8wRAzc1YcbGnFeAVNohOKZ2vMtuaKJC53MobN/4HXCQ4xRST2XaUl6CkdS2MyXXrrvcMC9jER5lIJntwKYc8aCtEeSoqQP6EjWfODBPjg0IJll63uZaYSm/iLdavhjA/ZQN5NN5tIlOx5T9C47rBDmRJdB+L+pobDrMaGt35qZNfI7ibugN/ecfUfJkzldVEWr5wh732zuBjLdpXTzx/mbNZyolfOPXubysGs8LJI2aqU9VB68HlbbvZXq7oUzvYz6t0L2DornD3DazsQXFemuMqAqB/17eEpO4HqtzDO+6o0WRtjctLXFhvGX8sXYlMH24GuZvEemQeUlWpDPAKS9BiUNpG/21ShA0wCK+2BuiZ8rNBngZjRK2YLVgIi+2582knf3HxXsDlEKuuVdJRY6dVYekTXSNIR95I/WtypoXWHM1L5HhHFfecVgGu5yIPMgbLK3h0uV8G4E/Og6IY/OWBi0ZWKgmK8xLbKVf/MsK6Xa1uBxJvf3709WzyzxY0mVJTTIEgHCcCWsF9ixJDwJ6C7yPibJfwE/66q7XbFLFuahUMYGtcpreI8OIx7dNzdGRnAVzafygFGdzfPrgzN9zPmV72T1y0uqAJpWznRShWco4yneLxddvymrQ/Qrn77QFKbLbzM+8VBKnKnBw3wm88WbYdg1a7y8Jm8kns= 52 | -------------------------------------------------------------------------------- /DESIGN.md: -------------------------------------------------------------------------------- 1 | # Design 2 | 3 | This document covers the design thoughts behind build images for speeding up Travis CI and a proposal for how to implement things going forward. 4 | 5 | ## Background 6 | 7 | Travis CI currently uses standard Ubuntu containers to build curl. There are lots and lots of entries in the curl build matrix and currently a single job takes a long time to complete. 8 | 9 | Part of this is due to downloading and compiling dependencies (e.g. brotli, wolfssl). There's also elapsed time taken in downloading and installing packages for compilation. 10 | 11 | Using Docker containers, we can create a build image whereby the dependencies are already downloaded and installed, and compiled where necessary. Theoretically it also means developers can have the same build environment as the CI jobs, which is good for reproducibility. 12 | 13 | ## Design thoughts 14 | 15 | - The build images should be based on Ubuntu Trusty, as Travis CI currently use Ubuntu Trusty as their base image. 16 | - An alternative could be Centos - any Linux distribution is likely fine. It might be that we choose Trusty to start with then, then move onto Centos for testing different distributions. 17 | - Another alternative is Xenial as many of the builds explicitly request the Xenial distribution. 18 | - Build images will be strictly versioned using semver for reproducibility. The "latest" tag will not be used. 19 | 20 | ## Current set of builds 21 | 22 | The current set of Linux curl CI builds (from .travis.yml) are as follows: 23 | 24 | - gcc-8 (with-gssapi, with-libssh2) CHECKSRC=1 25 | - gcc-8 (disable-http, disable-smtp, disable-imap) 26 | - gcc-8 (enable-ares) 27 | - gcc-8 (dist:xenial, disable-verbose, no-variadic-macros) NOTESTS=1 28 | - gcc-8 (with-brotli) 29 | - gcc-8 (dist:xenial, with-boringssl) T=novalgrind 30 | - gcc-8 (dist:xenial, with-wolfssl, without-ssl) 31 | - gcc-8 (dist:xenial, with-mesalink, without-ssl) 32 | - clang-7 (dist:xenial) 33 | - clang-7 (dist:xenial, enable-alt-svc) 34 | - clang-7 (dist:xenial, with-mbedtls, without-tls) 35 | - clang-7 (dist:xenial, with-gnutls, without-tls) 36 | - clang-7 (dist:xenial, disable-threaded-resolver) T=debug 37 | - clang-7 (dist:xenial, with-nss, without-ssl) T=debug NOTESTS=1 38 | - gcc-8 () T=iconv 39 | - gcc-8 (dist:xenial) T=cmake 40 | - clang-7 (dist:xenial) T=cmake 41 | - gcc-8 (dist:xenial) T=coverage 42 | - gcc-8 (dist:xenial) T=distcheck 43 | - clang-7 (dist:xenial) T=fuzzer 44 | - clang-7 (dist:xenial) T=tidy 45 | - clang-7 (dist:xenial) T=scan-build 46 | - clang-7 (dist:xenial, many sanitization options) T=debug 47 | 48 | There are several OSX builds which I don't intend to run through Docker. 49 | 50 | It probably makes sense to have an individual build image for each of these different options, caching the dependencies of each. Docker builds are "cheap". 51 | 52 | It might make sense to have several stages of build: 53 | 54 | - Build gcc-8, clang-7 images on top of the base image (Trusty/Xenial/whatever) 55 | - Build all these images on top of these bases. -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Specify the base image to build from as a --build-arg. 2 | ARG BASE_IMAGE 3 | FROM $BASE_IMAGE 4 | 5 | # The docker build process is noninteractive 6 | ARG DEBIAN_FRONTEND=noninteractive 7 | 8 | # Run the install process using the provided script (chosen using --build-arg) 9 | ARG DOCKER_SCRIPT 10 | COPY docker_scripts/ /scripts/ 11 | RUN /scripts/${DOCKER_SCRIPT} 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 The curl authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # build-images 2 | repository for curl build images 3 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.1.0 -------------------------------------------------------------------------------- /base-ubuntu-trusty/Dockerfile.gcc: -------------------------------------------------------------------------------- 1 | # Base from Ubuntu trusty 2 | FROM ubuntu:trusty 3 | 4 | # The docker build process is noninteractive 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | # Run the install process using the provided script 8 | COPY scripts/ /scripts/ 9 | RUN /scripts/dockerfile_gcc.sh -------------------------------------------------------------------------------- /base-ubuntu-trusty/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euxo pipefail 4 | 5 | # Build gcc image 6 | docker build -f Dockerfile.gcc \ 7 | -t curlimages/base-ubuntu-trusty:gcc \ 8 | . 9 | -------------------------------------------------------------------------------- /base-ubuntu-trusty/push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euxo pipefail 4 | 5 | # Push docker image. This assumes that the curlimages user is logged in. 6 | docker push curlimages/base-ubuntu-trusty:gcc 7 | -------------------------------------------------------------------------------- /base-ubuntu-trusty/scripts/dockerfile_gcc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Print out debug info and fail if a sub-step fails 4 | set -exuo pipefail 5 | 6 | # Define a set of temporary docker packages used during docker building. 7 | DOCKER_BUILD_PACKAGES="software-properties-common curl" 8 | 9 | # Update APT so we can install packages. 10 | apt-get -y update 11 | 12 | # Install temporary build packages 13 | apt-get -y install $DOCKER_BUILD_PACKAGES 14 | 15 | # Install the necessary repositories: 16 | # - ubuntu-toolchain-r-test 17 | add-apt-repository ppa:ubuntu-toolchain-r/test 18 | 19 | # Install all necessary packages for curl building 20 | apt-get -y update 21 | bash /scripts/install_common.sh 22 | apt-get -y install gcc-7 g++-7 23 | 24 | # Install gcc-7 as the standard cc and g++-7 as the standard c++ 25 | update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 10 26 | update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 10 27 | update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 28 | update-alternatives --set cc /usr/bin/gcc 29 | update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 30 | update-alternatives --set c++ /usr/bin/g++ 31 | 32 | # Download, compile and install other libraries. 33 | /scripts/install_nghttp2.sh 34 | /scripts/install_libidn2.sh 35 | /scripts/install_libpsl.sh 36 | 37 | # Remove the temporary build packages and any auto-installed dependencies 38 | apt-get --purge -y remove $DOCKER_BUILD_PACKAGES 39 | apt-get -y autoremove 40 | 41 | # Removing any apt cache data. 42 | rm -rf /var/lib/apt/lists/* 43 | -------------------------------------------------------------------------------- /base-ubuntu-trusty/scripts/install_common.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Install common dependencies 4 | apt-get -y install make \ 5 | pkg-config \ 6 | autoconf \ 7 | libtool \ 8 | valgrind \ 9 | libev-dev \ 10 | libc-ares-dev \ 11 | libstdc++-7-dev \ 12 | stunnel4 \ 13 | libssh2-1-dev \ 14 | libssh-dev \ 15 | krb5-user \ 16 | autopoint \ 17 | libunistring-dev \ 18 | python 19 | -------------------------------------------------------------------------------- /base-ubuntu-trusty/scripts/install_libidn2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exuo pipefail 4 | 5 | # Download, compile and install libidn2. 6 | pushd /tmp 7 | curl -L https://ftp.gnu.org/gnu/libidn/libidn2-2.0.4.tar.gz | tar -xzvf - 8 | pushd libidn2-2.0.4 9 | ./configure --prefix=/usr 10 | make 11 | make install 12 | popd 13 | 14 | # Clean up afterwards. 15 | rm -rf libidn2-2.0.4 16 | popd 17 | -------------------------------------------------------------------------------- /base-ubuntu-trusty/scripts/install_libpsl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exuo pipefail 4 | 5 | # Download, compile and install libpsl. 6 | pushd /tmp 7 | curl -L https://github.com/rockdaboot/libpsl/releases/download/libpsl-0.20.1/libpsl-0.20.1.tar.gz | tar -xzvf - 8 | pushd libpsl-0.20.1 9 | autoreconf -i 10 | ./configure --prefix=/usr 11 | make 12 | make install 13 | popd 14 | 15 | # Clean up afterwards. 16 | rm -rf libpsl-0.20.1 17 | popd 18 | -------------------------------------------------------------------------------- /base-ubuntu-trusty/scripts/install_nghttp2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euxo pipefail 4 | 5 | apt-get install -y libcunit1-dev \ 6 | libjemalloc-dev 7 | 8 | # Download, compile and install nghttp2. 9 | pushd /tmp 10 | curl -L https://github.com/nghttp2/nghttp2/releases/download/v1.24.0/nghttp2-1.24.0.tar.gz | tar xzvf - 11 | pushd nghttp2-1.24.0 12 | ./configure --prefix=/usr --disable-threads --enable-app 13 | make 14 | make install 15 | popd 16 | 17 | # Clean up afterwards. 18 | rm -rf nghttp2-1.24.0 19 | popd 20 | -------------------------------------------------------------------------------- /build_scripts/build_and_push.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euxo pipefail 4 | 5 | # Import the common functionality. 6 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 7 | 8 | # The first argument is the tag. This should match a repository in Docker and 9 | # is the "name" of the image. 10 | TAG=${1} 11 | 12 | # The second argument is the base image. This is what the image is built on top 13 | # of. 14 | BASE_IMAGE=${2} 15 | 16 | # The third argument is the build script. This is run inside Docker to install 17 | # the new files. 18 | DOCKER_SCRIPT=${3} 19 | 20 | # Use the common script functionality to build the image. 21 | build_image_versioned ${TAG} ${BASE_IMAGE} ${DOCKER_SCRIPT} 22 | 23 | # On Travis, push the newly generated image if possible. 24 | if [[ -n "${TRAVIS:-}" ]] 25 | then 26 | push_image_versioned ${TAG} 27 | fi 28 | -------------------------------------------------------------------------------- /build_scripts/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euxo pipefail 4 | 5 | # Import the common functionality. 6 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 7 | 8 | if [[ -n "${TRAVIS:-}" ]] 9 | then 10 | if [[ -z "${TRAVIS_TAG:-}" ]] 11 | then 12 | # Install requests. 13 | sudo apt-get update 14 | sudo apt-get -y install python3 python3-pip 15 | sudo pip3 install requests 16 | 17 | # Remove all the images. 18 | remove_image_versioned base_image 19 | remove_image_versioned base_gcc8 20 | remove_image_versioned base_clang7 21 | remove_image_versioned gssapi_libssh2 22 | fi 23 | fi 24 | -------------------------------------------------------------------------------- /build_scripts/common.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Calculate the current build locations 4 | export SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") 5 | export ROOTDIR=${SCRIPTDIR}/.. 6 | 7 | # If running on travis and processing a tag, work with curlbuildimages. 8 | # If running on travis and processing anything else, work with 9 | # curlbuildimagestemp. 10 | # If running locally, work with the curlbuildimagestemp namespace. 11 | if [[ -n "${TRAVIS:-}" ]] 12 | then 13 | 14 | # Check to see if we can log into Docker. Pull requests cannot log into Docker 15 | if [[ -n "${DOCKER_USER:-}" ]] 16 | then 17 | # Log into Docker. 18 | echo "${DOCKER_PASS}" | docker login -u "${DOCKER_USER}" --password-stdin 19 | fi 20 | 21 | if [[ -n "${TRAVIS_TAG:-}" ]] 22 | then 23 | # Processing a tagged build. Get the version from the version file. 24 | export VERSION=$(cat VERSION) 25 | 26 | # Use the main repository. 27 | export DOCKER_REPO=curlbuildimages 28 | else 29 | # Processing an untagged build. The version is the build ID, which is 30 | # shared between stages. 31 | export VERSION=${TRAVIS_BUILD_ID} 32 | 33 | # Use the temporary repository. 34 | export DOCKER_REPO=curlbuildimagestemp 35 | fi 36 | else 37 | # Get the version from the version file. 38 | export VERSION=$(cat VERSION) 39 | 40 | # Use the temporary repository. 41 | export DOCKER_REPO=curlbuildimagestemp 42 | fi 43 | 44 | 45 | # Simple function which uses docker to build images. 46 | build_image() 47 | { 48 | TAG=$1 49 | BASE_IMAGE=$2 50 | DOCKER_SCRIPT=$3 51 | 52 | echo "Building ${TAG}:${VERSION} from ${BASE_IMAGE}" 53 | 54 | docker build --build-arg BASE_IMAGE=${BASE_IMAGE} \ 55 | --build-arg DOCKER_SCRIPT=${DOCKER_SCRIPT} \ 56 | -t ${DOCKER_REPO}/${TAG}:${VERSION} \ 57 | . 58 | } 59 | 60 | # Simple wrapper function which adds the REPOSITORY and VERSION to the given 61 | # BASE_IMAGE, then builds. 62 | build_image_versioned() 63 | { 64 | TAG=$1 65 | BASE_IMAGE=$2 66 | DOCKER_SCRIPT=$3 67 | 68 | build_image ${TAG} ${DOCKER_REPO}/${BASE_IMAGE}:${VERSION} ${DOCKER_SCRIPT} 69 | } 70 | 71 | 72 | # Simple function which adds the REPOSITORY and VERSION to the given TAG, 73 | # then pushes to Docker. 74 | push_image_versioned() 75 | { 76 | if [[ -n "${DOCKER_USER:-}" ]] 77 | then 78 | # The Docker user environment variable exists, so pushing should work. 79 | TAG=$1 80 | docker push ${DOCKER_REPO}/${TAG}:${VERSION} 81 | else 82 | echo "Would have pushed to ${DOCKER_REPO}/${TAG}:${VERSION} but " 83 | echo "DOCKER_USER is not set." 84 | fi 85 | } 86 | 87 | 88 | # Function to remove an image tag from the Docker hub 89 | remove_image_versioned() 90 | { 91 | python3 ${ROOTDIR}/docker_hub_scripts/remove_tag.py \ 92 | --username ${DOCKER_USER} \ 93 | --password ${DOCKER_PASS} \ 94 | --repository ${1} \ 95 | --tag ${VERSION} 96 | } 97 | -------------------------------------------------------------------------------- /build_scripts/download_stable_curl.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | pushd /tmp 4 | 5 | # Download an archive of a stable version of curl. 6 | wget https://github.com/curl/curl/releases/download/curl-7_65_1/curl-7.65.1.tar.gz 7 | 8 | # Untar curl 9 | tar xvf curl-7.65.1.tar.gz 10 | 11 | # Rename the folder to curl 12 | mv curl-7.65.1 curl 13 | 14 | popd -------------------------------------------------------------------------------- /build_scripts/run_normal.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euxo pipefail 4 | 5 | # Import the common functionality. 6 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 7 | 8 | # Get the inputs to this script. 9 | IMAGE_NAME=${1} 10 | CURL_DIR=${2} 11 | 12 | # Shift the arguments along so we can pass $@ to the docker container. 13 | shift 2 14 | 15 | # Derive the full name of the image. 16 | FULL_IMAGE_NAME=${DOCKER_REPO}/${IMAGE_NAME}:${VERSION} 17 | 18 | # If we need to run with special environment variables, check now. 19 | ENV_CHECKSRC= 20 | 21 | if [[ -n ${CHECKSRC:-} ]] 22 | then 23 | ENV_CHECKSRC="-e CHECKSRC=1" 24 | fi 25 | 26 | # Finally, just run the docker test. 27 | # - Maps in ${CURL_DIR} as /opt/curl 28 | # - Runs /scripts/test_normal.sh as `testuser` with the current user's UID 29 | # and GID. 30 | docker run --rm -it $ENV_CHECKSRC \ 31 | -v ${CURL_DIR}:/opt/curl \ 32 | ${FULL_IMAGE_NAME} \ 33 | /scripts/testuser_run.sh $(id -u) $(id -g) /scripts/test_normal.sh $@ 34 | -------------------------------------------------------------------------------- /build_scripts/stage1.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euxo pipefail 4 | 5 | # Import the common functionality. 6 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 7 | 8 | # Build stage 1. Stage 1 consists of the root base image with some common tools installed. 9 | echo "Building stage 1 (version ${VERSION})" 10 | 11 | # Build the root base image from Xenial 12 | build_image base_image ubuntu:xenial-20190515 install_base_tools.sh 13 | 14 | 15 | # On Travis, push the images. 16 | if [[ -n "${TRAVIS:-}" ]] 17 | then 18 | push_image_versioned base_image 19 | fi 20 | -------------------------------------------------------------------------------- /build_scripts/stage2.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script encompasses all the builds that should happen in stage 2 of 4 | # the Travis build. 5 | 6 | set -euxo pipefail 7 | 8 | # Import the common functionality. 9 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 10 | 11 | # If a build name is passed then just build that build. 12 | REQUESTED=${1:-} 13 | 14 | # Build stage 2. Stage 2 produces images with gcc-8 and clang-7 installed. 15 | 16 | if [[ -z ${REQUESTED} ]] || [[ ${REQUESTED} == "gcc8" ]] 17 | then 18 | # Build gcc-8. Depends on `base_image` from stage1 19 | ${SCRIPTDIR}/build_and_push.sh base_gcc8 base_image install_gcc8.sh 20 | fi 21 | 22 | if [[ -z ${REQUESTED} ]] || [[ ${REQUESTED} == "clang7" ]] 23 | then 24 | # Build clang-7. Depends on `base_image` from stage1 25 | ${SCRIPTDIR}/build_and_push.sh base_clang7 base_image install_clang7.sh 26 | fi -------------------------------------------------------------------------------- /build_scripts/stage3.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script encompasses all the builds that should happen in stage 3 of 4 | # the Travis build. 5 | 6 | set -euxo pipefail 7 | 8 | # Import the common functionality. 9 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 10 | 11 | # If a build name is passed then just build that build. 12 | REQUESTED=${1:-} 13 | 14 | # Build stage 3. Stage 3 produces all the relevant test images. 15 | if [[ -z ${REQUESTED} ]] || [[ ${REQUESTED} == "gssapi_libssh2" ]] 16 | then 17 | # Build for test: gcc-8 (with-gssapi, with-libssh2) CHECKSRC=1 18 | ${SCRIPTDIR}/build_and_push.sh gssapi_libssh2 base_gcc8 create_gssapi_libssh2.sh 19 | fi 20 | -------------------------------------------------------------------------------- /build_scripts/stage4.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script encompasses all the tests that should happen in stage 4 of 4 | # the Travis build. 5 | 6 | set -euxo pipefail 7 | 8 | # Import the common functionality. 9 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 10 | 11 | # If a test name is passed then just run that test. 12 | REQUESTED=${1:-} 13 | 14 | # Download a temporary version of curl if required. 15 | if [[ ! -d /tmp/curl ]] 16 | then 17 | ${SCRIPTDIR}/download_stable_curl.sh 18 | fi 19 | 20 | if [[ -z ${REQUESTED} ]] || [[ ${REQUESTED} == "gssapi_libssh2" ]] 21 | then 22 | # Run test: gcc-8 (with-gssapi, with-libssh2) CHECKSRC=1 23 | CHECKSRC=1 ${SCRIPTDIR}/run_normal.sh gssapi_libssh2 /tmp/curl --with-gssapi --with-libssh2 24 | fi 25 | -------------------------------------------------------------------------------- /docker_hub_scripts/remove_tag.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """Removes a tag from Docker""" 4 | 5 | import argparse 6 | import sys 7 | import logging 8 | import requests 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | def remove_tag(args): 14 | """Main script for remove_tag 15 | 16 | """ 17 | # First, obtain a token from the Docker hub. 18 | login_data = { 19 | "username": args.username, 20 | "password": args.password, 21 | } 22 | r = requests.post("https://hub.docker.com/v2/users/login/", data=login_data) 23 | r.raise_for_status() 24 | 25 | # Obtain the token from the response. 26 | response_data = r.json() 27 | 28 | headers = { 29 | "Authorization": "JWT {0}".format(response_data["token"]) 30 | } 31 | 32 | # Now delete the tag. 33 | url = ("https://hub.docker.com/v2/repositories/{org}/{repo}/tags/{tag}" 34 | .format(org=args.organization, 35 | repo=args.repository, 36 | tag=args.tag)) 37 | 38 | r = requests.delete(url, headers=headers) 39 | r.raise_for_status() 40 | 41 | return ScriptRC.SUCCESS 42 | 43 | 44 | def main(): 45 | """Main handling function. Wraps remove_tag.""" 46 | 47 | # Set up basic logging 48 | logging.basicConfig(format="%(asctime)s %(levelname)-5.5s %(message)s", 49 | stream=sys.stdout, 50 | level=logging.INFO) 51 | 52 | # Run main script. 53 | parser = argparse.ArgumentParser() 54 | parser.add_argument("--username", required=True) 55 | parser.add_argument("--password", required=True) 56 | parser.add_argument("--organization", default="curlbuildimagestemp") 57 | parser.add_argument("--repository", required=True) 58 | parser.add_argument("--tag", required=True) 59 | 60 | args = parser.parse_args() 61 | 62 | try: 63 | exit_code = remove_tag(args) 64 | except Exception as e: 65 | log.exception(e) 66 | exit_code = ScriptRC.EXCEPTION 67 | 68 | log.info("Returning %d", exit_code) 69 | return exit_code 70 | 71 | 72 | class ScriptRC(object): 73 | SUCCESS = 0 74 | FAILURE = 1 75 | EXCEPTION = 2 76 | 77 | 78 | class ScriptException(Exception): 79 | pass 80 | 81 | 82 | if __name__ == '__main__': 83 | sys.exit(main()) 84 | -------------------------------------------------------------------------------- /docker_scripts/create_gssapi_libssh2.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Print out debug info and fail if a sub-step fails 4 | set -exuo pipefail 5 | 6 | # Install --with-gssapi and --with-libssh2 development packages. 7 | apt-get -y install --no-install-recommends krb5-user libkrb5-dev libssh2-1-dev 8 | 9 | # Install --with-gssapi and --with-libssh2 test packages. 10 | apt-get -y install --no-install-recommends openssh-server 11 | 12 | # Add the testuser in for testing purposes. 13 | /scripts/create_testuser.sh 14 | -------------------------------------------------------------------------------- /docker_scripts/create_testuser.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euxo pipefail 4 | 5 | # Create a user for running tests if it doesn't exist. 6 | if ! id -u testuser 7 | then 8 | useradd -ms /bin/bash testuser 9 | echo "testuser:testuser" | chpasswd 10 | fi 11 | -------------------------------------------------------------------------------- /docker_scripts/install_base_tools.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Print out debug info and fail if a sub-step fails 4 | set -exuo pipefail 5 | 6 | # Install the tool needed for `add-apt-repository`. 7 | apt-get -y update 8 | apt-get -y install --no-install-recommends software-properties-common 9 | 10 | # Install wget to download files. 11 | apt-get -y install --no-install-recommends wget 12 | 13 | # Install common build dependencies 14 | apt-get -y install --no-install-recommends make \ 15 | pkg-config \ 16 | autoconf \ 17 | libtool \ 18 | valgrind \ 19 | automake 20 | -------------------------------------------------------------------------------- /docker_scripts/install_clang7.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Print out debug info and fail if a sub-step fails 4 | set -exuo pipefail 5 | 6 | # Install the necessary repositories for clang: 7 | # - llvm-toolchain-xenial-7 8 | 9 | # Use wget to download the key as we're building curl... 10 | wget http://apt.llvm.org/llvm-snapshot.gpg.key 11 | apt-key add llvm-snapshot.gpg.key 12 | add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main" 13 | 14 | # Install clang7. Only install dependencies, not recommendations. 15 | apt-get -y update 16 | apt-get -y install --no-install-recommends clang-7 17 | 18 | # Install clang-7 as the standard cc and clang++-7 as the standard c++ 19 | update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 10 20 | update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-7 10 21 | update-alternatives --install /usr/bin/cc cc /usr/bin/clang 30 22 | update-alternatives --set cc /usr/bin/clang 23 | update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 30 24 | update-alternatives --set c++ /usr/bin/clang++ 25 | -------------------------------------------------------------------------------- /docker_scripts/install_gcc8.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Print out debug info and fail if a sub-step fails 4 | set -exuo pipefail 5 | 6 | # Install the necessary repositories for newer gccs: 7 | # - ubuntu-toolchain-r-test 8 | add-apt-repository ppa:ubuntu-toolchain-r/test 9 | 10 | # Install gcc-8. Only install dependencies, not recommendations. 11 | apt-get -y update 12 | apt-get -y install --no-install-recommends gcc-8 g++-8 13 | 14 | # Install gcc-8 as the standard cc and g++-8 as the standard c++ 15 | update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 10 16 | update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 10 17 | update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 18 | update-alternatives --set cc /usr/bin/gcc 19 | update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 20 | update-alternatives --set c++ /usr/bin/g++ 21 | -------------------------------------------------------------------------------- /docker_scripts/test_common.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Print out debug info and fail if a sub-step fails 4 | set -exuo pipefail 5 | 6 | # If /opt/curl has not been mounted into the Docker container, then fail the 7 | # build. 8 | if [[ ! -d /opt/curl ]] 9 | then 10 | echo "Mount a curl root directory into the Docker container" 11 | echo "by using -v :/opt/curl" 12 | exit 1 13 | fi 14 | 15 | # Run the buildconf script. 16 | pushd /opt/curl 17 | ./buildconf 18 | popd 19 | 20 | # Set MAKEFLAGS to the number of processors. 21 | export MAKEFLAGS+=-j$(nproc) 22 | -------------------------------------------------------------------------------- /docker_scripts/test_normal.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Print out debug info and fail if a sub-step fails 4 | set -exuo pipefail 5 | 6 | # Do initial common setup. 7 | . /scripts/test_common.sh 8 | 9 | # The common code checks for the existence of the codebase. 10 | pushd /opt/curl 11 | 12 | # Configure with standard options plus anything on the command line. 13 | ./configure --enable-warnings --enable-werror $@ 14 | 15 | # Make the main program and examples. 16 | make 17 | make examples 18 | 19 | # Ensure the permissions are correct for the test user. 20 | mkdir -p ./tests/log 21 | chmod 777 ./tests/log 22 | 23 | # Run the tests. 24 | if [[ -z ${NOTESTS:-} ]] 25 | then 26 | make test-nonflaky 27 | fi 28 | 29 | # Check source code if requested. 30 | if [[ -n ${CHECKSRC:-} ]] 31 | then 32 | make checksrc 33 | fi 34 | 35 | popd -------------------------------------------------------------------------------- /docker_scripts/testuser_run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Ensure that we're running with the same user credentials (UID, GID) as the 4 | # host OS, to ensure that any files are created with the right permissions. 5 | 6 | set -euxo pipefail 7 | 8 | EXTUID=${1} 9 | EXTGID=${2} 10 | 11 | shift 2 12 | COMMAND="$@" 13 | 14 | # Switch the testuser to use the passed in values for UID and GID. 15 | usermod -u ${EXTUID} testuser 16 | groupmod -g ${EXTGID} testuser 17 | usermod -g ${EXTGID} testuser 18 | 19 | # Run the command as testuser. 20 | su -c "${COMMAND}" testuser 21 | --------------------------------------------------------------------------------