├── .cramrc ├── tests ├── data │ ├── workdir-without-overlays.zip │ └── workdir-with-augur-auspice-overlays.zip ├── basic.t ├── aws-batch-endpoint-url.t ├── workdir.t ├── envdir.t ├── aws-batch-verbose.t ├── envdir-url.t └── aws-batch-workdir-url.t ├── chdir-workdir ├── .gitignore ├── devel ├── stop-localhost-registry ├── platform ├── pull-from-registry ├── start-localhost-registry ├── clean ├── validate-platforms ├── copy-images ├── build ├── summarize-buildkit-output └── delete-from-ghcr.js ├── drop-privs ├── bashrc ├── Makefile ├── .github ├── dependabot.yml └── workflows │ └── ci.yml ├── delete-envd ├── create-envd ├── entrypoint ├── entrypoint-aws-batch ├── README.md └── Dockerfile /.cramrc: -------------------------------------------------------------------------------- 1 | [cram] 2 | shell = /bin/bash 3 | indent = 2 4 | -------------------------------------------------------------------------------- /tests/data/workdir-without-overlays.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nextstrain/docker-base/HEAD/tests/data/workdir-without-overlays.zip -------------------------------------------------------------------------------- /tests/data/workdir-with-augur-auspice-overlays.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nextstrain/docker-base/HEAD/tests/data/workdir-with-augur-auspice-overlays.zip -------------------------------------------------------------------------------- /chdir-workdir: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | if [[ -n "${NEXTSTRAIN_WORKDIR:-}" ]]; then 5 | mkdir --parents "$NEXTSTRAIN_WORKDIR" 6 | cd "$NEXTSTRAIN_WORKDIR" 7 | fi 8 | 9 | exec "$@" 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | /tests/*.err 3 | /logs/ 4 | 5 | # OS generated files # 6 | ###################### 7 | .DS_Store 8 | .DS_Store? 9 | ._* 10 | .Spotlight-V100 11 | .Trashes 12 | Icon? 13 | ehthumbs.db 14 | Thumbs.db 15 | -------------------------------------------------------------------------------- /devel/stop-localhost-registry: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Stops a local Docker registry created by start-localhost-registry. 4 | # 5 | set -euo pipefail 6 | 7 | NAME=nextstrain-local-registry 8 | 9 | docker stop "$NAME" > /dev/null 10 | docker rm "$NAME" > /dev/null 11 | -------------------------------------------------------------------------------- /drop-privs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | if [[ "$(id -u):$(id -g)" == 0:0 ]]; then 5 | # Drop down to nextstrain:nextstrain if we're root. 6 | exec setpriv --reuid nextstrain --regid nextstrain --init-groups "$@" 7 | else 8 | # Otherwise, respect privs set externally. 9 | exec "$@" 10 | fi 11 | -------------------------------------------------------------------------------- /bashrc: -------------------------------------------------------------------------------- 1 | # Note that this file is overridden by newer versions of `nextstrain shell` and 2 | # so it provides only a fallback prompt for users of older CLI versions or 3 | # direct image users (e.g. `docker run …`). 4 | reset="\[\e[0m\]" 5 | bold="\[\e[1m\]" 6 | magenta="\[\e[35m\]" 7 | PS1="${bold}nextstrain:${magenta}\w${reset} ${bold}\$ ${reset}" 8 | unset reset bold magenta 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := bash -euo pipefail 2 | 3 | .PHONY: local-image test 4 | 5 | local-image: 6 | ./devel/start-localhost-registry 7 | ./devel/build 8 | 9 | # Don't pull base-builder image since it's huge and not needed for local 10 | # inspection most of the time. 11 | docker image pull localhost:5000/nextstrain/base:latest 12 | 13 | ./devel/stop-localhost-registry 14 | 15 | test: 16 | rm -f tests/*.t.err 17 | cram -v tests/ 18 | 19 | clean: 20 | ./devel/clean 21 | -------------------------------------------------------------------------------- /devel/platform: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Output the Docker platform matching the Docker server's (dockerd's) 4 | # architecture. This is usually the same as the current machine's (e.g. the 5 | # host's, the Docker client's) architecture, but not always as the Docker 6 | # server may be remote. 7 | # 8 | set -euo pipefail 9 | 10 | arch="$(docker info --format '{{.Architecture}}')" 11 | 12 | case "$arch" in 13 | x86_64) echo linux/amd64;; 14 | aarch64) echo linux/arm64;; 15 | *) 16 | echo "unsupported architecture: $arch" >&2 17 | exit 1 18 | esac 19 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Dependabot configuration file 2 | # 3 | # 4 | # Each ecosystem is checked on a scheduled interval defined below. To trigger 5 | # a check manually, go to 6 | # 7 | # https://github.com/nextstrain/docker-base/network/updates 8 | # 9 | # and look for a "Check for updates" button. You may need to click around a 10 | # bit first. 11 | --- 12 | version: 2 13 | updates: 14 | - package-ecosystem: "github-actions" 15 | directory: "/" 16 | schedule: 17 | interval: "weekly" 18 | -------------------------------------------------------------------------------- /devel/pull-from-registry: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Pull the Nextstrain images from a Docker registry. 4 | # 5 | set -euo pipefail 6 | 7 | # Set default values. 8 | registry=localhost:5000 9 | tag=latest 10 | 11 | # Read command-line arguments. 12 | while getopts "r:t:" opt; do 13 | case "$opt" in 14 | r) registry="$OPTARG";; 15 | t) tag="$OPTARG";; 16 | *) echo "Usage: $0 [-r ] [-t ]" 1>&2; exit 1;; 17 | esac 18 | done 19 | 20 | BUILDER_BUILD_PLATFORM_IMAGE=nextstrain/base-builder-build-platform 21 | BUILDER_TARGET_PLATFORM_IMAGE=nextstrain/base-builder-target-platform 22 | FINAL_IMAGE=nextstrain/base 23 | 24 | docker pull "$registry/$BUILDER_BUILD_PLATFORM_IMAGE:$tag" 25 | docker pull "$registry/$BUILDER_TARGET_PLATFORM_IMAGE:$tag" 26 | docker pull "$registry/$FINAL_IMAGE:$tag" 27 | -------------------------------------------------------------------------------- /delete-envd: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | # Optionally remove the contents of /nextstrain/env.d (and any remote archive) 5 | # so values remain only in process memory. We don't remove /nextstrain/env.d 6 | # itself because it might be mounted in from the host system and thus 7 | # undeletable. 8 | case "${NEXTSTRAIN_DELETE_ENVD:-}" in 9 | 1|yes|true) 10 | rm -rf /nextstrain/env.d/* 11 | 12 | if [[ -n "${NEXTSTRAIN_ENVD_URL:-}" ]]; then 13 | case "$NEXTSTRAIN_ENVD_URL" in 14 | s3://*) 15 | aws s3 rm "$NEXTSTRAIN_ENVD_URL" 16 | ;; 17 | *) 18 | echo "delete-envd: No handler for NEXTSTRAIN_ENVD_URL <$NEXTSTRAIN_ENVD_URL>" >&2 19 | exit 1 20 | ;; 21 | esac 22 | fi 23 | ;; 24 | esac 25 | 26 | exec "$@" 27 | -------------------------------------------------------------------------------- /create-envd: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | # Make an empty directory, at the least, so subsequent steps and programs can 5 | # expect it. Doing this at container run time instead of image build time 6 | # means the permissions match the current process. This is made possible by 7 | # the intentionally open, /tmp-like permissions of /nextstrain. 8 | mkdir -p /nextstrain/env.d 9 | 10 | # Populate env.d from a remote archive, if given. 11 | if [[ -n "${NEXTSTRAIN_ENVD_URL:-}" ]]; then 12 | case "$NEXTSTRAIN_ENVD_URL" in 13 | s3://*.zip) 14 | aws s3 cp --no-progress "$NEXTSTRAIN_ENVD_URL" /nextstrain/env.d.zip 15 | unzip -d /nextstrain/env.d{,.zip} 16 | rm -v /nextstrain/env.d.zip 17 | ;; 18 | *) 19 | echo "create-envd: No handler for NEXTSTRAIN_ENVD_URL <$NEXTSTRAIN_ENVD_URL>" >&2 20 | exit 1 21 | ;; 22 | esac 23 | fi 24 | 25 | exec "$@" 26 | -------------------------------------------------------------------------------- /devel/start-localhost-registry: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Starts a Docker registry on localhost using a Docker container. 4 | # 5 | set -euo pipefail 6 | 7 | # Paths to use for registry data. 8 | REPO="$(cd "$(dirname "$0")/.."; pwd)" 9 | DATA="$REPO"/build/registry 10 | mkdir -p "$DATA" 11 | 12 | # Port to run the registry on. If not provided, default to 5000. 13 | PORT="${1:-5000}" 14 | 15 | # Name of the docker container. 16 | NAME=nextstrain-local-registry 17 | 18 | # Docker image that provides the registry service. 19 | IMAGE=registry:2 20 | 21 | if docker container inspect "$NAME" &>/dev/null; then 22 | docker container start "$NAME" 23 | else 24 | docker run \ 25 | --detach \ 26 | --publish 127.0.0.1:"$PORT":"$PORT" \ 27 | --restart always \ 28 | --name "$NAME" \ 29 | --volume "$DATA":/var/lib/registry \ 30 | --user "$(id -u):$(id -g)" \ 31 | "$IMAGE" \ 32 | > /dev/null 33 | fi 34 | -------------------------------------------------------------------------------- /tests/basic.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cram 2 | 3 | Setup. 4 | 5 | $ : "${IMAGE:=localhost:5000/nextstrain/base:latest}" 6 | $ (docker image inspect "$IMAGE" || docker image pull "$IMAGE") &>/dev/null 7 | 8 | Default working directory. 9 | 10 | $ docker run --rm "$IMAGE" pwd 11 | /nextstrain/build 12 | 13 | Default home directory. 14 | 15 | $ docker run --rm "$IMAGE" env | grep ^HOME= 16 | HOME=/nextstrain 17 | 18 | Drops default root privileges. 19 | 20 | $ docker run --rm "$IMAGE" id 21 | uid=*(nextstrain) gid=*(nextstrain) groups=*(nextstrain) (glob) 22 | 23 | Respects externally-set user/group. 24 | 25 | $ docker run --rm --user 1234:5678 "$IMAGE" id 26 | uid=1234 gid=5678 groups=5678 27 | 28 | /nextstrain has open, /tmp-like permissions. 29 | 30 | $ docker run --rm "$IMAGE" ls -ld /nextstrain 31 | drwxrwxrwt * /nextstrain (glob) 32 | 33 | /nextstrain/build is writable by default nextstrain user. 34 | 35 | $ docker run --rm "$IMAGE" touch /nextstrain/build/test 36 | -------------------------------------------------------------------------------- /tests/aws-batch-endpoint-url.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cram 2 | 3 | Setup. 4 | 5 | $ : "${IMAGE:=localhost:5000/nextstrain/base:latest}" 6 | $ (docker image inspect "$IMAGE" || docker image pull "$IMAGE") &>/dev/null 7 | 8 | $ export NEXTSTRAIN_AWS_BATCH_VERBOSE=1 9 | 10 | --endpoint-url is used when AWS_ENDPOINT_URL is available. 11 | 12 | $ export NEXTSTRAIN_AWS_BATCH_WORKDIR_URL="s3://dummy-value/dummy-value.zip" 13 | $ export AWS_ENDPOINT_URL="https://custom-endpoint.example.com" 14 | 15 | $ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE} --env=AWS_ENDPOINT_URL "$IMAGE" \ 16 | > /sbin/entrypoint-aws-batch true 2>&1 | grep --only-matching 'aws s3 cp --endpoint-url [^ ]*' 17 | aws s3 cp --endpoint-url https://custom-endpoint.example.com 18 | 19 | --endpoint-url is not used when AWS_ENDPOINT_URL is unavailable. 20 | 21 | $ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE} "$IMAGE" \ 22 | > /sbin/entrypoint-aws-batch true 2>&1 | grep --only-matching 'aws s3 cp --no-progress [^ ]*' 23 | aws s3 cp --no-progress s3://dummy-value/dummy-value.zip 24 | -------------------------------------------------------------------------------- /entrypoint: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | if [[ $# -eq 0 ]]; then 5 | # If the container is run with no arguments, then print a message and exit. 6 | cut -c 9- <<<" 7 | This is the Nextstrain pathogen build container. It has a complete 8 | Nextstrain environment installed which is normally accessed via the 9 | \`nextstrain\` command on your computer (not in the container). 10 | 11 | If you're seeing this message though, you're probably not using the 12 | \`nextstrain\` command. Welcome! :-) 13 | 14 | The default container entrypoint will exec a command-line of your 15 | choosing in the build environment. You can use this functionality to 16 | debug or develop Nextstrain pathogen builds and components. Refer to 17 | the source of the \`nextstrain\` command for examples. 18 | " 19 | else 20 | # Otherwise, exec chain into whatever command is provided. 21 | exec \ 22 | drop-privs \ 23 | create-envd \ 24 | envdir /nextstrain/env.d \ 25 | delete-envd \ 26 | chdir-workdir \ 27 | "$@" 28 | fi 29 | -------------------------------------------------------------------------------- /tests/workdir.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cram 2 | 3 | Setup. 4 | 5 | $ : "${IMAGE:=localhost:5000/nextstrain/base:latest}" 6 | $ (docker image inspect "$IMAGE" || docker image pull "$IMAGE") &>/dev/null 7 | 8 | NEXTSTRAIN_WORKDIR changes initial working directory. 9 | 10 | $ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nextstrain/augur "$IMAGE" \ 11 | > bash -eu -c 'echo "$PWD"' 12 | /nextstrain/augur 13 | 14 | $ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nextstrain/augur -u 1234:5678 "$IMAGE" \ 15 | > bash -eu -c 'echo "$PWD"' 16 | /nextstrain/augur 17 | 18 | Missing directories are created, like with the `--workdir` option of `docker run`. 19 | 20 | $ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nextstrain/a/b/c "$IMAGE" \ 21 | > bash -eu -c 'ls -ld "$PWD"' 22 | drwxr-xr-x * nextstrain nextstrain * /nextstrain/a/b/c (glob) 23 | 24 | $ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nextstrain/a/b/c -u 1234:5678 "$IMAGE" \ 25 | > bash -eu -c 'ls -ld "$PWD"' 26 | drwxr-xr-x * 1234 5678 * /nextstrain/a/b/c (glob) 27 | 28 | …but permissions still apply, as the `mkdir` happens after drop-privs. 29 | 30 | $ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nope "$IMAGE" \ 31 | > bash -eu -c 'echo "$PWD"' 32 | mkdir: cannot create directory ‘/nope’: Permission denied 33 | [1] 34 | 35 | $ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nope -u 1234:5678 "$IMAGE" \ 36 | > bash -eu -c 'echo "$PWD"' 37 | mkdir: cannot create directory ‘/nope’: Permission denied 38 | [1] 39 | -------------------------------------------------------------------------------- /tests/envdir.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cram 2 | 3 | Setup. 4 | 5 | $ : "${IMAGE:=localhost:5000/nextstrain/base:latest}" 6 | $ (docker image inspect "$IMAGE" || docker image pull "$IMAGE") &>/dev/null 7 | 8 | $ mkdir env.d 9 | $ chmod a+rx env.d 10 | $ echo AAAA > env.d/a 11 | $ echo BB BB BB > env.d/b 12 | $ touch env.d/z 13 | 14 | Bind-mounted env.d. 15 | 16 | $ docker run --rm -v "$PWD"/env.d:/nextstrain/env.d "$IMAGE" \ 17 | > bash -eu -c 'echo "$a"' 18 | AAAA 19 | 20 | $ docker run --rm -v "$PWD"/env.d:/nextstrain/env.d "$IMAGE" \ 21 | > bash -eu -c 'echo "$b"' 22 | BB BB BB 23 | 24 | $ docker run --rm -v "$PWD"/env.d:/nextstrain/env.d "$IMAGE" \ 25 | > bash -eu -c 'echo "$z"' 26 | bash: line 1: z: unbound variable 27 | [1] 28 | 29 | Works with externally-set user/group. 30 | 31 | $ docker run --rm -v "$PWD"/env.d:/nextstrain/env.d -u 1234:5678 "$IMAGE" \ 32 | > bash -eu -c 'echo "$a"' 33 | AAAA 34 | 35 | $ docker run --rm -v "$PWD"/env.d:/nextstrain/env.d -u 1234:5678 "$IMAGE" \ 36 | > bash -eu -c 'echo "$b"' 37 | BB BB BB 38 | 39 | $ docker run --rm -v "$PWD"/env.d:/nextstrain/env.d -u 1234:5678 "$IMAGE" \ 40 | > bash -eu -c 'echo "$z"' 41 | bash: line 1: z: unbound variable 42 | [1] 43 | 44 | Files are removed with NEXTSTRAIN_DELETE_ENVD=1. 45 | 46 | $ ls -1 env.d 47 | a 48 | b 49 | z 50 | 51 | $ docker run --rm -e NEXTSTRAIN_DELETE_ENVD=1 -v "$PWD"/env.d:/nextstrain/env.d -u "$(id -u):$(id -g)" "$IMAGE" \ 52 | > bash -eu -c 'echo "$a"; echo "$b"; ls -1 /nextstrain/env.d' 53 | AAAA 54 | BB BB BB 55 | 56 | $ ls -1 env.d 57 | -------------------------------------------------------------------------------- /tests/aws-batch-verbose.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cram 2 | 3 | Setup. 4 | 5 | $ : "${IMAGE:=localhost:5000/nextstrain/base:latest}" 6 | $ (docker image inspect "$IMAGE" || docker image pull "$IMAGE") &>/dev/null 7 | 8 | A workdir URL is required and not setting one here causes the entrypoint to 9 | error, but that's enough for testing verbose mode. 10 | 11 | $ export NEXTSTRAIN_AWS_BATCH_WORKDIR_URL= 12 | 13 | Verbose mode is default. 14 | 15 | $ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_WORKDIR_URL "$IMAGE" \ 16 | > /sbin/entrypoint-aws-batch true 17 | + case "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" in 18 | + echo 'entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <>' 19 | entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <> 20 | + exit 1 21 | [1] 22 | 23 | Verbose mode is anything not zero. 24 | 25 | $ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE=yes} "$IMAGE" \ 26 | > /sbin/entrypoint-aws-batch true 27 | + case "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" in 28 | + echo 'entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <>' 29 | entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <> 30 | + exit 1 31 | [1] 32 | 33 | $ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE=} "$IMAGE" \ 34 | > /sbin/entrypoint-aws-batch true 35 | + case "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" in 36 | + echo 'entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <>' 37 | entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <> 38 | + exit 1 39 | [1] 40 | 41 | Verbose mode can be turned off. 42 | 43 | $ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE=0} "$IMAGE" \ 44 | > /sbin/entrypoint-aws-batch true 45 | entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <> 46 | [1] 47 | -------------------------------------------------------------------------------- /devel/clean: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Cleans up after the build process by removing build artifacts, caches, and logs. 4 | # 5 | set -euo pipefail 6 | 7 | # Set defaults and fixed values. 8 | repo="$(cd "$(dirname "$0")/.."; pwd)" 9 | registry=nextstrain-local-registry 10 | registry_addr=localhost:5000 11 | build_dir="$repo"/build 12 | builder=nextstrain-builder 13 | build_logs_dir="$repo"/logs 14 | 15 | # Read command-line arguments. 16 | while getopts "r:" opt; do 17 | case "$opt" in 18 | r) registry_addr="$OPTARG";; 19 | *) echo "Usage: $0 [-r ]" 1>&2; exit 1;; 20 | esac 21 | done 22 | 23 | 24 | echo "--> Deleting $registry container" 25 | 26 | if docker container inspect "$registry" &>/dev/null; then 27 | "$repo"/devel/stop-localhost-registry 28 | else 29 | echo "skipped; $registry container does not exist" 30 | fi 31 | 32 | 33 | echo "--> Deleting artifacts in $build_dir" 34 | 35 | if [[ -d "$build_dir" ]]; then 36 | du -hs "$build_dir" 37 | rm -rf "$build_dir" 38 | else 39 | echo "skipped; $build_dir does not exist" 40 | fi 41 | 42 | 43 | echo "--> Deleting $builder (and its caches)" 44 | 45 | if docker buildx inspect "$builder" &>/dev/null; then 46 | docker buildx du --builder "$builder" 47 | docker buildx rm --builder "$builder" 48 | else 49 | echo "skipped; $builder does not exist" 50 | fi 51 | 52 | 53 | echo "--> Deleting local images pulled from $registry_addr" 54 | 55 | for image in "$registry_addr"/nextstrain/base{,-builder}; do 56 | for id in $(docker image ls -q "$image"); do 57 | docker image rm "$id" 58 | done 59 | done 60 | if [[ -z "${id:-}" ]]; then 61 | echo "skipped; no local images" 62 | fi 63 | 64 | 65 | echo "--> Deleting build logs in $build_logs_dir" 66 | 67 | if [[ -d "$build_logs_dir" ]]; then 68 | du -hs "$build_logs_dir" 69 | rm -rf "$build_logs_dir" 70 | else 71 | echo "skipped; $build_logs_dir does not exist" 72 | fi 73 | -------------------------------------------------------------------------------- /tests/envdir-url.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cram 2 | 3 | Setup. 4 | 5 | $ [[ -n "$AWS_ACCESS_KEY_ID" && -n "$AWS_SECRET_ACCESS_KEY" ]] || exit 80 6 | 7 | $ : "${IMAGE:=localhost:5000/nextstrain/base:latest}" 8 | $ (docker image inspect "$IMAGE" || docker image pull "$IMAGE") &>/dev/null 9 | 10 | $ mkdir env.d 11 | $ chmod a+rx env.d 12 | $ echo AAAA > env.d/a 13 | $ echo BB BB BB > env.d/b 14 | $ touch env.d/z 15 | 16 | Files are populated from NEXTSTRAIN_ENVD_URL. 17 | 18 | $ zip --junk-paths env.d{.zip,/*} 19 | adding: a (*) (glob) 20 | adding: b (*) (glob) 21 | adding: z (*) (glob) 22 | 23 | $ export NEXTSTRAIN_ENVD_URL="s3://nextstrain-tmp/$(python3 -c 'import uuid; print(uuid.uuid4())').zip" 24 | 25 | $ aws s3 cp --quiet env.d.zip "$NEXTSTRAIN_ENVD_URL" 26 | 27 | $ docker run --rm -e NEXTSTRAIN_ENVD_URL --env=AWS_{ACCESS_KEY_ID,SECRET_ACCESS_KEY,SESSION_TOKEN} "$IMAGE" \ 28 | > bash -eu -c 'echo "$a"; echo "$b"' 29 | download: s3://nextstrain-tmp/*.zip to ../env.d.zip (glob) 30 | Archive: /nextstrain/env.d.zip 31 | extracting: /nextstrain/env.d/a 32 | inflating: /nextstrain/env.d/b 33 | extracting: /nextstrain/env.d/z 34 | removed '/nextstrain/env.d.zip' 35 | AAAA 36 | BB BB BB 37 | 38 | Files and NEXTSTRAIN_ENVD_URL are removed with NEXTSTRAIN_DELETE_ENVD=1. 39 | 40 | $ docker run --rm -e NEXTSTRAIN_DELETE_ENVD=1 -e NEXTSTRAIN_ENVD_URL --env=AWS_{ACCESS_KEY_ID,SECRET_ACCESS_KEY,SESSION_TOKEN} "$IMAGE" \ 41 | > bash -eu -c 'echo "$a"; echo "$b"; ls -1 /nextstrain/env.d' 42 | download: s3://nextstrain-tmp/*.zip to ../env.d.zip (glob) 43 | Archive: /nextstrain/env.d.zip 44 | extracting: /nextstrain/env.d/a 45 | inflating: /nextstrain/env.d/b 46 | extracting: /nextstrain/env.d/z 47 | removed '/nextstrain/env.d.zip' 48 | delete: s3://nextstrain-tmp/*.zip (glob) 49 | AAAA 50 | BB BB BB 51 | 52 | $ aws s3 ls "$NEXTSTRAIN_ENVD_URL" 53 | [1] 54 | -------------------------------------------------------------------------------- /entrypoint-aws-batch: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | # Show what we're running for the benefit of the logs. 5 | if [[ "${NEXTSTRAIN_AWS_BATCH_VERBOSE:=1}" != 0 ]]; then 6 | set -x 7 | fi 8 | 9 | # Download the working dir. 10 | case "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" in 11 | s3://*.zip) 12 | aws s3 cp ${AWS_ENDPOINT_URL:+--endpoint-url "$AWS_ENDPOINT_URL"} --no-progress "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" "$PWD.zip" 13 | 14 | for dir in /nextstrain/{augur,auspice,fauna}; do 15 | relative_dir="$(realpath "$dir" --relative-to="$PWD")"/ 16 | 17 | if zipinfo -1 "$PWD.zip" "$relative_dir" &>/dev/null; then 18 | echo "removing $dir because workdir ZIP contains $relative_dir overlay" 19 | rm -rf "$dir" 20 | fi 21 | done 22 | 23 | unzip -: -o "$PWD.zip" 24 | ;; 25 | s3://*) 26 | # Note that this doesn't preserve file permissions/modes. 27 | aws s3 ${AWS_ENDPOINT_URL:+--endpoint-url "$AWS_ENDPOINT_URL"} sync --no-progress "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" . 28 | ;; 29 | *) 30 | echo "entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL>" >&2 31 | exit 1 32 | ;; 33 | esac 34 | 35 | # Run the passed command, with the effect of "set -e" temporarily disabled, 36 | # saving the exit status for later. 37 | "$@" && exited=0 || exited=$? 38 | 39 | # Upload the new workdir state with results. 40 | # 41 | # XXX TODO: In the future this may want to be separate from the initial workdir 42 | # state instead of overwriting it. That would let us hash the local workdir 43 | # before uploading and re-use previously uploaded initial workdirs. 44 | # -trs, 13 Sept 2018 45 | case "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" in 46 | s3://*.zip) 47 | succeed-when-nothing-to-update() { 48 | local zipstatus="$1" 49 | if [[ $zipstatus -eq 12 ]]; then 50 | return 0 51 | else 52 | return "$zipstatus" 53 | fi 54 | } 55 | zip -u "$PWD.zip" -r . --exclude ".snakemake/*" || succeed-when-nothing-to-update $? 56 | zip -u "$PWD.zip" -r . --include ".snakemake/log/*" ".snakemake/metadata/*" ".snakemake/storage/*" || succeed-when-nothing-to-update $? 57 | aws s3 cp ${AWS_ENDPOINT_URL:+--endpoint-url "$AWS_ENDPOINT_URL"} --no-progress "$PWD.zip" "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" 58 | ;; 59 | s3://*) 60 | aws s3 sync ${AWS_ENDPOINT_URL:+--endpoint-url "$AWS_ENDPOINT_URL"} --no-progress . "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" --exclude ".snakemake/*" --include ".snakemake/log/*" 61 | ;; 62 | *) 63 | echo "entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL>" >&2 64 | exit 1 65 | ;; 66 | esac 67 | 68 | # Exit with the same status as the passed command. 69 | exit "$exited" 70 | -------------------------------------------------------------------------------- /devel/validate-platforms: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Validate different platform builds of the final Nextstrain image. 4 | # 5 | set -euo pipefail 6 | 7 | # Set default values. 8 | registry=localhost:5000 9 | tag=latest 10 | 11 | # Read command-line arguments. 12 | while getopts "r:t:" opt; do 13 | case "$opt" in 14 | r) registry="$OPTARG";; 15 | t) tag="$OPTARG";; 16 | *) echo "Usage: $0 [-r ] [-t ]" 1>&2; exit 1;; 17 | esac 18 | done 19 | 20 | IMAGE="$registry/nextstrain/base:$tag" 21 | PLATFORMS=(linux/amd64 linux/arm64) 22 | 23 | main() { 24 | # Check that every platform image got the same versions of important (e.g. 25 | # first-party) software for which we don't pin a specific version (e.g. we 26 | # install whatever the latest version is at build time). 27 | 28 | local report_dir 29 | report_dir="$(mktemp -dt "$(basename "$0")"-XXXXXX)" 30 | 31 | for platform in "${PLATFORMS[@]}"; do 32 | echo "[$platform] Pulling image..." 33 | docker pull -q --platform "$platform" "$IMAGE" 34 | 35 | echo "[$platform] Checking that the platform is expected..." 36 | check-platform "$platform" 37 | 38 | # Initialize a directory for the report file, ensuring slashes in the 39 | # platform name are subdirs. 40 | report="$report_dir/$platform" 41 | echo "[$platform] Generating report file: $report" 42 | mkdir -p "$(dirname "$report")" 43 | 44 | # Create a report file for the platform. 45 | # This should include all software below ARG CACHE_DATE in the Dockerfile 46 | # in addition to other important software. 47 | echo "[$platform] Determining software versions..." 48 | # shellcheck disable=SC2016 49 | docker-run "$platform" bash -c ' 50 | function echo-command { 51 | echo "$ $BASH_COMMAND" 52 | } 53 | trap echo-command DEBUG 54 | 55 | nextstrain --version 56 | nextalign --version 57 | nextclade --version 58 | augur --version 59 | auspice --version 60 | python3 -c "from importlib.metadata import version; print(version(\"evofr\"))" 61 | datasets --version 62 | dataformat version 63 | 64 | python3 -c "from importlib.metadata import version; print(version(\"phylo-treetime\"))" 65 | ' >"$report" 66 | done 67 | 68 | # Compare contents of the first platform's report file against others. 69 | first_report="$report_dir/${PLATFORMS[0]}" 70 | echo "The report for ${PLATFORMS[0]} has the following contents:" 71 | cat "$first_report" 72 | 73 | echo "Comparing against other platforms..." 74 | # NOTE: if running on macOS ≥13, you may need to install GNU diff for the 75 | # --from-file option. 76 | if cd "$report_dir" && diff --unified=1 --from-file="${PLATFORMS[0]}" "${PLATFORMS[@]:1}"; then 77 | echo "Success! All versions the same." >&2 78 | else 79 | echo "Failure!" >&2 80 | exit 1 81 | fi 82 | } 83 | 84 | check-platform() { 85 | # Check that the platform is actually what we expect it to be. 86 | local platform="$1" 87 | 88 | python_platform_string="$(docker-run "$platform" python -c "import platform; print(platform.platform())")" 89 | 90 | case "$platform" in 91 | linux/amd64) 92 | if [[ "$python_platform_string" != *"x86_64"* ]]; then 93 | echo "Platform $platform not detected." 1>&2; exit 1 94 | fi;; 95 | linux/arm64) 96 | if [[ "$python_platform_string" != *"aarch64"* ]]; then 97 | echo "Platform $platform not detected." 1>&2; exit 1 98 | fi;; 99 | *) 100 | echo "Platform $platform not supported." 1>&2; exit 1;; 101 | esac 102 | } 103 | 104 | docker-run() { 105 | # Run a command under the final Nextstrain image built for a specific 106 | # platform. 107 | local platform="$1" 108 | local command=("${@:2}") 109 | 110 | docker run --rm --platform "$platform" "$IMAGE" "${command[@]}" 111 | } 112 | 113 | main 114 | -------------------------------------------------------------------------------- /devel/copy-images: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copy the Nextstrain images from one Docker registry (-i ) to 4 | # another (-o ). 5 | # 6 | # If authentication is required for a registry, ensure the credentials are 7 | # available in ~/.docker/config.json. 8 | # 9 | # This copies just the tag specified by -t . If the boolean -l flag is 10 | # specified, the tag will also be copied to "latest" on the destnation. 11 | # 12 | set -euo pipefail 13 | 14 | # Set default values. 15 | registry_in=localhost:5000 16 | registry_out=docker.io 17 | tag="" 18 | push_latest=false 19 | 20 | # Read command-line arguments. 21 | while getopts "i:o:t:l" opt; do 22 | case "$opt" in 23 | i) registry_in="$OPTARG";; 24 | o) registry_out="$OPTARG";; 25 | t) tag="$OPTARG";; 26 | l) push_latest=true;; 27 | *) echo "Usage: $0 [-i ] [-o ] [-t ] [-l]" 1>&2; exit 1;; 28 | esac 29 | done 30 | 31 | if [[ "$tag" = "" ]]; then 32 | echo "Please provide a tag." >&2 33 | exit 1 34 | fi 35 | 36 | BUILDER_BUILD_PLATFORM_IMAGE=nextstrain/base-builder-build-platform 37 | BUILDER_TARGET_PLATFORM_IMAGE=nextstrain/base-builder-target-platform 38 | FINAL_IMAGE=nextstrain/base 39 | 40 | 41 | # Use Skopeo via a Docker container¹ to copy a tagged image between registries. 42 | # 43 | # Two positional parameters are required, representing the source and 44 | # destination images each qualified with a Docker registry. 45 | # Format should be /image:tag, e.g. docker.io/nextstrain/base:latest. 46 | # 47 | # If a registry starts with localhost, do not require HTTPS or verify 48 | # certificates, and access the registry without authentication. 49 | # 50 | # ¹ https://github.com/containers/skopeo/blob/07da29fd371dd88615a0b86e91c6824237484172/install.md#container-images 51 | copy-image() { 52 | local src="$1" 53 | local dest="$2" 54 | 55 | docker_run_params=(--rm --network=host) 56 | skopeo_copy_params=(--multi-arch=all) 57 | 58 | if [[ "$src" == localhost* ]]; then 59 | skopeo_copy_params+=(--src-tls-verify=false) 60 | else 61 | docker_run_params+=(-v "$HOME"/.docker/config.json:/config.json:ro) 62 | skopeo_copy_params+=(--src-authfile config.json) 63 | fi 64 | 65 | if [[ "$dest" == localhost* ]]; then 66 | skopeo_copy_params+=(--dest-tls-verify=false) 67 | else 68 | docker_run_params+=(-v "$HOME"/.docker/config.json:/config.json:ro) 69 | skopeo_copy_params+=(--dest-authfile config.json) 70 | fi 71 | 72 | docker run "${docker_run_params[@]}" \ 73 | quay.io/skopeo/stable \ 74 | copy "${skopeo_copy_params[@]}" \ 75 | "docker://$src" \ 76 | "docker://$dest" 77 | } 78 | 79 | # Copy $tag between registries. 80 | 81 | echo "Copying $registry_in/$BUILDER_BUILD_PLATFORM_IMAGE:$tag to $registry_out/$BUILDER_BUILD_PLATFORM_IMAGE:$tag." 82 | copy-image \ 83 | "$registry_in/$BUILDER_BUILD_PLATFORM_IMAGE:$tag" \ 84 | "$registry_out/$BUILDER_BUILD_PLATFORM_IMAGE:$tag" 85 | 86 | echo "Copying $registry_in/$BUILDER_TARGET_PLATFORM_IMAGE:$tag to $registry_out/$BUILDER_TARGET_PLATFORM_IMAGE:$tag." 87 | copy-image \ 88 | "$registry_in/$BUILDER_TARGET_PLATFORM_IMAGE:$tag" \ 89 | "$registry_out/$BUILDER_TARGET_PLATFORM_IMAGE:$tag" 90 | 91 | echo "Copying $registry_in/$FINAL_IMAGE:$tag to $registry_out/$FINAL_IMAGE:$tag." 92 | copy-image \ 93 | "$registry_in/$FINAL_IMAGE:$tag" \ 94 | "$registry_out/$FINAL_IMAGE:$tag" 95 | 96 | if [[ "$push_latest" = true ]]; then 97 | # Copy $tag to latest. 98 | 99 | echo "Copying $registry_in/$BUILDER_BUILD_PLATFORM_IMAGE:$tag to $registry_out/$BUILDER_BUILD_PLATFORM_IMAGE:latest." 100 | copy-image \ 101 | "$registry_in/$BUILDER_BUILD_PLATFORM_IMAGE:$tag" \ 102 | "$registry_out/$BUILDER_BUILD_PLATFORM_IMAGE:latest" 103 | 104 | echo "Copying $registry_in/$BUILDER_TARGET_PLATFORM_IMAGE:$tag to $registry_out/$BUILDER_TARGET_PLATFORM_IMAGE:latest." 105 | copy-image \ 106 | "$registry_in/$BUILDER_TARGET_PLATFORM_IMAGE:$tag" \ 107 | "$registry_out/$BUILDER_TARGET_PLATFORM_IMAGE:latest" 108 | 109 | echo "Copying $registry_in/$FINAL_IMAGE:$tag to $registry_out/$FINAL_IMAGE:latest." 110 | copy-image \ 111 | "$registry_in/$FINAL_IMAGE:$tag" \ 112 | "$registry_out/$FINAL_IMAGE:latest" 113 | fi 114 | -------------------------------------------------------------------------------- /devel/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Builds the nextstrain/base and nextstrain/base-builder images with useful 4 | # caching and pushes to a registry. 5 | # 6 | # By default this tags images using "latest" and pushes to localhost:5000, but 7 | # you can provide a custom tag with -t and specify a different 8 | # registry with -r . 9 | # 10 | # Set CACHE_DATE in your environment to force layers after our custom cache 11 | # point to be re-built. See the ARG CACHE_DATE line in the Dockerfile for more 12 | # information. 13 | # 14 | set -euo pipefail 15 | 16 | devel="$(dirname "$0")" 17 | 18 | # Set default values. 19 | platform="$("$devel"/platform)" 20 | registry=localhost:5000 21 | tag=latest 22 | log_directory="logs/" 23 | 24 | # Read command-line arguments. 25 | while getopts "p:r:t:l:" opt; do 26 | case "$opt" in 27 | p) platform="$OPTARG";; 28 | r) registry="$OPTARG";; 29 | t) tag="$OPTARG";; 30 | l) log_directory="$OPTARG";; 31 | *) echo "Usage: $0 [-p ] [-r ] [-t ] [-l ]" 1>&2; exit 1;; 32 | esac 33 | done 34 | 35 | # shellcheck disable=SC2155 36 | export GIT_REVISION=$(git describe --tags --abbrev=40 --always --dirty || true) 37 | 38 | # The nextstrain/base Dockerfile is a multi-stage with both a "builder" target 39 | # and a main target. To enable proper caching via --cache-from we need both 40 | # these images available to pull layers from. This means pulling both in at 41 | # the start and pushing both up at the end. 42 | # 43 | # Calling `docker run nextstrain/base` will still only pull down the small base 44 | # image rather than pulling down the larger nextstrain/base-builder image. 45 | 46 | # `buildx create` is necessary to use a driver that supports multi-platform 47 | # images. 48 | builder=nextstrain-builder 49 | 50 | if ! docker buildx inspect "$builder" &>/dev/null; then 51 | # Using a persistent builder allows for faster local development. 52 | # However, if this is changed and it was previously run on your machine, 53 | # you may need to remove the builder manually before running the script: 54 | # docker buildx rm "nextstrain-builder" 55 | docker buildx create --name "$builder" --driver docker-container --driver-opt network=host 56 | fi 57 | 58 | mkdir -p "$log_directory" 59 | 60 | BUILDER_BUILD_PLATFORM_IMAGE=nextstrain/base-builder-build-platform 61 | BUILDER_TARGET_PLATFORM_IMAGE=nextstrain/base-builder-target-platform 62 | FINAL_IMAGE=nextstrain/base 63 | 64 | docker buildx build \ 65 | --target builder-build-platform \ 66 | --builder "$builder" \ 67 | --platform "$platform" \ 68 | --build-arg CACHE_DATE \ 69 | --cache-from "$BUILDER_BUILD_PLATFORM_IMAGE:latest" \ 70 | --cache-from "$BUILDER_BUILD_PLATFORM_IMAGE:$tag" \ 71 | --cache-from "$registry/$BUILDER_BUILD_PLATFORM_IMAGE:latest" \ 72 | --cache-from "$registry/$BUILDER_BUILD_PLATFORM_IMAGE:$tag" \ 73 | --cache-to type=inline \ 74 | --tag "$registry/$BUILDER_BUILD_PLATFORM_IMAGE:$tag" \ 75 | --push \ 76 | --provenance false \ 77 | --progress=plain \ 78 | . 2>&1 | tee "$log_directory"/builder-build-platform 79 | 80 | 81 | docker buildx build \ 82 | --target builder-target-platform \ 83 | --builder "$builder" \ 84 | --platform "$platform" \ 85 | --build-arg CACHE_DATE \ 86 | --cache-from "$BUILDER_TARGET_PLATFORM_IMAGE:latest" \ 87 | --cache-from "$BUILDER_TARGET_PLATFORM_IMAGE:$tag" \ 88 | --cache-from "$registry/$BUILDER_TARGET_PLATFORM_IMAGE:latest" \ 89 | --cache-from "$registry/$BUILDER_TARGET_PLATFORM_IMAGE:$tag" \ 90 | --cache-to type=inline \ 91 | --tag "$registry/$BUILDER_TARGET_PLATFORM_IMAGE:$tag" \ 92 | --push \ 93 | --provenance false \ 94 | --progress=plain \ 95 | . 2>&1 | tee "$log_directory"/builder-target-platform 96 | 97 | docker buildx build \ 98 | --target final \ 99 | --builder "$builder" \ 100 | --platform "$platform" \ 101 | --build-arg GIT_REVISION \ 102 | --build-arg CACHE_DATE \ 103 | --cache-from "$BUILDER_BUILD_PLATFORM_IMAGE:latest" \ 104 | --cache-from "$BUILDER_BUILD_PLATFORM_IMAGE:$tag" \ 105 | --cache-from "$BUILDER_TARGET_PLATFORM_IMAGE:latest" \ 106 | --cache-from "$BUILDER_TARGET_PLATFORM_IMAGE:$tag" \ 107 | --cache-from "$FINAL_IMAGE:latest" \ 108 | --cache-from "$FINAL_IMAGE:$tag" \ 109 | --cache-from "$registry/$BUILDER_BUILD_PLATFORM_IMAGE:latest" \ 110 | --cache-from "$registry/$BUILDER_BUILD_PLATFORM_IMAGE:$tag" \ 111 | --cache-from "$registry/$BUILDER_TARGET_PLATFORM_IMAGE:latest" \ 112 | --cache-from "$registry/$BUILDER_TARGET_PLATFORM_IMAGE:$tag" \ 113 | --cache-from "$registry/$FINAL_IMAGE:latest" \ 114 | --cache-from "$registry/$FINAL_IMAGE:$tag" \ 115 | --cache-to type=inline \ 116 | --tag "$registry/$FINAL_IMAGE:$tag" \ 117 | --push \ 118 | --provenance false \ 119 | --progress=plain \ 120 | . 2>&1 | tee "$log_directory"/final 121 | -------------------------------------------------------------------------------- /devel/summarize-buildkit-output: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import re 3 | import sys 4 | """ 5 | Summarize BuildKit output by layer execution time. 6 | """ 7 | 8 | 9 | def main(file): 10 | with open(file) as f: 11 | id_map = get_id_map(f) 12 | print_summary(id_map) 13 | 14 | 15 | def get_id_map(f): 16 | """Return information extracted from a BuildKit output file handle. 17 | 18 | Returns 19 | ------- 20 | { 21 | : { 22 | "line": The full layer start line from the output. 23 | "name": A shorter version of the line that should still uniquely identify the layer. 24 | "time": The build time for the layer, or None if it was cached. 25 | } 26 | } 27 | """ 28 | start_pattern = re.compile(r'^(?P#\d+) \[(?P.*?)\] (?P.*)$') 29 | cached_pattern = re.compile(r'^(?P#\d+) CACHED$') 30 | done_pattern = re.compile(r'^(?P#\d+) DONE (?P