├── .eslintrc.js ├── .gitignore ├── .hadolint.yaml ├── Dockerfile.build ├── LICENSE ├── README.markdown ├── concourse ├── .gitignore ├── docker-compose.yml ├── keys │ ├── generate │ ├── web │ │ └── .gitkeep │ └── worker │ │ └── .gitkeep └── run ├── go ├── go.variables ├── package.json ├── pipeline.generated.yml ├── pipeline.jsonnet ├── pipeline └── tasks │ ├── build │ ├── task.sh │ └── task.yml │ ├── linter │ ├── task.sh │ └── task.yml │ ├── serverspec │ └── task.yml │ ├── tests │ ├── task.sh │ └── task.yml │ └── update-pipeline │ └── task.yml ├── scripts └── .gitkeep ├── serverspec ├── .ruby-version ├── Dockerfile.serverspec ├── Gemfile ├── Gemfile.lock ├── entrypoint.sh ├── run └── spec │ ├── dev_container_spec.rb │ └── spec_helper.rb ├── src ├── .gitkeep ├── main.css ├── main.js └── main.spec.js └── yarn.lock /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "extends": "standard", 3 | "plugins": ["jest"], 4 | "env": { 5 | "jest/globals": true 6 | } 7 | }; 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | -------------------------------------------------------------------------------- /.hadolint.yaml: -------------------------------------------------------------------------------- 1 | ignored: 2 | - DL3013 3 | - DL3008 4 | - DL3018 5 | - DL3028 6 | -------------------------------------------------------------------------------- /Dockerfile.build: -------------------------------------------------------------------------------- 1 | FROM node:14.2-buster 2 | 3 | ENV CONCOURSE_SHA1='45c0af130299dfa9769d0eae9ee96ceecfbfd7bf' \ 4 | CONCOURSE_VERSION='6.5.1' \ 5 | HADOLINT_VERSION='v1.10.4' \ 6 | HADOLINT_SHA256='66815d142f0ed9b0ea1120e6d27142283116bf26' 7 | 8 | SHELL ["/bin/bash", "-o", "pipefail", "-c"] 9 | 10 | RUN apt-get update && \ 11 | apt-get -y install --no-install-recommends sudo curl shellcheck && \ 12 | curl -Lk "https://github.com/concourse/concourse/releases/download/v${CONCOURSE_VERSION}/fly-${CONCOURSE_VERSION}-linux-amd64.tgz" -o fly.tgz && \ 13 | tar xzvf fly.tgz && \ 14 | mv fly /usr/bin/fly && \ 15 | echo "${CONCOURSE_SHA1} fly.tgz" | sha1sum -c - && \ 16 | chmod +x /usr/bin/fly && \ 17 | rm fly.tgz && \ 18 | curl -Lk "https://github.com/hadolint/hadolint/releases/download/${HADOLINT_VERSION}/hadolint-Linux-x86_64" -o /usr/bin/hadolint && \ 19 | echo "${HADOLINT_SHA256} /usr/bin/hadolint" | sha1sum -c - && \ 20 | chmod +x /usr/bin/hadolint && \ 21 | apt-get clean && \ 22 | rm -rf /var/lib/apt/lists/* 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 hceris.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # Example Concourse Pipeline 2 | 3 | This is an example for a pipeline for [Concourse CI](https://concourse-ci.org/). It is intended to be used as a base to build high quality pipelines. 4 | 5 | It is using a sample JavaScript project, but it can be easily adapted to serve any other language. 6 | 7 | ## Running a local concourse 8 | 9 | Inspired by [this](https://github.com/concourse/concourse-docker), I've added a [script](./concourse/run) to run an instance of _Concourse_ locally for testing purposes. Just do: 10 | 11 | ```shell 12 | ./concourse/run 13 | ``` 14 | 15 | It is available under `localhost:8080` (test/test for auth) 16 | 17 | ### Adding the pipeline to the local installation 18 | 19 | A local docker registry is running so that the development image can be stored. Add the pipeline to the local installation with 20 | 21 | ```shell 22 | ./go update-pipeline 23 | ``` 24 | 25 | ### Changing the pipeline 26 | 27 | Note that the pipeline is generated dynamically by using [jsonnet](https://jsonnet.org/). See [this blog post](https://hceris.com/templating-concourse-pipelines-with-jsonnet/) for more context about it. 28 | 29 | ## Running things 30 | 31 | There is a `go` script that is the entrypoint of all the tasks. Simply run it without arguments to get a list of available targets. 32 | 33 | ### Requirements 34 | 35 | - `node` (last tested with `v13.6.0`) 36 | - `yarn` (last tested with `1.21.1`) 37 | - `ruby` (last tested with `2.6.3`) 38 | 39 | ## ServerSpec 40 | 41 | [ServerSpec](https://serverspec.org/) allows you to run unit tests for infrastructure. You can use it to test that your containers, both build and production, are being created correctly. 42 | 43 | Running it in a pipeline is notoriously difficult, and you end up running in the [Docker in Docker issue](https://jpetazzo.github.io/2015/09/03/do-not-use-docker-in-docker-for-ci/). This repository contains some ready-to-use blocks that can make your life easier. 44 | 45 | In order to run these tests, you need: 46 | 47 | - A [Docker image](./serverspec/Dockerfile.serverspec) that can run docker-in-docker and has `ruby` installed 48 | - [Building that image](./pipeline.yml#L30-L34) as part of the pipeline 49 | - A [task](./pipeline/tasks/serverspec.yml) that runs that image with [elevated privileges](./pipeline.yml#L36-L41) 50 | - That script requires a special [entrypoint](./serverspec/entrypoint.sh) 51 | - The [run](./serverspec/run) script itself is making sure the image we built is accessible and running [all the tests](./serverspec/spec) that we have defined 52 | 53 | -------------------------------------------------------------------------------- /concourse/.gitignore: -------------------------------------------------------------------------------- 1 | keys/web/authorized_worker_keys 2 | keys/web/session_signing_key 3 | keys/web/tsa_host_key 4 | keys/web/tsa_host_key.pub 5 | keys/web/authorized_worker_keys 6 | keys/worker/worker_key 7 | keys/worker/worker_key.pub 8 | keys/worker/tsa_host_key.pub 9 | -------------------------------------------------------------------------------- /concourse/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | db: 5 | image: postgres 6 | 7 | environment: 8 | POSTGRES_DB: concourse 9 | POSTGRES_USER: concourse_user 10 | POSTGRES_PASSWORD: concourse_pass 11 | 12 | logging: 13 | driver: "json-file" 14 | options: 15 | max-file: "5" 16 | max-size: "10m" 17 | 18 | web: 19 | image: concourse/concourse:6.5.1-ubuntu 20 | command: web 21 | 22 | depends_on: 23 | - db 24 | - registry 25 | 26 | ports: ["8080:8080"] 27 | volumes: ["./keys/web:/concourse-keys"] 28 | 29 | environment: 30 | CONCOURSE_EXTERNAL_URL: http://localhost:8080 31 | CONCOURSE_POSTGRES_HOST: db 32 | CONCOURSE_POSTGRES_USER: concourse_user 33 | CONCOURSE_POSTGRES_PASSWORD: concourse_pass 34 | CONCOURSE_POSTGRES_DATABASE: concourse 35 | CONCOURSE_ADD_LOCAL_USER: test:test 36 | CONCOURSE_MAIN_TEAM_LOCAL_USER: test 37 | logging: 38 | driver: "json-file" 39 | options: 40 | max-file: "5" 41 | max-size: "10m" 42 | 43 | worker: 44 | image: concourse/concourse:6.5.1-ubuntu 45 | command: worker 46 | privileged: true 47 | 48 | depends_on: [web] 49 | 50 | volumes: ["./keys/worker:/concourse-keys"] 51 | 52 | stop_signal: SIGUSR2 53 | 54 | environment: 55 | CONCOURSE_TSA_HOST: web:2222 56 | CONCOURSE_GARDEN_DNS_PROXY_ENABLE: "true" 57 | 58 | logging: 59 | driver: "json-file" 60 | options: 61 | max-file: "5" 62 | max-size: "10m" 63 | 64 | registry: 65 | image: registry 66 | 67 | ports: ["5000:5000"] 68 | -------------------------------------------------------------------------------- /concourse/keys/generate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e -u 4 | 5 | cd $(dirname $0) 6 | 7 | docker run --rm -v $PWD/web:/keys concourse/concourse \ 8 | generate-key -t rsa -f /keys/session_signing_key 9 | 10 | docker run --rm -v $PWD/web:/keys concourse/concourse \ 11 | generate-key -t ssh -f /keys/tsa_host_key 12 | 13 | docker run --rm -v $PWD/worker:/keys concourse/concourse \ 14 | generate-key -t ssh -f /keys/worker_key 15 | 16 | cp ./worker/worker_key.pub ./web/authorized_worker_keys 17 | cp ./web/tsa_host_key.pub ./worker -------------------------------------------------------------------------------- /concourse/keys/web/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sirech/example-concourse-pipeline/666c46fbd776ab71ad87ecf0621e8cbdb90ccbcd/concourse/keys/web/.gitkeep -------------------------------------------------------------------------------- /concourse/keys/worker/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sirech/example-concourse-pipeline/666c46fbd776ab71ad87ecf0621e8cbdb90ccbcd/concourse/keys/worker/.gitkeep -------------------------------------------------------------------------------- /concourse/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e -u 4 | 5 | cd $(dirname $0) 6 | 7 | ./keys/generate 8 | docker-compose up -------------------------------------------------------------------------------- /go: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -o nounset 5 | set -o pipefail 6 | 7 | SCRIPT_DIR=$(cd "$(dirname "$0")" ; pwd -P) 8 | 9 | # shellcheck source=./go.variables 10 | source "${SCRIPT_DIR}/go.variables" 11 | 12 | goal_login-pipeline() { 13 | pushd "${SCRIPT_DIR}" > /dev/null 14 | fly --target "${CONCOURSE_TARGET}" login \ 15 | --concourse-url "${CONCOURSE_URL}" \ 16 | --username "${CONCOURSE_USER}" \ 17 | --password "${CONCOURSE_PASSWORD}" 18 | popd > /dev/null 19 | } 20 | 21 | goal_validate-pipeline() { 22 | pushd "${SCRIPT_DIR}" > /dev/null 23 | fly validate-pipeline -c "${PIPELINE_FILE}" 24 | popd > /dev/null 25 | } 26 | 27 | goal_update-pipeline() { 28 | pushd "${SCRIPT_DIR}" > /dev/null 29 | goal_login-pipeline 30 | fly --target "${CONCOURSE_TARGET}" sync 31 | fly --target "${CONCOURSE_TARGET}" set-pipeline \ 32 | --non-interactive \ 33 | --pipeline "${PIPELINE_NAME}" \ 34 | --config "${PIPELINE_FILE}" 35 | popd > /dev/null 36 | } 37 | 38 | json2yaml() { 39 | python3 -c 'import sys, yaml, json; print(yaml.dump(json.loads(sys.stdin.read())))' 40 | } 41 | 42 | goal_generate-pipeline() { 43 | FILES=$(jsonnet pipeline.jsonnet -J ../concourse-jsonnet-utils -m .) 44 | 45 | for file in $FILES; do 46 | json2yaml < "$file" > "${file%.json}.yml" 47 | rm "$file" 48 | done 49 | } 50 | 51 | goal_linter-sh() { 52 | shellcheck go* 53 | } 54 | 55 | goal_linter-js() { 56 | npm run linter:js 57 | } 58 | 59 | goal_linter-css() { 60 | npm run linter:css 61 | } 62 | 63 | goal_linter-docker() { 64 | dockerfiles=$(find . -name 'Dockerfile*' -not -path './node_modules/*' -print | tr '\n' ' ') 65 | # shellcheck disable=SC2086 66 | hadolint ${dockerfiles} 67 | } 68 | 69 | goal_test-js() { 70 | npm test 71 | } 72 | 73 | goal_build() { 74 | npm run build 75 | } 76 | 77 | validate-args() { 78 | acceptable_args="$(declare -F | sed -n "s/declare -f goal_//p" | tr '\n' ' ')" 79 | 80 | if [[ -z $1 ]]; then 81 | echo "usage: $0 " 82 | # shellcheck disable=SC1117,SC2059 83 | printf "\n$(declare -F | sed -n "s/declare -f goal_/ - /p")" 84 | exit 1 85 | fi 86 | 87 | if [[ ! " $acceptable_args " =~ .*\ $1\ .* ]]; then 88 | echo "Invalid argument: $1" 89 | # shellcheck disable=SC1117,SC2059 90 | printf "\n$(declare -F | sed -n "s/declare -f goal_/ - /p")" 91 | exit 1 92 | fi 93 | } 94 | 95 | CMD=${1:-} 96 | shift || true 97 | if validate-args "${CMD}"; then 98 | "goal_${CMD}" 99 | exit 0 100 | fi 101 | -------------------------------------------------------------------------------- /go.variables: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -o nounset 5 | set -o pipefail 6 | 7 | # This is the domain where concourse runs 8 | export CONCOURSE_URL=${CONCOURSE_URL:-http://localhost:8080} 9 | # This is the name used for fly to identify the instance 10 | export CONCOURSE_TARGET=example-concourse 11 | # The name of the pipeline to create 12 | export PIPELINE_NAME="example-concourse-pipeline" 13 | # The file where the pipeline is located 14 | export PIPELINE_FILE="pipeline.generated.yml" 15 | 16 | # Name of the user used for login 17 | export CONCOURSE_USER=test 18 | # Password for the user. This should *NOT* be written down here outside local development 19 | export CONCOURSE_PASSWORD=test 20 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "example-concourse-pipeline", 3 | "version": "1.0.0", 4 | "private": true, 5 | "dependencies": { 6 | "express": "4.17.1" 7 | }, 8 | "devDependencies": { 9 | "eslint": "^7.32.0", 10 | "eslint-config-standard": "^16.0.3", 11 | "eslint-plugin-import": "^2.24.2", 12 | "eslint-plugin-jest": "^24.4.2", 13 | "eslint-plugin-node": "^11.1.0", 14 | "eslint-plugin-promise": "^5.1.0", 15 | "eslint-plugin-standard": "^5.0.0", 16 | "jest": "^27.2.2", 17 | "stylelint": "^13.13.1", 18 | "stylelint-config-recommended": "^5.0.0" 19 | }, 20 | "scripts": { 21 | "test": "jest", 22 | "linter:js": "node_modules/.bin/eslint src --ext jsx --ext js", 23 | "linter:css": "node_modules/.bin/stylelint \"src/**/*.css\"", 24 | "build": "echo 'Built'" 25 | }, 26 | "stylelint": { 27 | "extends": "stylelint-config-recommended" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /pipeline.generated.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | - name: prepare 3 | plan: 4 | - get: git 5 | trigger: true 6 | - in_parallel: 7 | - get: serverspec-container 8 | trigger: true 9 | - params: 10 | build: git 11 | cache: true 12 | cache_tag: latest 13 | dockerfile: git/Dockerfile.build 14 | tag: git/.git/HEAD 15 | tag_as_latest: true 16 | put: dev-container 17 | - file: git/pipeline/tasks/update-pipeline/task.yml 18 | image: dev-container 19 | params: 20 | CI: true 21 | CONCOURSE_PASSWORD: test 22 | CONCOURSE_URL: http://web:8080 23 | CONCOURSE_USER: test 24 | task: update-pipeline 25 | serial: true 26 | - name: lint 27 | plan: 28 | - in_parallel: 29 | - get: git 30 | passed: 31 | - prepare 32 | trigger: true 33 | - get: dev-container 34 | passed: 35 | - prepare 36 | trigger: true 37 | - in_parallel: 38 | - file: git/pipeline/tasks/linter/task.yml 39 | image: dev-container 40 | params: 41 | CI: true 42 | TARGET: sh 43 | task: lint-sh 44 | - file: git/pipeline/tasks/linter/task.yml 45 | image: dev-container 46 | params: 47 | CI: true 48 | TARGET: js 49 | task: lint-js 50 | - file: git/pipeline/tasks/linter/task.yml 51 | image: dev-container 52 | params: 53 | CI: true 54 | TARGET: css 55 | task: lint-css 56 | - file: git/pipeline/tasks/linter/task.yml 57 | image: dev-container 58 | params: 59 | CI: true 60 | TARGET: docker 61 | task: lint-docker 62 | serial: true 63 | - name: test 64 | plan: 65 | - in_parallel: 66 | - get: git 67 | passed: 68 | - prepare 69 | trigger: true 70 | - get: dev-container 71 | passed: 72 | - prepare 73 | trigger: true 74 | - file: git/pipeline/tasks/tests/task.yml 75 | image: dev-container 76 | params: 77 | CI: true 78 | TARGET: js 79 | task: test-js 80 | serial: true 81 | - name: build 82 | plan: 83 | - in_parallel: 84 | - get: git 85 | passed: 86 | - lint 87 | - test 88 | trigger: true 89 | - get: dev-container 90 | passed: 91 | - lint 92 | - test 93 | trigger: true 94 | - file: git/pipeline/tasks/build/task.yml 95 | image: dev-container 96 | params: 97 | CI: true 98 | task: build 99 | serial: true 100 | resources: 101 | - name: git 102 | source: 103 | branch: master 104 | uri: https://github.com/sirech/example-concourse-pipeline.git 105 | type: git 106 | - name: dev-container 107 | source: 108 | insecure_registries: 109 | - registry:5000 110 | repository: registry:5000/dev-container 111 | tag: latest 112 | type: docker-image 113 | - name: serverspec-container 114 | source: 115 | repository: sirech/dind-ruby 116 | tag: 2.6.3 117 | type: docker-image 118 | 119 | -------------------------------------------------------------------------------- /pipeline.jsonnet: -------------------------------------------------------------------------------- 1 | local source = 'git'; 2 | local container = 'dev-container'; 3 | 4 | local concourse = import 'concourse.libsonnet'; 5 | 6 | local docker_params(name) = { 7 | tag: '%s/.git/HEAD' % [name], 8 | tag_as_latest: true, 9 | cache: true, 10 | cache_tag: 'latest' 11 | }; 12 | 13 | local Inputs(dependencies = []) = concourse.Parallel( 14 | [concourse.Get(s, dependencies = dependencies) for s in [source, container]] 15 | ); 16 | 17 | local Task(name, file = name, image = container, params = {}) = { 18 | task: name, 19 | image: image, 20 | params: { CI: true } + params, 21 | file: '%s/pipeline/tasks/%s/task.yml' % [source, file] 22 | }; 23 | 24 | local resources = [ 25 | concourse.GitResource(source, 'https://github.com/sirech/example-concourse-pipeline.git'), 26 | concourse.DockerResource(container, 'registry:5000/dev-container', allow_insecure = true), 27 | concourse.DockerResource('serverspec-container', 'sirech/dind-ruby', tag = '2.6.3'), 28 | ]; 29 | 30 | local jobs = [ 31 | concourse.Job('prepare', plan = [ 32 | concourse.Get(source), 33 | concourse.Parallel([ 34 | concourse.Get('serverspec-container'), 35 | { 36 | put: 'dev-container', 37 | params: docker_params(source) + { 38 | build: source, 39 | dockerfile: '%s/Dockerfile.build' % source 40 | } 41 | } 42 | ]), 43 | Task('update-pipeline', params = { 44 | CONCOURSE_USER: 'test', 45 | CONCOURSE_PASSWORD: 'test', 46 | CONCOURSE_URL: 'http://web:8080' 47 | }) 48 | ]), 49 | 50 | concourse.Job('lint', plan = [ 51 | Inputs('prepare'), 52 | concourse.Parallel( 53 | [Task('lint-%s' % lang, 'linter', params = { TARGET: lang }) for lang in ['sh', 'js', 'css', 'docker']] 54 | ) 55 | ]), 56 | 57 | concourse.Job('test', plan = [ 58 | Inputs('prepare'), 59 | Task('test-js', 'tests', params = { TARGET: 'js' }) 60 | ]), 61 | 62 | concourse.Job('build', plan = [ 63 | Inputs(['lint', 'test']), 64 | Task('build') 65 | ]) 66 | ]; 67 | 68 | { 69 | "pipeline.generated.json": { 70 | resources: resources, 71 | jobs: jobs 72 | } 73 | } + { 74 | ['pipeline/tasks/%s/task.json' % [task]]: concourse.FileTask('pipeline/tasks/%s/task.sh' % [task], inputs = source, caches = '%s/node_modules' % [source]) for task in ['build', 'linter', 'tests'] 75 | } 76 | -------------------------------------------------------------------------------- /pipeline/tasks/build/task.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | yarn 6 | ./go build 7 | -------------------------------------------------------------------------------- /pipeline/tasks/build/task.yml: -------------------------------------------------------------------------------- 1 | caches: 2 | - path: git/node_modules 3 | inputs: 4 | - name: git 5 | platform: linux 6 | run: 7 | dir: git 8 | path: pipeline/tasks/build/task.sh 9 | 10 | -------------------------------------------------------------------------------- /pipeline/tasks/linter/task.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | yarn 6 | ./go "linter-${TARGET}" 7 | -------------------------------------------------------------------------------- /pipeline/tasks/linter/task.yml: -------------------------------------------------------------------------------- 1 | caches: 2 | - path: git/node_modules 3 | inputs: 4 | - name: git 5 | platform: linux 6 | run: 7 | dir: git 8 | path: pipeline/tasks/linter/task.sh 9 | 10 | -------------------------------------------------------------------------------- /pipeline/tasks/serverspec/task.yml: -------------------------------------------------------------------------------- 1 | platform: linux 2 | inputs: 3 | - name: git 4 | run: 5 | path: bash 6 | dir: git/serverspec 7 | args: 8 | - -c 9 | - ./run 10 | -------------------------------------------------------------------------------- /pipeline/tasks/tests/task.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | yarn 6 | ./go "test-${TARGET}" 7 | -------------------------------------------------------------------------------- /pipeline/tasks/tests/task.yml: -------------------------------------------------------------------------------- 1 | caches: 2 | - path: git/node_modules 3 | inputs: 4 | - name: git 5 | platform: linux 6 | run: 7 | dir: git 8 | path: pipeline/tasks/tests/task.sh 9 | 10 | -------------------------------------------------------------------------------- /pipeline/tasks/update-pipeline/task.yml: -------------------------------------------------------------------------------- 1 | platform: linux 2 | inputs: 3 | - name: git 4 | run: 5 | path: bash 6 | dir: git 7 | args: 8 | - -c 9 | - ./go update-pipeline 10 | -------------------------------------------------------------------------------- /scripts/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sirech/example-concourse-pipeline/666c46fbd776ab71ad87ecf0621e8cbdb90ccbcd/scripts/.gitkeep -------------------------------------------------------------------------------- /serverspec/.ruby-version: -------------------------------------------------------------------------------- 1 | 2.6.3 2 | -------------------------------------------------------------------------------- /serverspec/Dockerfile.serverspec: -------------------------------------------------------------------------------- 1 | FROM docker:edge-dind 2 | 3 | RUN addgroup -g 2999 docker 4 | 5 | # From https://hub.docker.com/r/frolvlad/alpine-glibc/ 6 | RUN ALPINE_GLIBC_BASE_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download" && \ 7 | ALPINE_GLIBC_PACKAGE_VERSION="2.25-r0" && \ 8 | ALPINE_GLIBC_BASE_PACKAGE_FILENAME="glibc-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \ 9 | ALPINE_GLIBC_BIN_PACKAGE_FILENAME="glibc-bin-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \ 10 | ALPINE_GLIBC_I18N_PACKAGE_FILENAME="glibc-i18n-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \ 11 | apk add --no-cache --virtual=.build-dependencies wget ca-certificates && \ 12 | apk add --no-cache python py-pip && \ 13 | pip install --upgrade --no-cache-dir pip && \ 14 | pip install --no-cache-dir awscli && \ 15 | apk --purge del py-pip && \ 16 | wget -q -O "/etc/apk/keys/sgerrand.rsa.pub" "https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub" && \ 17 | wget -q \ 18 | "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \ 19 | "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \ 20 | "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \ 21 | apk add --no-cache \ 22 | "$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \ 23 | "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \ 24 | "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \ 25 | \ 26 | rm "/etc/apk/keys/sgerrand.rsa.pub" && \ 27 | /usr/glibc-compat/bin/localedef --force --inputfile POSIX --charmap UTF-8 C.UTF-8 || true && \ 28 | echo "export LANG=C.UTF-8" > /etc/profile.d/locale.sh && \ 29 | \ 30 | apk del glibc-i18n && \ 31 | \ 32 | rm "/root/.wget-hsts" && \ 33 | apk del .build-dependencies && \ 34 | rm \ 35 | "$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \ 36 | "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \ 37 | "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" 38 | 39 | # The following is from docker/ruby, https://github.com/docker-library/ruby 40 | RUN mkdir -p /usr/local/etc \ 41 | && { \ 42 | echo 'install: --no-document'; \ 43 | echo 'update: --no-document'; \ 44 | } >> /usr/local/etc/gemrc 45 | 46 | ENV RUBY_MAJOR 2.5 47 | ENV RUBY_VERSION 2.5.1 48 | ENV RUBY_DOWNLOAD_SHA256 886ac5eed41e3b5fc699be837b0087a6a5a3d10f464087560d2d21b3e71b754d 49 | ENV RUBYGEMS_VERSION 2.7.6 50 | ENV BUNDLER_VERSION 1.16.1 51 | 52 | # some of ruby's build scripts are written in ruby 53 | # we purge system ruby later to make sure our final image uses what we just built 54 | # readline-dev vs libedit-dev: https://bugs.ruby-lang.org/issues/11869 and https://github.com/docker-library/ruby/issues/75 55 | # hadolint ignore=DL3003,DL3019,DL4006,SC2086 56 | RUN set -ex \ 57 | \ 58 | && apk add --no-cache --virtual .ruby-builddeps \ 59 | bash \ 60 | autoconf \ 61 | bison \ 62 | bzip2 \ 63 | bzip2-dev \ 64 | ca-certificates \ 65 | coreutils \ 66 | dpkg-dev dpkg \ 67 | gcc \ 68 | gdbm-dev \ 69 | glib-dev \ 70 | libc-dev \ 71 | libffi-dev \ 72 | openssl \ 73 | openssl-dev \ 74 | libxml2-dev \ 75 | libxslt-dev \ 76 | linux-headers \ 77 | make \ 78 | ncurses-dev \ 79 | procps \ 80 | readline-dev \ 81 | ruby \ 82 | tar \ 83 | xz \ 84 | yaml-dev \ 85 | zlib-dev \ 86 | \ 87 | && wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz" \ 88 | && echo "$RUBY_DOWNLOAD_SHA256 *ruby.tar.xz" | sha256sum -c - \ 89 | \ 90 | && mkdir -p /usr/src/ruby \ 91 | && tar -xJf ruby.tar.xz -C /usr/src/ruby --strip-components=1 \ 92 | && rm ruby.tar.xz \ 93 | \ 94 | && cd /usr/src/ruby \ 95 | \ 96 | # hack in "ENABLE_PATH_CHECK" disabling to suppress: 97 | # warning: Insecure world writable dir 98 | && { \ 99 | echo '#define ENABLE_PATH_CHECK 0'; \ 100 | echo; \ 101 | cat file.c; \ 102 | } > file.c.new \ 103 | && mv file.c.new file.c \ 104 | \ 105 | && autoconf \ 106 | && gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \ 107 | # the configure script does not detect isnan/isinf as macros 108 | && export ac_cv_func_isnan=yes ac_cv_func_isinf=yes \ 109 | && ./configure \ 110 | --build="$gnuArch" \ 111 | --disable-install-doc \ 112 | --enable-shared \ 113 | && make -j "$(nproc)" \ 114 | && make install \ 115 | \ 116 | && runDeps="$( \ 117 | scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \ 118 | | tr ',' '\n' \ 119 | | sort -u \ 120 | | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ 121 | )" \ 122 | && apk add --virtual .ruby-rundeps $runDeps \ 123 | bzip2 \ 124 | ca-certificates \ 125 | libffi-dev \ 126 | openssl-dev \ 127 | procps \ 128 | yaml-dev \ 129 | zlib-dev \ 130 | # WARNING: This is commented so that we can build gems ourselves later 131 | # && apk del .ruby-builddeps \ 132 | && cd / \ 133 | && rm -r /usr/src/ruby \ 134 | \ 135 | && gem update --system "$RUBYGEMS_VERSION" \ 136 | && gem install bundler --version "$BUNDLER_VERSION" --force \ 137 | && rm -r /root/.gem/ 138 | 139 | # install things globally, for great justice 140 | # and don't create ".bundle" in all our apps 141 | ENV GEM_HOME /usr/local/bundle 142 | ENV BUNDLE_PATH="$GEM_HOME" \ 143 | BUNDLE_BIN="$GEM_HOME/bin" \ 144 | BUNDLE_SILENCE_ROOT_WARNING=1 \ 145 | BUNDLE_APP_CONFIG="$GEM_HOME" 146 | ENV PATH $BUNDLE_BIN:$PATH 147 | RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" \ 148 | && chmod 777 "$GEM_HOME" "$BUNDLE_BIN" 149 | 150 | # End docker/ruby contents 151 | -------------------------------------------------------------------------------- /serverspec/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'docker-api' 4 | gem 'rspec-wait' 5 | gem 'rubocop' 6 | gem 'rubocop-rspec' 7 | gem 'serverspec' 8 | -------------------------------------------------------------------------------- /serverspec/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | ast (2.4.0) 5 | diff-lcs (1.3) 6 | docker-api (1.34.2) 7 | excon (>= 0.47.0) 8 | multi_json 9 | excon (0.73.0) 10 | multi_json (1.14.1) 11 | net-scp (3.0.0) 12 | net-ssh (>= 2.6.5, < 7.0.0) 13 | net-ssh (6.0.2) 14 | net-telnet (0.1.1) 15 | parallel (1.19.1) 16 | parser (2.7.1.2) 17 | ast (~> 2.4.0) 18 | rainbow (3.0.0) 19 | rexml (3.2.4) 20 | rspec (3.9.0) 21 | rspec-core (~> 3.9.0) 22 | rspec-expectations (~> 3.9.0) 23 | rspec-mocks (~> 3.9.0) 24 | rspec-core (3.9.2) 25 | rspec-support (~> 3.9.3) 26 | rspec-expectations (3.9.2) 27 | diff-lcs (>= 1.2.0, < 2.0) 28 | rspec-support (~> 3.9.0) 29 | rspec-its (1.3.0) 30 | rspec-core (>= 3.0.0) 31 | rspec-expectations (>= 3.0.0) 32 | rspec-mocks (3.9.1) 33 | diff-lcs (>= 1.2.0, < 2.0) 34 | rspec-support (~> 3.9.0) 35 | rspec-support (3.9.3) 36 | rspec-wait (0.0.9) 37 | rspec (>= 3, < 4) 38 | rubocop (0.83.0) 39 | parallel (~> 1.10) 40 | parser (>= 2.7.0.1) 41 | rainbow (>= 2.2.2, < 4.0) 42 | rexml 43 | ruby-progressbar (~> 1.7) 44 | unicode-display_width (>= 1.4.0, < 2.0) 45 | rubocop-rspec (1.39.0) 46 | rubocop (>= 0.68.1) 47 | ruby-progressbar (1.10.1) 48 | serverspec (2.41.5) 49 | multi_json 50 | rspec (~> 3.0) 51 | rspec-its 52 | specinfra (~> 2.72) 53 | sfl (2.3) 54 | specinfra (2.82.16) 55 | net-scp 56 | net-ssh (>= 2.7) 57 | net-telnet (= 0.1.1) 58 | sfl 59 | unicode-display_width (1.7.0) 60 | 61 | PLATFORMS 62 | ruby 63 | 64 | DEPENDENCIES 65 | docker-api 66 | rspec-wait 67 | rubocop 68 | rubocop-rspec 69 | serverspec 70 | 71 | BUNDLED WITH 72 | 1.17.2 73 | -------------------------------------------------------------------------------- /serverspec/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | 5 | # sanitize_cgroups, start_docker and stop_docker are taken from https://github.com/concourse/docker-image-resource 6 | # https://raw.githubusercontent.com/concourse/docker-image-resource/master/LICENSE 7 | 8 | sanitize_cgroups() { 9 | mkdir -p /sys/fs/cgroup 10 | mountpoint -q /sys/fs/cgroup || \ 11 | mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup 12 | 13 | mount -o remount,rw /sys/fs/cgroup 14 | 15 | sed -e 1d /proc/cgroups | while read sys hierarchy num enabled; do 16 | if [ "$enabled" != "1" ]; then 17 | # subsystem disabled; skip 18 | continue 19 | fi 20 | 21 | grouping="$(cat /proc/self/cgroup | cut -d: -f2 | grep "\\<$sys\\>")" || true 22 | if [ -z "$grouping" ]; then 23 | # subsystem not mounted anywhere; mount it on its own 24 | grouping="$sys" 25 | fi 26 | 27 | mountpoint="/sys/fs/cgroup/$grouping" 28 | 29 | mkdir -p "$mountpoint" 30 | 31 | # clear out existing mount to make sure new one is read-write 32 | if mountpoint -q "$mountpoint"; then 33 | umount "$mountpoint" 34 | fi 35 | 36 | mount -n -t cgroup -o "$grouping" cgroup "$mountpoint" 37 | 38 | if [ "$grouping" != "$sys" ]; then 39 | if [ -L "/sys/fs/cgroup/$sys" ]; then 40 | rm "/sys/fs/cgroup/$sys" 41 | fi 42 | 43 | ln -s "$mountpoint" "/sys/fs/cgroup/$sys" 44 | fi 45 | done 46 | 47 | if ! test -e /sys/fs/cgroup/systemd ; then 48 | mkdir /sys/fs/cgroup/systemd 49 | mount -t cgroup -o none,name=systemd none /sys/fs/cgroup/systemd 50 | fi 51 | } 52 | 53 | start_docker() { 54 | mkdir -p /var/log 55 | mkdir -p /var/run 56 | 57 | sanitize_cgroups 58 | 59 | # check for /proc/sys being mounted readonly, as systemd does 60 | if grep '/proc/sys\s\+\w\+\s\+ro,' /proc/mounts >/dev/null; then 61 | mount -o remount,rw /proc/sys 62 | fi 63 | 64 | local mtu=$(cat /sys/class/net/$(ip route get 8.8.8.8|awk '{ print $5 }')/mtu) 65 | 66 | dockerd \ 67 | --mtu ${mtu} \ 68 | --data-root /scratch/docker \ 69 | --storage-driver=overlay2 \ 70 | --host=unix:///var/run/docker.sock \ 71 | --host=tcp://0.0.0.0:2375 >/tmp/docker.log 2>&1 & 72 | 73 | echo $! > /tmp/docker.pid 74 | 75 | trap stop_docker EXIT 76 | 77 | sleep 1 78 | 79 | if ! docker info >/dev/null 2>&1; then 80 | echo waiting for docker to come up... 81 | until docker info >/dev/null 2>&1; do 82 | sleep 1 83 | done 84 | fi 85 | } 86 | 87 | stop_docker() { 88 | local pid=$(cat /tmp/docker.pid) 89 | if [ -z "$pid" ]; then 90 | return 0 91 | fi 92 | 93 | kill -TERM $pid 94 | } 95 | 96 | 97 | start_docker 98 | 99 | # docker load -i ./images/* > /dev/null 100 | 101 | exec $@ 102 | -------------------------------------------------------------------------------- /serverspec/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -o nounset 5 | set -o pipefail 6 | 7 | SCRIPT_DIR=$(cd "$(dirname "$0")" ; pwd -P) 8 | AWS_ACCOUNT="your_account" 9 | FULL_IMAGE_NAME="${AWS_ACCOUNT}.dkr.ecr.eu-central-1.amazonaws.com/${IMAGE_NAME}" 10 | REGION="us-east-1" 11 | 12 | function dockerWorking() { 13 | docker pull ${FULL_IMAGE_NAME} &> /dev/null 14 | } 15 | 16 | function ecrLogin() { 17 | echo "Not logged into ECR yet, logging in" 18 | login_command=$(aws ecr get-login --registry-ids ${AWS_ACCOUNT} --region eu-central-1 --no-include-email) 19 | eval "${login_command}" &> /dev/null 20 | if dockerWorking ; then 21 | echo "Login succeeded" 22 | else 23 | echo "Login failed" 24 | exit 1 25 | fi 26 | } 27 | 28 | function main() { 29 | dockerWorking || ecrLogin 30 | 31 | pushd "${SCRIPT_DIR}/spec" > /dev/null 32 | bundle install --path vendor/bundle 33 | bundle exec rubocop 34 | for spec in *_spec.rb; do 35 | IMAGE=${FULL_IMAGE_NAME} bundle exec rspec "${spec}" 36 | done 37 | popd > /dev/null 38 | } 39 | 40 | main -------------------------------------------------------------------------------- /serverspec/spec/dev_container_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative 'spec_helper' 2 | 3 | describe 'dev-container' do 4 | describe 'node' do 5 | describe file('/usr/local/bin/node') do 6 | it { is_expected.to be_executable } 7 | end 8 | 9 | [ 10 | [:node, /14.2/], 11 | [:npm, /6.14/] 12 | ].each do |executable, version| 13 | describe command("#{executable} -v") do 14 | its(:stdout) { is_expected.to match(version) } 15 | end 16 | end 17 | 18 | describe command('npm doctor') do 19 | its(:exit_status) { is_expected.to eq 0 } 20 | end 21 | end 22 | 23 | describe 'shell' do 24 | %i[shellcheck].each do |executable| 25 | describe file("/usr/bin/#{executable}") do 26 | it { is_expected.to be_executable } 27 | end 28 | end 29 | end 30 | 31 | describe 'fly' do 32 | describe command('fly -v') do 33 | its(:stdout) { is_expected.to match(/6.0.0/) } 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /serverspec/spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'serverspec' 2 | require 'docker-api' 3 | require 'docker' 4 | require 'rspec/wait' 5 | 6 | set :backend, :docker 7 | set :docker_image, ENV['IMAGE'] 8 | 9 | RSpec.configure do |config| 10 | config.wait_timeout = 60 # seconds 11 | end 12 | -------------------------------------------------------------------------------- /src/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sirech/example-concourse-pipeline/666c46fbd776ab71ad87ecf0621e8cbdb90ccbcd/src/.gitkeep -------------------------------------------------------------------------------- /src/main.css: -------------------------------------------------------------------------------- 1 | .main { 2 | padding: 0; 3 | margin: 0; 4 | } 5 | -------------------------------------------------------------------------------- /src/main.js: -------------------------------------------------------------------------------- 1 | const express = require('express') 2 | const app = express() 3 | const port = 3000 4 | 5 | app.get('/', (req, res) => res.send('Hello World!')) 6 | app.get('/secret', (req, res) => res.send(`The super secret value is ${process.env.SECRET}`)) 7 | 8 | app.listen(port, () => console.log(`Example app listening on port ${port}!`)) 9 | -------------------------------------------------------------------------------- /src/main.spec.js: -------------------------------------------------------------------------------- 1 | describe('sample', () => { 2 | it('adds 1 + 2 to equal 3', () => { 3 | expect(1 + 2).toBe(3) 4 | }) 5 | }) 6 | --------------------------------------------------------------------------------