├── README.md └── .github └── workflows └── python-container-ci.yml /README.md: -------------------------------------------------------------------------------- 1 | # GitHub Action Reusable Workflows 2 | 3 | You can find more information about this project/repository and how to use it in following blog post: 4 | 5 | - [Ultimate CI Pipeline for All of Your Python Projects](https://martinheinz.dev/blog/69) 6 | 7 | ## Configure Sonar 8 | 9 | - Go to , Click _Configure_ next to SonarCloud App 10 | - In Repository access select your desired repository, Click save 11 | - At , Choose your repo, Click _Set Up_ 12 | - Navigate to , Enter name, Click _Generate_, Copy the token 13 | - In repository, navigate to Settings -> Secrets -> Actions 14 | - `https://github.com///settings/secrets/actions` 15 | - Click _New Repository Secret_: 16 | - Name: `SONAR_TOKEN` 17 | - Value: _SonarCloud Token_ 18 | 19 | - In `https://sonarcloud.io/project/analysis_method?id=` disable the _SonarCloud Automatic Analysis_ 20 | 21 | ## Configure CodeClimate 22 | 23 | - Navigate to and add repository 24 | - Click _Repo Settings_ (top left) 25 | - Go to _Test Coverage_ tab under _Analysis_ and copy _Test reporter ID_ 26 | 27 | - In repository, navigate to Settings -> Secrets -> Actions 28 | - `https://github.com///settings/secrets/actions` 29 | - Click _New Repository Secret_: 30 | - Name: `CC_TEST_REPORTER_ID` 31 | - Value: _Test reporter ID_ 32 | 33 | ## Configure Slack Notification 34 | 35 | Create Slack App: 36 | - Navigate to , Click _Create an App_, Give it name, choose workspace 37 | - Click _Incoming Webhook_, Switch on with slider 38 | - Click _Add New Webhook to Workspace_ 39 | - Choose Channel, Click _Allow_, Copy _Webhook URL_ 40 | 41 | - In repository, navigate to Settings -> Secrets -> Actions 42 | - `https://github.com///settings/secrets/actions` 43 | - Click _New Repository Secret_: 44 | - Name: `SLACK_WEBHOOK` 45 | - Value: _Webhook URL_ 46 | -------------------------------------------------------------------------------- /.github/workflows/python-container-ci.yml: -------------------------------------------------------------------------------- 1 | name: Python Container CI Pipeline 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | PYTHON_VERSION: 7 | required: false 8 | type: string 9 | default: '3.10' 10 | DEPENDENCY_MANAGER: 11 | required: false 12 | type: string 13 | default: 'pip' # or poetry, pipenv 14 | ENABLE_SONAR: 15 | required: false 16 | type: boolean 17 | default: false 18 | ENABLE_CODE_CLIMATE: 19 | required: false 20 | type: boolean 21 | default: false 22 | ENABLE_SLACK: 23 | required: false 24 | type: boolean 25 | default: false 26 | ENFORCE_PYLINT: 27 | required: false 28 | type: boolean 29 | default: true 30 | ENFORCE_BLACK: 31 | required: false 32 | type: boolean 33 | default: true 34 | ENFORCE_FLAKE8: 35 | required: false 36 | type: boolean 37 | default: true 38 | ENFORCE_DIVE: 39 | required: false 40 | type: boolean 41 | default: true 42 | ENFORCE_BANDIT: 43 | required: false 44 | type: boolean 45 | default: true 46 | PYLINT_CONFIG: 47 | required: false 48 | type: string 49 | default: '' 50 | # Will by default scan `.` if you want to configure it otherwise use `targets` stanza in config file 51 | # Important: Make sure to exclude `./venv` from scan paths! 52 | BANDIT_CONFIG: 53 | required: false 54 | type: string 55 | default: '' 56 | DIVE_CONFIG: 57 | required: false 58 | type: string 59 | default: '' 60 | 61 | # ----------------------- 62 | CONTAINER_REGISTRY: 63 | required: false 64 | type: string 65 | default: 'ghcr.io' # 'docker.io' 66 | CONTAINER_REPOSITORY: 67 | required: false 68 | type: string 69 | default: '' # in case of Docker Hub: username/image-name 70 | secrets: 71 | CONTAINER_REGISTRY_USERNAME: 72 | description: 'Username for container registry' 73 | required: false 74 | CONTAINER_REGISTRY_PASSWORD: 75 | description: 'Password for container registry' 76 | required: false 77 | SONAR_TOKEN: 78 | description: 'SonarCloud project token' 79 | required: false 80 | CC_TEST_REPORTER_ID: 81 | description: 'CodeClimate Test Reported ID' 82 | required: false 83 | SLACK_WEBHOOK: 84 | description: 'Slack webhook URL' 85 | required: false 86 | 87 | jobs: 88 | python-ci: 89 | runs-on: ubuntu-latest 90 | 91 | permissions: 92 | contents: read 93 | packages: write # Push 94 | id-token: write # Cosign - signing the images with GitHub OIDC Token 95 | security-events: write # Trivy - write vulnerability report 96 | 97 | steps: 98 | - uses: actions/checkout@v1 99 | 100 | - uses: actions/setup-python@v1 101 | id: setup-python 102 | with: 103 | python-version: ${{ inputs.PYTHON_VERSION }} 104 | 105 | - name: Get cache metadata 106 | id: cache-meta 107 | run: | 108 | CACHE_KEY="" 109 | CACHE_PATH="" 110 | 111 | if [ ${{ inputs.DEPENDENCY_MANAGER }} = 'pip' ]; then 112 | CACHE_KEY="venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/requirements.txt') }}" 113 | CACHE_PATH=$(pip cache dir) 114 | elif [ ${{ inputs.DEPENDENCY_MANAGER }} = 'poetry' ]; then 115 | CACHE_KEY="venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}" 116 | CACHE_PATH="./venv" 117 | elif [ ${{ inputs.DEPENDENCY_MANAGER }} = 'pipenv' ]; then 118 | CACHE_KEY="venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/Pipfile.lock') }}" 119 | CACHE_PATH="./venv" 120 | fi 121 | echo "::set-output name=cache-key::$CACHE_KEY" 122 | echo "::set-output name=cache-path::$CACHE_PATH" 123 | 124 | - name: Install Poetry 125 | uses: snok/install-poetry@v1 126 | if: ${{ inputs.DEPENDENCY_MANAGER == 'poetry' }} 127 | with: 128 | virtualenvs-create: false 129 | virtualenvs-in-project: true 130 | virtualenvs-path: ${{ steps.cache-meta.outputs.cache-path }} 131 | 132 | - name: Load cached venv 133 | id: cache 134 | uses: actions/cache@v2 135 | with: 136 | path: ${{ steps.cache-meta.outputs.cache-path }} 137 | key: ${{ steps.cache-meta.outputs.cache-key }} 138 | 139 | - name: Install cosign 140 | uses: sigstore/cosign-installer@main 141 | 142 | - name: Install Dependencies 143 | run: | 144 | python -m pip install --upgrade pip 145 | python -m venv venv 146 | source venv/bin/activate 147 | pip install pylint flake8 bandit pytest pytest-cov 148 | if [ ${{ inputs.DEPENDENCY_MANAGER }} = 'pip' ]; then 149 | pip install -r requirements.txt; 150 | elif [ ${{ inputs.DEPENDENCY_MANAGER }} = 'poetry' ]; then 151 | poetry install --no-root 152 | elif [ ${{ inputs.DEPENDENCY_MANAGER }} = 'pipenv' ]; then 153 | pip install pipenv 154 | pipenv install 155 | fi 156 | 157 | - name: Run Tests 158 | # Will find config automatically in `pytest.ini`, `pyproject.toml`, `tox.ini` or `setup.cfg` - https://docs.pytest.org/en/6.2.x/customize.html#configuration-file-formats 159 | run: | 160 | source venv/bin/activate 161 | pytest 162 | 163 | - name: Verify code style (Black) 164 | uses: psf/black@stable 165 | with: 166 | options: "--verbose ${{ inputs.ENFORCE_BLACK && '--check' || '' }}" 167 | 168 | - name: Enforce code style (Flake8) 169 | # Will find config automatically in `setup.cfg`, `tox.ini`, or `.flake8` - https://flake8.pycqa.org/en/latest/user/configuration.html#configuration-locations 170 | run: | 171 | source venv/bin/activate 172 | flake8 ${{ inputs.ENFORCE_FLAKE8 && '' || '--exit-zero' }} 173 | 174 | - name: Lint code 175 | # Will find config automatically in `pylintrc`, `.pylintrc`, `pyproject.toml`, NOT `setup.cfg` - https://pylint.pycqa.org/en/latest/user_guide/run.html#command-line-options 176 | run: | 177 | source venv/bin/activate 178 | PYLINT_CONDITIONAL_ARGS=() 179 | if [ -n "${{ inputs.PYLINT_CONFIG }}" ]; then 180 | PYLINT_CONDITIONAL_ARGS+=( --rcfile=${{ inputs.PYLINT_CONFIG }} ) 181 | fi 182 | 183 | pylint **/*.py ${{ inputs.ENFORCE_PYLINT && '' || '--exit-zero' }} "${PYLINT_CONDITIONAL_ARGS[@]}" 184 | 185 | - name: Code security check 186 | # Will find config automatically in `.bandit`, others have to specified with --ini - see https://github.com/PyCQA/bandit/issues/396#issuecomment-475152672 187 | run: | 188 | source venv/bin/activate 189 | BANDIT_CONDITIONAL_ARGS=() 190 | if [ -n "${{ inputs.BANDIT_CONFIG }}" ]; then 191 | BANDIT_CONDITIONAL_ARGS+=( --exclude ./venv --ini ${{ inputs.BANDIT_CONFIG }} ) 192 | else 193 | BANDIT_CONDITIONAL_ARGS+=( . --exclude ./venv ) 194 | fi 195 | bandit -r ${{ inputs.ENFORCE_BANDIT && '' || '--exit-zero' }} "${BANDIT_CONDITIONAL_ARGS[@]}" 196 | 197 | - name: Send report to CodeClimate 198 | uses: paambaati/codeclimate-action@v3.0.0 199 | if: ${{ inputs.ENABLE_CODE_CLIMATE }} 200 | env: 201 | CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} 202 | with: 203 | coverageLocations: | 204 | ${{github.workspace}}/coverage.xml:coverage.py 205 | 206 | - name: SonarCloud scanner 207 | uses: sonarsource/sonarcloud-github-action@master 208 | if: ${{ inputs.ENABLE_SONAR }} 209 | env: 210 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 211 | SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} 212 | 213 | - name: Get repository accesses 214 | id: get-repo 215 | run: | 216 | REPO="" 217 | USERNAME="" 218 | PASSWORD="" 219 | if [ ${{ inputs.CONTAINER_REGISTRY }} = 'ghcr.io' ]; then 220 | REPO=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') 221 | USERNAME=${{ github.actor }} 222 | PASSWORD=${{ secrets.GITHUB_TOKEN }} 223 | else 224 | REPO=${{ inputs.CONTAINER_REPOSITORY }} 225 | USERNAME=${{ secrets.CONTAINER_REGISTRY_USERNAME }} 226 | PASSWORD=${{ secrets.CONTAINER_REGISTRY_PASSWORD }} 227 | fi 228 | 229 | echo "::set-output name=repo::$REPO" 230 | echo "::set-output name=username::$USERNAME" 231 | echo "::set-output name=password::$PASSWORD" 232 | 233 | - name: Set up Docker Buildx 234 | uses: docker/setup-buildx-action@v1 235 | 236 | - name: Login to GitHub Container Registry 237 | uses: docker/login-action@v1 238 | with: 239 | registry: ${{ inputs.CONTAINER_REGISTRY }} 240 | username: ${{ steps.get-repo.outputs.username }} 241 | password: ${{ steps.get-repo.outputs.password }} 242 | 243 | - name: Generate tags and image meta 244 | id: meta 245 | uses: docker/metadata-action@v3 246 | with: 247 | images: | 248 | ${{ inputs.CONTAINER_REGISTRY }}/${{ steps.get-repo.outputs.repo }} 249 | tags: | 250 | type=ref,event=tag 251 | type=sha 252 | 253 | - name: Build image 254 | uses: docker/build-push-action@v2 255 | with: 256 | context: . 257 | load: true 258 | tags: ${{ steps.meta.outputs.tags }} 259 | labels: ${{ steps.meta.outputs.labels }} 260 | cache-from: type=registry,ref=${{ inputs.CONTAINER_REGISTRY }}/${{ steps.get-repo.outputs.repo }}:latest 261 | cache-to: type=registry,ref=${{ inputs.CONTAINER_REGISTRY }}/${{ steps.get-repo.outputs.repo }}:latest,mode=max 262 | 263 | - name: Analyze image efficiency 264 | uses: MartinHeinz/dive-action@v0.1.3 265 | with: 266 | image: '${{ inputs.CONTAINER_REGISTRY }}/${{ steps.get-repo.outputs.repo }}:${{ steps.meta.outputs.version }}' 267 | config: ${{ inputs.DIVE_CONFIG }} 268 | exit-zero: ${{ !inputs.ENFORCE_DIVE }} 269 | 270 | - name: Trivy vulnerability scan 271 | uses: aquasecurity/trivy-action@master 272 | with: 273 | image-ref: '${{ inputs.CONTAINER_REGISTRY }}/${{ steps.get-repo.outputs.repo }}:${{ steps.meta.outputs.version }}' 274 | format: 'sarif' 275 | output: 'trivy-results.sarif' 276 | 277 | - name: Push container image 278 | uses: docker/build-push-action@v2 279 | with: 280 | context: . 281 | push: true 282 | tags: ${{ steps.meta.outputs.tags }} 283 | cache-from: type=registry,ref=${{ inputs.CONTAINER_REGISTRY }}/${{ steps.get-repo.outputs.repo }}:latest 284 | cache-to: type=registry,ref=${{ inputs.CONTAINER_REGISTRY }}/${{ steps.get-repo.outputs.repo }}:latest,mode=max 285 | 286 | - name: Sign the published Docker image 287 | env: 288 | COSIGN_EXPERIMENTAL: "true" 289 | run: cosign sign ${{ inputs.CONTAINER_REGISTRY }}/${{ steps.get-repo.outputs.repo }}:${{ steps.meta.outputs.version }} 290 | 291 | - name: Upload Trivy scan results to GitHub Security tab 292 | uses: github/codeql-action/upload-sarif@v1 293 | with: 294 | sarif_file: 'trivy-results.sarif' 295 | 296 | - name: Prepare content for Slack notification 297 | if: ${{ always() && inputs.ENABLE_SLACK }} 298 | id: gen-slack-messages 299 | run: | 300 | TITLE="" 301 | if [ "${{ job.status }}" = "success" ]; then 302 | TITLE="Job Success" 303 | elif [ "${{ job.status }}" = "failure" ]; then 304 | TITLE="Job Failed" 305 | else 306 | TITLE="Job Cancelled" 307 | fi 308 | echo "::set-output name=message::$MESSAGE" 309 | echo "::set-output name=title::$TITLE" 310 | 311 | - name: Slack notification 312 | uses: rtCamp/action-slack-notify@v2 313 | if: ${{ always() && inputs.ENABLE_SLACK }} 314 | env: 315 | SLACK_CHANNEL: general 316 | SLACK_COLOR: ${{ job.status }} 317 | SLACK_ICON: https://github.com/${{ github.actor }}.png?size=48 318 | SLACK_TITLE: ${{ steps.gen-slack-messages.outputs.title }} 319 | SLACK_USERNAME: ${{ github.actor }} 320 | SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} 321 | --------------------------------------------------------------------------------