├── .github └── workflows │ └── main.yml ├── .gitignore ├── .pre-commit-config.yaml ├── README.md ├── project ├── Dockerfile ├── Dockerfile.prod ├── app │ ├── __init__.py │ ├── app.py │ ├── assets │ │ ├── custom.css │ │ └── fig_layout.json │ └── functions.py ├── requirements.txt └── tests │ └── test_functions.py ├── release.sh └── setup.cfg /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration and Delivery 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | env: 9 | IMAGE: docker.pkg.github.com/$(echo $GITHUB_REPOSITORY | tr '[A-Z]' '[a-z]')/app 10 | 11 | jobs: 12 | 13 | build: 14 | name: Build Docker Image 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout master 18 | uses: actions/checkout@v3 19 | - name: Log in to Github Packages 20 | run: echo ${GITHUB_TOKEN} | docker login -u ${GITHUB_ACTOR} --password-stdin docker.pkg.github.com 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | - name: Pull image 24 | run: | 25 | docker pull ${{ env.IMAGE }}:latest || true 26 | - name: Build Image 27 | run: | 28 | docker build \ 29 | --cache-from ${{ env.IMAGE }}:latest \ 30 | --tag ${{ env.IMAGE }}:latest \ 31 | --file ./project/Dockerfile.prod \ 32 | "./project" 33 | - name: Push image 34 | run: | 35 | docker push ${{ env.IMAGE }}:latest 36 | 37 | test: 38 | name: Test Docker Image 39 | runs-on: ubuntu-latest 40 | needs: build 41 | steps: 42 | - name: Checkout master 43 | uses: actions/checkout@v3 44 | - name: Log in to GitHub Packages 45 | run: echo ${GITHUB_TOKEN} | docker login -u ${GITHUB_ACTOR} --password-stdin docker.pkg.github.com 46 | env: 47 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 48 | - name: Pull images 49 | run: | 50 | docker pull ${{ env.IMAGE }}:latest || true 51 | - name: Build images 52 | run: | 53 | docker build \ 54 | --cache-from ${{ env.IMAGE }}:latest \ 55 | --tag ${{ env.IMAGE }}:latest \ 56 | --file ./project/Dockerfile.prod \ 57 | "./project" 58 | - name: Run container 59 | run: | 60 | docker run \ 61 | -d \ 62 | --name docker-dash \ 63 | -e PORT=8050 \ 64 | -p 8050:8050 \ 65 | ${{ env.IMAGE }}:latest 66 | - name: Install requirements 67 | run: docker exec docker-dash pip install black==22.12.0 flake8===6.0.0 pytest==7.2.1 bandit==1.7.4 68 | - name: Pytest 69 | run: docker exec docker-dash python -m pytest . 70 | - name: Flake8 71 | run: docker exec docker-dash python -m flake8 --max-line-length=119 . 72 | - name: Black 73 | run: docker exec docker-dash python -m black . --check 74 | 75 | deploy: 76 | name: Deploy to Google Cloud Run 77 | runs-on: ubuntu-latest 78 | needs: [build, test] 79 | env: 80 | IMAGE_NAME: gcr.io/${{ secrets.GCP_PROJECT_ID }}/${{ secrets.GCP_APP_NAME }} 81 | steps: 82 | - name: Checkout master 83 | uses: actions/checkout@v3 84 | - name: Log in to Github Packages 85 | run: echo ${GITHUB_TOKEN} | docker login -u ${GITHUB_ACTOR} --password-stdin docker.pkg.github.com 86 | env: 87 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 88 | - name: Pull image 89 | run: | 90 | docker pull ${{ env.IMAGE }}:latest || true 91 | - name: Build image 92 | run: | 93 | docker build \ 94 | --cache-from ${{ env.IMAGE }}:latest \ 95 | --tag ${{ env.IMAGE_NAME }}:latest \ 96 | --file ./project/Dockerfile.prod \ 97 | "./project" 98 | - name: Login to Google Cloud 99 | uses: google-github-actions/auth@v1 100 | with: 101 | credentials_json: ${{ secrets.GCP_CREDENTIALS }} 102 | - name: Configure Docker 103 | run: gcloud auth configure-docker --quiet 104 | - name: Push to registry 105 | run: docker push ${{ env.IMAGE_NAME }} 106 | - id: deploy 107 | name: Deploy Docker image 108 | uses: "google-github-actions/deploy-cloudrun@v1" 109 | with: 110 | image: ${{ env.IMAGE_NAME }} 111 | region: europe-north1 112 | service: app 113 | flags: --port=8080 --allow-unauthenticated 114 | - name: Clean up old images 115 | run: gcloud container images list-tags ${{ env.IMAGE_NAME }} --filter='-tags:*' --format="get(digest)" --limit=10 > tags && while read p; do gcloud container images delete "${{ env.IMAGE_NAME }}@$p" --quiet; done < tags 116 | 117 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.vscode 2 | __pycache__/ 3 | *.ipynb -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pycqa/flake8 3 | rev: 6.0.0 4 | hooks: 5 | - id: flake8 6 | exclude: project/app/app.py 7 | args: ['--config=setup.cfg'] 8 | - repo: https://github.com/ambv/black 9 | rev: 22.12.0 10 | hooks: 11 | - id: black 12 | - repo: https://github.com/pycqa/isort 13 | rev: 5.11.4 14 | hooks: 15 | - id: isort 16 | name: isort (python) 17 | - repo: https://github.com/pre-commit/pre-commit-hooks 18 | rev: v4.4.0 19 | hooks: 20 | - id: detect-private-key 21 | - repo: https://github.com/PyCQA/bandit 22 | rev: 1.7.4 23 | hooks: 24 | - id: bandit 25 | args: [--skip, "B101"] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | - **Note:** hotfix for werkzeug issue [#1992](https://github.com/plotly/dash/issues/1992) in place 2 | - **Note:** changed deployment from Heroku to Google Cloud Run 3 | 4 | # Docker Dash Example 5 | A simple design for a plotly-dash app with sklearn running within a docker container deployed to [Google Cloud Run](https://docker-dash-example.com/) using CI/CD. [![Continuous Integration and Delivery](https://github.com/ROpdam/docker-dash-example/actions/workflows/main.yml/badge.svg?branch=master)](https://github.com/ROpdam/docker-dash-example/actions/workflows/main.yml) 6 | 7 | For a deep dive on the implementation please see: 8 | 1. [Initial deployment through Heroku](https://towardsdatascience.com/deploy-containeriazed-plotly-dash-app-to-heroku-with-ci-cd-f82ca833375c). 9 | 2. [Current deployment to GCP through Google Cloud Run](https://medium.com/towards-data-science/deploy-containerised-plotly-dash-app-with-ci-cd-p2-gcp-dfa33edc5f2f) 10 | 11 | Inspired by [This TDD Course](https://testdriven.io/courses/tdd-fastapi/) 12 | 13 | ## Using [pre-commit-hooks](https://pre-commit.com/) 14 | - [flake8](https://github.com/pycqa/flake8) 15 | - [black](https://github.com/ambv/black) 16 | - [isort](https://github.com/pycqa/isort) 17 | - [detect-private-key](https://github.com/pre-commit/pre-commit-hooks#detect-private-key) 18 | - [bandit](https://github.com/PyCQA/bandit) 19 | 20 | ## Repo structure 21 | ``` 22 | ├── .github 23 | │ └── workflows 24 | │ └── main.yml 25 | │ 26 | ├── project 27 | │ ├── app 28 | │ │ ├── __init__.py 29 | │ │ ├── app.py 30 | │ │ ├── functions.py 31 | │ │ └── assets 32 | │ ├── tests 33 | │ │ └── test_functions.py 34 | │ ├── Dockerfile 35 | │ ├── Dockerfile.prod 36 | │ └── requirements.txt 37 | │ 38 | ├── release.sh 39 | ├── setup.cfg 40 | ├── .pre-commit-config.yaml 41 | ├── .gitignore 42 | │ 43 | └── README.md 44 | ``` 45 | 46 | ## Run locally 47 | To run the image locally, cd into the docker-dash-example folder and: 48 | ``` 49 | docker build -t docker-dash project/. 50 | ``` 51 | And run the container 52 | ``` 53 | docker run -p 8050:8050 docker-dash 54 | ``` 55 | You can find to the app on your local machine http://localhost:8050/ (or localhost:8050). This way the image is created using the Dockerfile, instead of the Dockerfile.prod. 56 | 57 | ## Project context 58 | This was a fun project to apply different learnings in practice, very happy to hear **your** thoughts in the repo, including feedback of course! 59 | -------------------------------------------------------------------------------- /project/Dockerfile: -------------------------------------------------------------------------------- 1 | #From https://towardsdatascience.com/docker-for-python-dash-r-shiny-6097c8998506 2 | 3 | FROM python:3.9-slim-buster 4 | 5 | LABEL maintainer "Robin Opdam, robinopdam@hotmail.com" 6 | 7 | # set working directory in container 8 | WORKDIR /usr/src/app 9 | 10 | # Copy and install packages 11 | COPY requirements.txt / 12 | RUN pip install --upgrade pip 13 | RUN pip install -r /requirements.txt 14 | 15 | # Copy app folder to app folder in container 16 | COPY /app /usr/src/app/ 17 | 18 | # Changing to non-root user 19 | RUN useradd -m appUser 20 | USER appUser 21 | 22 | # Run locally 23 | CMD gunicorn --bind 0.0.0.0:8050 app:server -------------------------------------------------------------------------------- /project/Dockerfile.prod: -------------------------------------------------------------------------------- 1 | #From https://towardsdatascience.com/docker-for-python-dash-r-shiny-6097c8998506 2 | 3 | FROM python:3.9-slim-buster 4 | 5 | LABEL maintainer "Robin Opdam, robinopdam@hotmail.com" 6 | 7 | # set working directory in container 8 | WORKDIR /usr/src/app 9 | 10 | # Copy and install packages 11 | COPY requirements.txt / 12 | RUN pip install --upgrade pip 13 | RUN pip install -r /requirements.txt 14 | 15 | # Copy app folder to app folder in container 16 | COPY /app /usr/src/app/ 17 | # Copying tests to app folder for running in workflow 18 | COPY /tests /usr/src/app/ 19 | 20 | # Changing to non-root user 21 | RUN useradd -m appUser 22 | USER appUser 23 | 24 | # For running on Heroku 25 | CMD gunicorn --bind 0.0.0.0:$PORT app:server 26 | -------------------------------------------------------------------------------- /project/app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ROpdam/docker-dash-example/78b2f5860cb649d788201bc920a7fc3324f6084e/project/app/__init__.py -------------------------------------------------------------------------------- /project/app/app.py: -------------------------------------------------------------------------------- 1 | import dash_bootstrap_components as dbc 2 | from dash import Dash, dcc, html 3 | from dash.dependencies import Input, Output 4 | from functions import plot_regression 5 | 6 | app = Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) 7 | server = app.server 8 | 9 | app.layout = html.Div( 10 | [ 11 | html.Div( 12 | [ 13 | dcc.Graph(id="regression_plot"), 14 | html.P( 15 | "Standard deviation, try changing it!", 16 | style={"color": "white", "marginLeft": "20px"}, 17 | ), 18 | dcc.Slider( 19 | id="std_slider", 20 | min=0, 21 | max=40, 22 | step=0.5, 23 | value=10, 24 | marks={i: str(i) for i in range(0, 40, 5)}, 25 | ), 26 | ] 27 | ), 28 | ] 29 | ) 30 | 31 | 32 | @app.callback( 33 | Output(component_id="regression_plot", component_property="figure"), 34 | [Input(component_id="std_slider", component_property="value")], 35 | ) 36 | def update_regression_plot(std: int) -> None: 37 | return plot_regression(std) 38 | 39 | 40 | # Developing the app locally with debug=True enables 41 | # auto-reloading when making changes 42 | # if __name__ == "__main__": 43 | # app.run_server(host="0.0.0.0", port=8050, debug=True) 44 | -------------------------------------------------------------------------------- /project/app/assets/custom.css: -------------------------------------------------------------------------------- 1 | body { 2 | background-color: black; 3 | } -------------------------------------------------------------------------------- /project/app/assets/fig_layout.json: -------------------------------------------------------------------------------- 1 | { 2 | "plot_bgcolor": "black", 3 | "paper_bgcolor": "black", 4 | "title": { 5 | "font": { 6 | "size": 20, 7 | "color": "white" 8 | } 9 | }, 10 | "legend": { 11 | "font": { 12 | "size": 14, 13 | "color": "white" 14 | }, 15 | "orientation": "h", 16 | "yanchor": "bottom", 17 | "y": 1.02, 18 | "xanchor": "right", 19 | "x": 1 20 | }, 21 | "xaxis": { 22 | "color": "lightgray", 23 | "showgrid": false 24 | }, 25 | "yaxis": { 26 | "color": "lightgray" 27 | } 28 | } -------------------------------------------------------------------------------- /project/app/functions.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Union 3 | 4 | import numpy as np 5 | import plotly.graph_objects as go 6 | from sklearn import linear_model 7 | 8 | _data_size = 1000 9 | 10 | 11 | def create_data(std: float) -> np.array: 12 | """Create x based on std and ceate a related y 13 | 14 | Args: 15 | std (float): standard deviation 16 | 17 | Returns: 18 | np.array: x and y stacked 19 | """ 20 | x = np.arange(0, _data_size) 21 | noise = np.random.normal(size=_data_size, loc=1, scale=std / 10) 22 | y = x * (1 + noise) 23 | data = np.stack([x, y]) 24 | 25 | return data 26 | 27 | 28 | def create_lr_preds(data: np.array, std: float) -> Union[float, np.array]: 29 | """ 30 | Creates random data based on input std and fits a linear 31 | regression model through this data 32 | 33 | Args: 34 | std (float): standard deviation for random data 35 | x (np.array): randomly generated std 36 | 37 | Returns: 38 | r_sq: R squared of the linear regression model 39 | preds: np.array with the predicitons 40 | """ 41 | x = data[0, :].reshape(-1, 1) 42 | y = data[1, :] 43 | 44 | model = linear_model.LinearRegression().fit(x, y) 45 | r_sq = model.score(x, y) 46 | preds = model.predict(x) 47 | 48 | return r_sq, preds 49 | 50 | 51 | def create_figure(**kwds) -> go.Figure: 52 | """Create a go.Figure plot using input data x and 53 | predicitons preds 54 | 55 | Args: 56 | x (np.array): input data 57 | preds (np.array): predictions 58 | 59 | Returns: 60 | go.Figure: plot showing input 61 | data (dots) and predictions (line) 62 | """ 63 | layout = go.Layout( 64 | title=f"Regression fit example with R squared: {round(kwds['r_sq'], 3)}", 65 | height=700, 66 | ) 67 | fig = go.Figure(layout=layout) 68 | 69 | fig.add_trace( 70 | go.Scatter( 71 | x=kwds["x"], 72 | y=kwds["y"], 73 | mode="markers", 74 | name=f"x * (1 + rand_norm(mean=1, std={kwds['std']}/10))", 75 | ) 76 | ) 77 | fig.add_trace( 78 | go.Scatter(x=kwds["x"], y=kwds["preds"], mode="lines", name="linear regression") 79 | ) 80 | 81 | return fig 82 | 83 | 84 | def style_figure(fig: go.Figure) -> go.Figure: 85 | """Style the figure according to the fig_layout.json 86 | 87 | Args: 88 | fig (go.Figure): Figure 89 | 90 | Returns: 91 | go.Figure: styled figure 92 | """ 93 | f = open("assets/fig_layout.json") 94 | fig_layout = json.load(f) 95 | 96 | fig.update_layout(fig_layout) 97 | 98 | return fig 99 | 100 | 101 | def plot_regression(std: float = 10) -> go.Figure: 102 | """Create a regression plot from random input data 103 | that varies with the standard deviation input 104 | 105 | Args: 106 | std (float, optional): standard deviation. Defaults to 10. 107 | 108 | Returns: 109 | go.Figure: go.Figure: plot showing input 110 | data (dots) and predictions (line) 111 | """ 112 | data = create_data(std) 113 | r_sq, preds = create_lr_preds(data, std) 114 | 115 | fig = create_figure(std=std, r_sq=r_sq, x=data[0, :], y=data[1, :], preds=preds) 116 | fig = style_figure(fig) 117 | 118 | return fig 119 | -------------------------------------------------------------------------------- /project/requirements.txt: -------------------------------------------------------------------------------- 1 | dash==2.7 2 | numpy==1.20.1 3 | dash-bootstrap-components==1.2.1 4 | scikit-learn==0.24.1 5 | plotly==5.11.0 6 | gunicorn==20.1.0 7 | werkzeug==2.0.3 -------------------------------------------------------------------------------- /project/tests/test_functions.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from app.functions import _data_size, create_data, create_lr_preds 3 | 4 | 5 | def test_create_data(): 6 | # Test with std = 0.5 7 | data = create_data(0.5) 8 | x, y = data[0], data[1] 9 | assert x.shape == (_data_size,) 10 | assert y.shape == (_data_size,) 11 | assert x[0] == 0 12 | assert x[-1] == _data_size - 1 13 | # assert np.std(y[1:] / x[1:]) == pytest.approx(0.5 / 10, rel=0.1) 14 | 15 | 16 | def test_create_lr_preds(): 17 | data = np.stack([np.arange(0, 10), np.arange(0, 10)]) 18 | std = 0.5 19 | 20 | r_sq, preds = create_lr_preds(data, std) 21 | assert isinstance(r_sq, float) 22 | assert isinstance(preds, np.ndarray) 23 | assert preds.shape == (10,) 24 | # assert preds == pytest.approx(data[1], abs=0.001) 25 | # assert r_sq == pytest.approx(1, abs=0.001) 26 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #https://devcenter.heroku.com/articles/container-registry-and-runtime#releasing-an-image 2 | 3 | set -e 4 | 5 | IMAGE_ID=$(docker inspect ${HEROKU_REGISTRY_IMAGE} --format={{.Id}}) 6 | 7 | curl --netrc -X PATCH https://api.heroku.com/apps/$HEROKU_APP_NAME/formation \ 8 | -d '{ 9 | "updates": [ 10 | { 11 | "type": "web", 12 | "docker_image": "'"$IMAGE_ID"'" 13 | } 14 | ] 15 | }' \ 16 | -H "Content-Type: application/json" \ 17 | -H "Accept: application/vnd.heroku+json; version=3.docker-releases" \ 18 | -H "Authorization: Bearer ${HEROKU_AUTH_TOKEN}" 19 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 119 --------------------------------------------------------------------------------