├── .github ├── dependabot.yml └── workflows │ └── ci.yaml ├── Dockerfile ├── Dockerfile.no-wheelhouse ├── LICENSE ├── README.md ├── alpine ├── Dockerfile ├── README.md ├── example │ └── Dockerfile └── onbuild │ └── alpine.dockerfile ├── celery-entrypoint.sh ├── django-entrypoint.sh ├── gunicorn ├── config.py └── requirements.txt ├── images ├── architecture.svg ├── containers.svg └── pod.svg ├── nginx ├── conf.d │ ├── django.conf │ └── django.conf.d │ │ ├── locations │ │ ├── media.conf │ │ ├── root.conf │ │ └── static.conf │ │ ├── maps │ │ └── static_cache_control.conf │ │ └── upstream.conf └── nginx.conf └── tests ├── .dockerignore ├── .flake8 ├── .gitignore ├── Dockerfile ├── README.md ├── conftest.py ├── definitions.py ├── django2 ├── manage.py ├── mysite │ ├── __init__.py │ ├── celery.py │ ├── docker_settings.py │ ├── settings.py │ ├── templates │ │ └── template.html │ ├── urls.py │ └── wsgi.py └── setup.py ├── pytest.ini ├── requirements.txt └── test.py /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "07:00" 8 | open-pull-requests-limit: 10 9 | ignore: 10 | - dependency-name: idna 11 | versions: 12 | - "< 2.9, >= 2.8.a" 13 | - dependency-name: gunicorn 14 | versions: 15 | - 20.0.4 16 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration 2 | on: 3 | push: 4 | branches: 5 | - develop 6 | pull_request: 7 | branches: 8 | - develop 9 | jobs: 10 | build-no-wheelhouse: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | python_version: ["3.10", "3.9", "3.8", "3.7"] 16 | variant: [bullseye] 17 | test_project: [django2] 18 | env: 19 | TAG: py${{ matrix.python_version }}-${{ matrix.variant }} 20 | IMAGE: ghcr.io/praekeltfoundation/docker-django-bootstrap-nw:py${{ matrix.python_version }}-${{ matrix.variant }} 21 | steps: 22 | - uses: actions/checkout@v3 23 | - uses: actions/setup-python@v4 24 | with: 25 | python-version: 3.9 26 | - name: run tests 27 | run: | 28 | docker build -f Dockerfile.no-wheelhouse --pull --cache-from "$IMAGE" --build-arg PYTHON_VERSION="${{matrix.python_version}}-${{matrix.variant}}" --tag "$IMAGE" . 29 | docker build -t "mysite:$TAG" --build-arg BASE_IMAGE="$IMAGE" --build-arg PROJECT="${{matrix.test_project}}" tests 30 | pip install -r tests/requirements.txt 31 | pytest -v tests/test.py --django-bootstrap-image="mysite:$TAG" 32 | flake8 gunicorn/config.py 33 | cd tests; flake8 34 | - uses: docker/setup-buildx-action@v2 35 | - name: construct image metadata 36 | uses: docker/metadata-action@v4 37 | id: meta 38 | with: 39 | images: | 40 | ghcr.io/praekeltfoundation/docker-django-bootstrap-nw 41 | tags: | 42 | type=pep440,pattern=py{{major}},value=${{matrix.python_version}} 43 | type=raw,value=py${{matrix.python_version}} 44 | type=raw,value=${{matrix.variant}} 45 | type=pep440,pattern=py{{major}}-${{matrix.variant}},value=${{matrix.python_version}} 46 | type=raw,value=py${{matrix.python_version}}-${{matrix.variant}} 47 | - name: login to ghcr 48 | uses: docker/login-action@v2 49 | with: 50 | registry: ghcr.io 51 | username: ${{github.actor}} 52 | password: ${{secrets.GITHUB_TOKEN}} 53 | - name: build and push 54 | uses: docker/build-push-action@v4 55 | with: 56 | context: . 57 | file: Dockerfile.no-wheelhouse 58 | push: ${{github.event_name != 'pull_request'}} 59 | tags: ${{steps.meta.outputs.tags}} 60 | build-args: | 61 | PYTHON_VERSION=${{matrix.python_version}}-${{matrix.variant}} 62 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION=3.7-stretch 2 | FROM praekeltfoundation/python-base:${PYTHON_VERSION} 3 | 4 | # Create the user and group first as they shouldn't change often. 5 | # Specify the UID/GIDs so that they do not change somehow and mess with the 6 | # ownership of external volumes. 7 | RUN addgroup --system --gid 107 django \ 8 | && adduser --system --uid 104 --ingroup django django \ 9 | && mkdir /etc/gunicorn 10 | 11 | # Install libpq for psycopg2 for PostgreSQL support 12 | RUN apt-get-install.sh libpq5 13 | 14 | # Install a modern Nginx and configure 15 | ENV NGINX_VERSION=1.18.0 \ 16 | NGINX_GPG_KEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62 17 | RUN set -ex; \ 18 | fetchDeps=" \ 19 | wget \ 20 | $(command -v gpg > /dev/null || echo 'dirmngr gnupg') \ 21 | "; \ 22 | apt-get-install.sh $fetchDeps; \ 23 | wget https://nginx.org/keys/nginx_signing.key; \ 24 | [ "$(gpg --batch -q --with-fingerprint --with-colons nginx_signing.key | awk -F: '/^fpr:/ { print $10 }')" \ 25 | = $NGINX_GPG_KEY ]; \ 26 | apt-key add nginx_signing.key; \ 27 | codename="$(. /etc/os-release; echo $VERSION | grep -oE [a-z]+)"; \ 28 | echo "deb http://nginx.org/packages/debian/ $codename nginx" > /etc/apt/sources.list.d/nginx.list; \ 29 | rm nginx_signing.key; \ 30 | apt-get-purge.sh $fetchDeps; \ 31 | \ 32 | apt-get-install.sh "nginx=$NGINX_VERSION-1\~$codename"; \ 33 | rm /etc/nginx/conf.d/default.conf; \ 34 | # Add nginx user to django group so that Nginx can read/write to gunicorn socket 35 | adduser nginx django 36 | COPY nginx/ /etc/nginx/ 37 | 38 | # Install gunicorn 39 | COPY gunicorn/ /etc/gunicorn/ 40 | RUN pip install -r /etc/gunicorn/requirements.txt 41 | 42 | EXPOSE 8000 43 | WORKDIR /app 44 | 45 | COPY django-entrypoint.sh celery-entrypoint.sh \ 46 | /scripts/ 47 | ENTRYPOINT ["tini", "--", "django-entrypoint.sh"] 48 | CMD [] 49 | -------------------------------------------------------------------------------- /Dockerfile.no-wheelhouse: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION=3.10-bullseye 2 | FROM ghcr.io/praekeltfoundation/python-base-nw:${PYTHON_VERSION} 3 | 4 | # Create the user and group first as they shouldn't change often. 5 | # Specify the UID/GIDs so that they do not change somehow and mess with the 6 | # ownership of external volumes. 7 | RUN addgroup --system --gid 107 django \ 8 | && adduser --system --uid 104 --ingroup django django \ 9 | && mkdir /etc/gunicorn 10 | 11 | # Install libpq for psycopg2 for PostgreSQL support 12 | RUN apt-get-install.sh libpq5 13 | 14 | # Install a modern Nginx and configure 15 | ENV NGINX_VERSION=1.26.1 \ 16 | NGINX_GPG_KEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62 17 | RUN set -ex; \ 18 | fetchDeps=" \ 19 | wget \ 20 | $(command -v gpg > /dev/null || echo 'dirmngr gnupg') \ 21 | "; \ 22 | apt-get-install.sh $fetchDeps; \ 23 | wget https://nginx.org/keys/nginx_signing.key; \ 24 | [ "$(gpg --batch -q --with-fingerprint --with-colons nginx_signing.key | awk -F: '/^fpr:/ { print $10 }'|grep $NGINX_GPG_KEY)" \ 25 | = $NGINX_GPG_KEY ]; \ 26 | apt-key add nginx_signing.key; \ 27 | codename="$(. /etc/os-release; echo $VERSION | grep -oE [a-z]+)"; \ 28 | echo "deb http://nginx.org/packages/debian/ $codename nginx" > /etc/apt/sources.list.d/nginx.list; \ 29 | rm nginx_signing.key; \ 30 | apt-get-purge.sh $fetchDeps; \ 31 | \ 32 | apt-get-install.sh "nginx=$NGINX_VERSION-1\~$codename"; \ 33 | rm /etc/nginx/conf.d/default.conf; \ 34 | # Add nginx user to django group so that Nginx can read/write to gunicorn socket 35 | adduser nginx django 36 | COPY nginx/ /etc/nginx/ 37 | 38 | RUN pip install -U pip setuptools 39 | 40 | # Install gunicorn 41 | COPY gunicorn/ /etc/gunicorn/ 42 | RUN pip install -r /etc/gunicorn/requirements.txt 43 | 44 | EXPOSE 8000 45 | WORKDIR /app 46 | 47 | COPY django-entrypoint.sh celery-entrypoint.sh \ 48 | /scripts/ 49 | ENTRYPOINT ["tini", "--", "django-entrypoint.sh"] 50 | CMD [] 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019, Praekelt.org 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of docker-django-bootstrap nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docker-django-bootstrap 2 | 3 | [![Docker Pulls](https://img.shields.io/docker/pulls/praekeltfoundation/django-bootstrap)](https://hub.docker.com/r/praekeltfoundation/django-bootstrap/) 4 | [![GitHub Workflow](https://img.shields.io/github/workflow/status/praekeltfoundation/docker-django-bootstrap/Continuous%20Integration)](https://github.com/praekeltfoundation/docker-django-bootstrap/actions/workflows/ci.yaml) 5 | 6 | 7 | 8 | Dockerfile for quickly running Django projects in a Docker container. 9 | 10 | Run [Django](https://www.djangoproject.com) projects from source using [Gunicorn](http://gunicorn.org) and [Nginx](http://nginx.org). 11 | 12 | Images are available on [Github Container Registry](https://github.com/orgs/praekeltfoundation/packages?repo_name=docker-django-bootstrap). See [Choosing an image tag](#choosing-an-image-tag). All images are tested using [Seaworthy](https://github.com/praekeltfoundation/seaworthy) before release. 13 | 14 | > **NOTE:** Integration with the [`prometheus_client`](https://github.com/prometheus/client_python) library was recently added to the image. This may impact users who were using that library already. Please read the [metrics](#metrics) documentation for more information. 15 | 16 | For more background on running Django in Docker containers, see [this talk](https://www.youtube.com/watch?v=T2hooQzvurQ) ([slides](https://speakerdeck.com/jayh5/deploying-django-web-applications-in-docker-containers)) from PyConZA 2017. 17 | 18 | ## Index 19 | 1. [Usage](#usage) 20 | - [Step 1: Get your Django project in shape](#step-1-get-your-django-project-in-shape) 21 | - [Step 2: Write a Dockerfile](#step-2-write-a-dockerfile) 22 | - [Step 3: Add a .dockerignore file](#step-3-add-a-dockerignore-file-if-copying-in-the-project-source) 23 | - [Configuring Gunicorn](#configuring-gunicorn) 24 | - [Running other commands](#running-other-commands) 25 | 2. [Celery](#celery) 26 | - [Option 1: Celery containers](#option-1-celery-containers) 27 | - [Option 2: Celery in the same container](#option-2-celery-in-the-same-container) 28 | - [Celery environment variable configuration](#celery-environment-variable-configuration) 29 | 3. [Choosing an image tag](#choosing-an-image-tag) 30 | 4. [Monitoring and metrics](#monitoring-and-metrics) 31 | - [Health checks](#health-checks) 32 | - [Metrics](#metrics) 33 | 5. [Production-readiness](#production-readiness) 34 | 6. [Frequently asked questions](#frequently-asked-questions) 35 | - [How is this deployed?](#how-is-this-deployed) 36 | - [Why is Nginx needed?](#why-is-nginx-needed) 37 | - [What about WhiteNoise?](#what-about-whitenoise) 38 | - [What about Gunicorn's async workers?](#what-about-gunicorns-async-workers) 39 | - [What about Django Channels?](#what-about-django-channels) 40 | 7. [Other configuration](#other-configuration) 41 | - [Gunicorn](#gunicorn) 42 | - [Nginx](#nginx) 43 | 44 | ## Usage 45 | #### Step 1: Get your Django project in shape 46 | There are a few ways that your Django project needs to be set up in order to be compatible with this Docker image. 47 | 48 | **setup.py** 49 | Your project must have a [`setup.py`](https://packaging.python.org/distributing/#setup-py). All dependencies need to be listed in the [`install_requires`](https://packaging.python.org/distributing/#install-requires). 50 | 51 | Your dependencies should include at least: 52 | * `Django` 53 | * `celery` (if using) 54 | * ...but **not** `gunicorn` 55 | 56 | Django *isn't* installed in this image as different projects may use different versions of Django. Celery is completely optional. 57 | 58 | Gunicorn is the only Python package installed in this image. It is kept up-to-date and tested here so you should not be pinning the `gunicorn` package in your application. Gunicorn is considered a deployment detail and your Django project should not rely on its use. 59 | 60 | **Static files** 61 | Your project's [static files](https://docs.djangoproject.com/en/1.10/howto/static-files/) must be set up as follows in your Django settings: 62 | * `STATIC_URL = '/static/'` 63 | * `STATIC_ROOT` = `'static'` (relative) or `'/app/static'` (absolute) 64 | 65 | **Media files** 66 | If your project makes use of user-uploaded media files, it must be set up as follows: 67 | * `MEDIA_URL = '/media/'` 68 | * `MEDIA_ROOT` = `'media'` (relative) or `'/app/media'` (absolute) 69 | 70 | > The `staticfiles` and `mediafiles` directories are also used for serving static and media files, but this is deprecated. 71 | 72 | ***Note:*** Any files stored in directories called `static`, `staticfiles`, `media`, or `mediafiles` in the project root directory (`/app`) will be served by Nginx. Do not store anything here that you do not want the world to see. 73 | 74 | **Django settings file** 75 | You'll probably want to make your Django settings file *Docker-friendly* so that the app is easier to deploy on container-based infrastructure. There are a lot of ways to do this and many project-specific considerations, but the [settings file](tests/django2/mysite/docker_settings.py) in the example project is a good place to start and has lots of documentation. 76 | 77 | #### Step 2: Write a Dockerfile 78 | In the root of the repo for your Django project, add a Dockerfile for the project. For example, this file could contain: 79 | ```dockerfile 80 | FROM praekeltfoundation/django-bootstrap 81 | 82 | COPY . /app 83 | RUN pip install -e . 84 | 85 | ENV DJANGO_SETTINGS_MODULE my_django_project.settings 86 | ENV CELERY_APP my_django_project 87 | 88 | RUN django-admin collectstatic --noinput 89 | 90 | CMD ["my_django_project.wsgi:application"] 91 | ``` 92 | 93 | Let's go through these lines one-by-one: 94 | 1. The `FROM` instruction here tells us which image to base this image on. We use the `django-bootstrap` base image. 95 | 2. Copy the source (in the current working directory-- `.`) of your project into the image (`/app` in the container) 96 | 3. Execute (`RUN`) a `pip` command inside the container to install your project from the source 97 | 4. We set the `DJANGO_SETTINGS_MODULE` environment variable so that Django knows where to find its settings. This is necessary for any `django-admin` commands to work. 98 | 5. *Optional:* If you are using Celery, setting the `CELERY_APP` environment variable lets Celery know what app instance to use (i.e. you don't have to provide [`--app`](http://docs.celeryproject.org/en/latest/reference/celery.bin.celery.html#cmdoption-celery-a)). 99 | 6. *Optional:* If you need to run any build-time tasks, such as collecting static assets, now's the time to do that. 100 | 7. We set the container command (`CMD`) to a list of arguments that will be passed to `gunicorn`. We need to provide Gunicorn with the [`APP_MODULE`](http://docs.gunicorn.org/en/stable/run.html?highlight=app_module#gunicorn), so that it knows which WSGI app to run.* 101 | 102 | > Note that previously the way to do point 5 was to set the `APP_MODULE` environment variable. That still works, but is no longer the recommended way and is deprecated. 103 | 104 | By default, the [`django-entrypoint.sh`](django-entrypoint.sh) script is run when the container is started. This script runs a once-off `django-admin migrate` to update the database schemas and then launches `nginx` and `gunicorn` to run the application. 105 | 106 | The script also allows you to create a Django super user account if needed. Setting the `SUPERUSER_PASSWORD` environment variable will result in a Django superuser account being made with the `admin` username. This will only happen if no `admin` user exists. 107 | 108 | By default the script will run the migrations when starting up. This may not be desirable in all situations. If you want to run migrations separately using `django-admin` then setting the `SKIP_MIGRATIONS` environment variable will result in them not being run. 109 | 110 | By default the script assumes that static files have been collected as part of the Docker build step. If they need to be run on container start with `django-admin collecstatic` then setting the `RUN_COLLECTSTATIC` environment variable will make that happen. 111 | 112 | #### Step 3: Add a `.dockerignore` file (if copying in the project source) 113 | If you are copying the full source of your project into your Docker image (i.e. doing `COPY . /app`), then it is important to add a `.dockerignore` file. 114 | 115 | Add a file called `.dockerignore` to the root of your project. A good start is just to copy in the [`.dockerignore` file](tests/.dockerignore) from the example Django project in this repo. 116 | 117 | When copying in the source of your project, some of those files probably *aren't* needed inside the Docker image you're building. We tell Docker about those unneeded files using a `.dockerignore` file, much like how one would tell Git not to track files using a `.gitignore` file. 118 | 119 | As a general rule, you should list all the files in your `.gitignore` in your `.dockerignore` file. If you don't need it in Git, you shouldn't need it in Docker. 120 | 121 | Additionally, you shouldn't need any *Git* stuff inside your Docker image. It's especially important to have Docker ignore the `.git` directory because every Git operation you perform will result in files changing in that directory (whether you end up in the same state in Git as you were previously or not). This could result in unnecessary invalidation of Docker's cached image layers. 122 | 123 | **NOTE:** Unlike `.gitignore` files, `.dockerignore` files do *not* apply recursively to subdirectories. So, for example, while the entry `*.pyc` in a `.gitignore` file will cause Git to ignore `./abc.pyc` and `./def/ghi.pyc`, in a `.dockerignore` file, that entry will cause Docker to ignore only `./abc.pyc`. This is very unfortunate. In order to get the same behaviour from a `.dockerignore` file, you need to add an extra leading `**/` glob pattern — i.e. `**/*.pyc`. For more information on the `.dockerignore` file syntax, see the [Docker documentation](https://docs.docker.com/engine/reference/builder/#dockerignore-file). 124 | 125 | ### Configuring Gunicorn 126 | The recommended way to specify additional Gunicorn arguments is by using the `CMD` directive in your Dockerfile: 127 | ```dockerfile 128 | CMD ["my_django_project.wsgi:application", "--timeout", "1800"] 129 | ``` 130 | Alternatively, this can also be done at runtime: 131 | ``` 132 | > $ docker run my-django-bootstrap-image my_django_project.wsgi:application --timeout 1800 133 | ``` 134 | 135 | Note that, since Gunicorn 19.7.0, the `GUNICORN_CMD_ARGS` environment variable can also be used to specify arguments: 136 | ```dockerfile 137 | ENV GUNICORN_CMD_ARGS "--timeout 1800" 138 | ``` 139 | Or at runtime: 140 | ``` 141 | > $ docker run -e GUNICORN_CMD_ARGS="--timeout 1800" my-django-bootstrap-image my_django_project.wsgi:application 142 | ``` 143 | Arguments specified via the CLI (i.e. `CMD`) will take precedence over arguments specified via this environment variable. 144 | 145 | See all the settings available for Gunicorn [here](http://docs.gunicorn.org/en/latest/settings.html). A common setting is the number of Gunicorn workers which can be set with the `WEB_CONCURRENCY` environment variable. 146 | 147 | Gunicorn can also be configured using a [configuration file](http://docs.gunicorn.org/en/latest/configure.html#configuration-file). We **do not recommend** this because django-bootstrap already uses a config file to set [some basic options for Gunicorn](#gunicorn). Note that the config file has the _lowest_ precedence of all the configuration methods so any option specified through either the CLI or the environment variable will override the same option in the config file. 148 | 149 | Gunicorn in this image is essentially hard-coded to use a config file at `/etc/gunicorn/config.py`. If you _must_ use your own config file, you could overwrite that file. 150 | 151 | ### Running other commands 152 | You can skip the execution of the `django-entrypoint.sh` script bootstrapping processes and run other commands by overriding the container's launch command. 153 | 154 | You can do this at image build-time by setting the `CMD` directive in your Dockerfile... 155 | ```dockerfile 156 | CMD ["django-admin", "runserver"] 157 | ``` 158 | ...or at runtime by passing an argument to the `docker run` command: 159 | ``` 160 | > $ docker run my-django-bootstrap-image django-admin runserver 161 | ``` 162 | 163 | 164 | If the entrypoint script sees a command for `gunicorn` then it will run all bootstrapping processes (database migration, starting Nginx, etc.). Otherwise, the script will execute the command directly. A special case is Celery, which is described next. 165 | 166 | Note that overwriting the `django-entrypoint.sh` script with a new file is **not supported**. Code in django-bootstrap may change at any time and break compatibility with a custom entrypoint script. 167 | 168 | ## Celery 169 | It's common for Django applications to have [Celery](http://docs.celeryproject.org/en/latest/django/first-steps-with-django.html) workers performing tasks alongside the actual website. Using this image, there are 2 different ways to run Celery: 170 | 171 | 1. Run separate containers for Celery (recommended) 172 | 2. Run Celery alongside the Django site inside the same container (simpler) 173 | 174 | In most cases it makes sense to run each Celery process in a container separate from the Django/Gunicorn one, so as to follow the rule of one(*-ish*) process per container. But in some cases, running a whole bunch of containers for a relatively simple site may be overkill. Additional containers generally have some overhead in terms of CPU and, especially, memory usage. 175 | 176 | Note that, as with Django, your project needs to specify Celery in its `install_requires` in order to use Celery. Celery is not installed in this image by default. 177 | 178 | ### Option 1: Celery containers 179 | To run a Celery container simply override the container command as described earlier. If the `django-entrypoint.sh` script sees a `celery` command, it will instead run the command using the [`celery-entrypoint.sh`](celery-entrypoint.sh) script. This script switches to the correct user to run Celery and sets some basic config options, depending on which Celery command is being run. 180 | 181 | You can override the command in your Dockerfile... 182 | ```dockerfile 183 | CMD ["celery", "worker"] 184 | ``` 185 | ...or at runtime: 186 | ``` 187 | > $ docker run my-django-bootstrap-image celery worker 188 | ``` 189 | 190 | You can also create dedicated Celery images by overriding the image entrypoint: 191 | ```dockerfile 192 | ENTRYPOINT ["dinit", "celery-entrypoint.sh"] 193 | CMD ["worker"] 194 | ``` 195 | The above assume that you have set the `CELERY_APP` environment variable. 196 | 197 | ### Option 2: Celery in the same container 198 | Celery can be run alongside Django/Gunicorn by adjusting a set of environment variables. Setting the `CELERY_WORKER` variable to a non-empty value will enable a Celery worker process. Similarly, setting the `CELERY_BEAT` variable will enable a Celery beat process. 199 | 200 | #### `CELERY_WORKER`: 201 | Set this option to any non-empty value (e.g. `1`) to have a [Celery worker](http://docs.celeryproject.org/en/latest/userguide/workers.html) process run. This requires that `CELERY_APP` is set. 202 | * Required: no 203 | * Default: none 204 | * Celery option: n/a 205 | 206 | #### `CELERY_BEAT`: 207 | Set this option to any non-empty value (e.g. `1`) to have a [Celery beat](http://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html) process run. This requires that `CELERY_APP` is set. 208 | * Required: no 209 | * Default: none 210 | * Celery option: n/a 211 | 212 | Note that when running a Celery worker in this way, the process pool implementation used is the ['solo' pool](http://docs.celeryproject.org/en/latest/internals/reference/celery.concurrency.solo.html). This means that instead of a pair of processes (master/worker) for the Celery worker, there is just one process. This saves on resources. 213 | 214 | The worker is always single-process (the `--concurrency` option is ignored) and is **blocking**. A number of worker configuration options can't be used with this pool implementation. See the [worker guide](http://docs.celeryproject.org/en/latest/userguide/workers.html) in the Celery documentation for more information. 215 | 216 | ### Celery environment variable configuration 217 | The following environment variables can be used to configure Celery, but, other than the `CELERY_APP` variable, you should configure Celery in your Django settings file. See the example project's [settings file](example/mysite/docker_settings.py) for an example of how to do that. 218 | 219 | #### `CELERY_APP`: 220 | * Required: yes, if `CELERY_WORKER` or `CELERY_BEAT` is set. 221 | * Default: none 222 | * Celery option: `-A`/`--app` 223 | 224 |
225 | Deprecated environment variables 226 |
NOTE: The following 3 environment variables are deprecated. They will continue to work for now but it is recommended that you set these values in your Django settings file rather.
227 | 228 |

CELERY_BROKER:

229 | 234 | 235 |

CELERY_LOGLEVEL:

236 | 241 | 242 |

CELERY_CONCURRENCY:

243 | 248 | 249 |
250 | 251 | #### A note on worker processes 252 | By default Celery runs as many worker processes as there are processors. **We instead default to 1 worker process** in this image to ensure containers use a consistent and small amount of resources no matter what kind of host the containers happen to run on. 253 | 254 | If you need more Celery worker processes, you have the choice of either upping the processes per container or running multiple container instances. 255 | 256 | ## Choosing an image tag 257 | See [Packages](https://github.com/orgs/praekeltfoundation/packages?repo_name=docker-django-bootstrap) for the available packages. Select the type of package (`-nw` for `no-wheelhouse`) to view the available tags. 258 | 259 | It's recommended that you pick the most specific tag for what you need, as shorter tags are likely to change their Python and Debian versions over time. `py3` tags currently track the latest Python 3.x version. The default Python version is the latest release and the default operating system is the latest stable Debian release. 260 | 261 | ## Monitoring and metrics 262 | django-bootstrap doesn't implement or mandate any particular monitoring or metrics setup, but we can suggest some ways to go about instrumenting a container based on django-bootstrap. 263 | 264 | ### Health checks 265 | Health checks are important to implement when using an automated container orchestration system like Kubernetes or DC/OS. Health checks can allow these systems to wait for a container to become completely ready before routing user requests to the container. Containers can also be restarted if their health checks start failing. 266 | 267 | There are a few popular libraries available for implementing health checks in Django, such as: 268 | * [`django-health-check`](https://github.com/KristianOellegaard/django-health-check) 269 | * [`django-watchman`](https://github.com/mwarkentin/django-watchman) 270 | * [`django-healthchecks`](https://github.com/mvantellingen/django-healthchecks) 271 | 272 | The [example Django projects](tests) we use to test `django-bootstrap` use a very basic configuration of [`django-health-check`](https://github.com/KristianOellegaard/django-health-check). Health checks can also be implemented from scratch in Django quite easily. 273 | 274 | ### Metrics 275 | Metrics are also very important for ensuring the performance and reliability of your application. [Prometheus](https://prometheus.io) is a popular and modern system for working with metrics and alerts. 276 | 277 | We recommend instrumenting your Django project with [`django-prometheus`](https://github.com/korfuri/django-prometheus) which leverages the [official Prometheus Python client](https://github.com/prometheus/client_python). The [test Django projects](tests) are instrumented in this way. You can also implement custom metrics using the client library. 278 | 279 | One important note is that when using Gunicorn with its default configuration, the Prometheus client **must be used in [multiprocess mode](https://github.com/prometheus/client_python#multiprocess-mode-gunicorn)**. Because Gunicorn is designed with supervised worker processes, the multiprocess mode is necessary to preserve metrics across multiple worker processes or worker restarts. This mode has a number of limitations so you should read the docs and be aware of those. 280 | 281 | django-bootstrap **will configure multiprocess mode** for the Prometheus client if it is detected that multiple workers are configured or the synchronous worker type (the default) is used. 282 | 283 | You can also enable multiprocess mode yourself by setting the `prometheus_multiproc_dir` environment variable to the path for a directory to be used for temporary files. If you set this variable, django-bootstrap will attempt to create that directory and its parents. 284 | 285 | Note that multiprocess mode requires that metrics are temporarily written to disk and so may have performance implications. 286 | 287 | ## Production-readiness 288 | django-bootstrap has been used in production at [Praekelt.org](https://www.praekelt.org) for several years now for thousands of containers serving millions of users around the world. django-bootstrap was designed to encapsulate many of our best practices for deploying production-ready Django. 289 | 290 | That said, care should be taken in configuring containers at runtime with the appropriate settings. Here are a few points to check on: 291 | * If using Gunicorn's default synchronous workers, you should set the `WEB_CONCURRENCY` environment variable to some number greater than 1 (the default). Gunicorn has some [recommendations](http://docs.gunicorn.org/en/latest/design.html#how-many-workers). 292 | * Consider mounting the `/run` directory as a `tmpfs` volume. This can help improve performance consistency due to [the way Gunicorn handles signaling](http://docs.gunicorn.org/en/latest/faq.html#blocking-os-fchmod) between workers. 293 | 294 | ## Frequently asked questions 295 | ### How is this deployed? 296 | This will depend very much on your infrastructure. This Docker image was designed with an architecture like this in mind: 297 | 298 | Architecture diagram 299 | 300 | django-bootstrap does not require that you use PostgreSQL or RabbitMQ. You can configure Django however you would like, those are just the systems we use with it. 301 | 302 | The image was also designed to be used with a container orchestration system. We use Mesosphere DC/OS but it should work just as well on Kubernetes. We run services that require persistent storage, such as databases or message brokers, outside of our container orchestration system. This looks something like this: 303 | 304 | Container diagram 305 | 306 | Generally a single image based on django-bootstrap takes on the role of running Django or Celery depending on how each container running the image is configured. 307 | 308 | ### Why is Nginx needed? 309 | The primary reason Nginx is necessary is that a key part of Gunicorn's design relies on a proxy to buffer incoming requests. This design goes back to the original [design of Unicorn for Ruby](https://bogomips.org/unicorn/DESIGN.html): 310 | > [...] neither keepalive nor pipelining are supported. These aren't needed since Unicorn is only designed to serve fast, low-latency clients directly. Do one thing, do it well; let nginx handle slow clients. 311 | 312 | You can also read about this in [Gunicorn's design documentation](http://docs.gunicorn.org/en/latest/design.html). So, when using Gunicorn (with the default "sync workers"), it's critical that a buffering proxy (such as Nginx) is used. 313 | 314 | In addition to this reason, Nginx is used to perform the following functions: 315 | * Serves static files for Django which it can do very efficiently, rather than requiring Python code to do so. 316 | * Performs some basic optimisations such as gzipping responses, setting some cache-control headers for static files, etc. 317 | * Adjusts some headers received by clients (see [Other configuration: Nginx](#nginx)). 318 | 319 | ### What about WhiteNoise? 320 | [WhiteNoise](http://whitenoise.evans.io) is a library to simplify static file serving with Python webservers that integrates with Django. It encapsulates a lot of best-practices and useful optimisations when serving static files. In fact, some of the optimisations it uses we copied and used in the Nginx configuration for django-bootstrap. 321 | 322 | WhiteNoise is not typically used in conjunction with a static file-serving reverse proxy and so we don't recommend using it with django-bootstrap. Additionally, WhiteNoise [does not support serving Django media files](http://whitenoise.evans.io/en/stable/django.html#serving-media-files)--which is **not** a thing we recommend you do, for the reasons outlined in the WhiteNoise documentation--but a requirement we have had for some of our projects. 323 | 324 | WhiteNoise does not solve the problem of buffering requests for Gunicorn's workers. 325 | 326 | ### What about Gunicorn's async workers? 327 | Gunicorn does provide various implementations of asynchronous workers. See [Choosing a Worker Type](http://docs.gunicorn.org/en/latest/design.html#choosing-a-worker-type). These asynchronous workers can work well with django-bootstrap. 328 | 329 | When using async workers, it could be more practical to use WhiteNoise without Nginx, but that is beyond the scope of this project. 330 | 331 | The sync worker type is simple, easy to reason about, and can scale well when deployed properly and used for its intended purpose. 332 | 333 | ### What about Django Channels? 334 | [Django Channels](https://channels.readthedocs.io) extends Django for protocols beyond HTTP/1.1 and generally enables Django to be used for more asynchronous applications. Django Channels does not use WSGI and instead uses a protocol called [Asynchronous Server Gateway Interface (ASGI)](https://channels.readthedocs.io/en/latest/asgi.html). Gunicorn does not support ASGI and instead the reference ASGI server implementation, [Daphne](https://github.com/django/daphne/), is typically used instead. 335 | 336 | Django Channels is beyond the scope of this project. We may one day start a `docker-django-channels` project, though :wink:. 337 | 338 | ### What about using container groups (i.e. pods)? 339 | django-bootstrap currently runs both Nginx and Gunicorn processes in the same container. It is generally considered best-practice to run only one thing inside a container. Technically, it would be possible to run Nginx and Gunicorn in separate containers that are grouped together and share some volumes. The idea of a "pod" of containers was popularised by Kubernetes. Containers in a pod are typically co-located, so sharing files between the containers is practical: 340 | 341 | Pod diagram 342 | 343 | This is a direction we want to take the project, but currently our infrastructure does not support the pod pattern. We have experimented with this [before](https://github.com/praekeltfoundation/docker-django-bootstrap/pull/69) and would welcome pull requests. 344 | 345 | ## Other configuration 346 | ### Gunicorn 347 | Gunicorn is run with some basic configuration using the [config file](gunicorn/config.py) at `/etc/gunicorn/config.py`: 348 | * Listens on a Unix socket at `/run/gunicorn/gunicorn.sock` 349 | * Places a PID file at `/run/gunicorn/gunicorn.pid` 350 | * [Worker temporary files](http://docs.gunicorn.org/en/latest/settings.html#worker-tmp-dir) are placed in `/run/gunicorn` 351 | * Access logs can be logged to stderr by setting the `GUNICORN_ACCESS_LOGS` environment variable to a non-empty value. 352 | 353 | ### Nginx 354 | Nginx is set up with mostly default config: 355 | * Access logs are sent to stdout, error logs to stderr and log messages are formatted to be JSON-compatible for easy parsing. 356 | * Listens on port 8000 (and this port is exposed in the Dockerfile) 357 | * Has gzip compression enabled for most common, compressible mime types 358 | * Serves files from `/static/` and `/media/` 359 | * All other requests are proxied to the Gunicorn socket 360 | 361 | Generally you shouldn't need to adjust Nginx's settings. If you do, the configuration is split into several files that can be overridden individually: 362 | * `/etc/nginx/nginx.conf`: Main configuration (including logging and gzip compression) 363 | * `/etc/nginx/conf.d/` 364 | * `django.conf`: The primary server configuration 365 | * `django.conf.d/` 366 | * `upstream.conf`: Upstream connection to Gunicorn 367 | * `locations/*.conf`: Each server location (static, media, root) 368 | * `maps/*.conf`: Nginx maps for setting variables 369 | 370 | We make a few adjustments to Nginx's default configuration to better work with Gunicorn. See the [config file](nginx/conf.d/django.conf) for all the details. One important point is that we consider the `X-Forwarded-Proto` header, when set to the value of `https`, as an indicator that the client connection was made over HTTPS and is secure. Gunicorn considers a few more headers for this purpose, `X-Forwarded-Protocol` and `X-Forwarded-Ssl`, but our Nginx config is set to remove those headers to prevent misuse. 371 | -------------------------------------------------------------------------------- /alpine/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM praekeltfoundation/python-base:alpine 2 | MAINTAINER Praekelt Foundation 3 | 4 | # Install libpq for PostgreSQL support and Nginx to serve everything 5 | RUN apk --no-cache add libpq nginx 6 | 7 | # Install gunicorn 8 | COPY ./requirements.txt /requirements.txt 9 | RUN pip install -r /requirements.txt 10 | 11 | # Copy in the Nginx config 12 | COPY ./nginx/ /etc/nginx/ 13 | 14 | # Create gunicorn user and group, make directory for socket, and add nginx user 15 | # to gunicorn group so that it can read/write to the socket. 16 | RUN addgroup -S gunicorn \ 17 | && adduser -S -G gunicorn gunicorn \ 18 | && mkdir /var/run/gunicorn \ 19 | && chown gunicorn:gunicorn /var/run/gunicorn \ 20 | && adduser nginx gunicorn 21 | 22 | # Create celery user and group, make directory for beat schedule file. 23 | RUN addgroup -S celery \ 24 | && adduser -S -G celery celery \ 25 | && mkdir /var/run/celery \ 26 | && chown celery:celery /var/run/celery 27 | 28 | EXPOSE 8000 29 | 30 | COPY ./django-entrypoint.sh /scripts/ 31 | CMD ["django-entrypoint.sh"] 32 | 33 | WORKDIR /app 34 | -------------------------------------------------------------------------------- /alpine/README.md: -------------------------------------------------------------------------------- 1 | # Alpine Linux 2 | 3 | Alpine Linux images were maintained at one point but we ended up not using them very often. The source is kept here for posterity and in case we change our minds. 4 | -------------------------------------------------------------------------------- /alpine/example/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM praekeltfoundation/django-bootstrap:alpine-onbuild 2 | ENV DJANGO_SETTINGS_MODULE "mysite.settings" 3 | RUN django-admin collectstatic --noinput 4 | ENV APP_MODULE "mysite.wsgi:application" 5 | -------------------------------------------------------------------------------- /alpine/onbuild/alpine.dockerfile: -------------------------------------------------------------------------------- 1 | FROM praekeltfoundation/django-bootstrap:alpine 2 | ONBUILD COPY . /app 3 | # chown the app directory after copying in case the copied files include 4 | # subdirectories that will be written to, e.g. the media directory 5 | ONBUILD RUN chown -R gunicorn:gunicorn /app 6 | ONBUILD RUN pip install -e . 7 | -------------------------------------------------------------------------------- /celery-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | set -e 3 | 4 | _is_celery_command () { 5 | local cmd="$1"; shift 6 | 7 | python - <&2 27 | set -- "$@" --broker "$CELERY_BROKER" 28 | fi 29 | 30 | if [ -n "$CELERY_LOGLEVEL" ]; then 31 | echo 'DEPRECATED: The CELERY_LOGLEVEL environment variable is deprecated. 32 | Please set the Celery log level in your Django settings file rather.' 1>&2 33 | set -- "$@" --loglevel "$CELERY_LOGLEVEL" 34 | fi 35 | 36 | # Set the concurrency if this is a worker 37 | if [ "$2" = 'worker' ]; then 38 | if [ -n "$CELERY_CONCURRENCY" ]; then 39 | echo 'DEPRECATED: The CELERY_CONCURRENCY environment variable is deprecated. 40 | Please set the Celery worker concurrency in your Django settings file rather.' 1>&2 41 | fi 42 | set -- "$@" --concurrency "${CELERY_CONCURRENCY:-1}" 43 | fi 44 | 45 | # Run under the celery user 46 | set -- su-exec django "$@" 47 | 48 | # Create the Celery runtime directory at runtime in case /run is a tmpfs 49 | if mkdir /run/celery 2> /dev/null; then 50 | chown django:django /run/celery 51 | fi 52 | # Celery by default writes files like pidfiles and the beat schedule file to 53 | # the current working directory. Change to the Celery working directory so 54 | # that these files end up there. 55 | cd /run/celery 56 | fi 57 | 58 | exec "$@" 59 | -------------------------------------------------------------------------------- /django-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | set -e 3 | 4 | # No args or looks like options or the APP_MODULE for Gunicorn 5 | if [ "$#" = 0 ] || \ 6 | [ "${1#-}" != "$1" ] || \ 7 | echo "$1" | grep -Eq '^([_A-Za-z]\w*\.)*[_A-Za-z]\w*:[_A-Za-z]\w*$'; then 8 | set -- gunicorn "$@" 9 | fi 10 | 11 | # Looks like a Celery command, let's run that with Celery's entrypoint script 12 | if [ "$1" = 'celery' ]; then 13 | set -- celery-entrypoint.sh "$@" 14 | fi 15 | 16 | if [ "$1" = 'gunicorn' ]; then 17 | # Do a chown of the /app/media & /app/mediafiles directories (if they exist) 18 | # at runtime in case the directory was mounted as a root-owned volume. 19 | for media in /app/media /app/mediafiles; do 20 | if [ -d $media ] && [ "$(stat -c %U $media)" != 'django' ]; then 21 | chown -R django:django $media 22 | fi 23 | done 24 | 25 | # Run the migration as the django user so that if it creates a local DB 26 | # (e.g. when using sqlite in development), that DB is still writable. 27 | # Ultimately, the user shouldn't really be using a local DB and it's difficult 28 | # to offer support for all the cases in which a local DB might be created -- 29 | # but here we do the minimum. 30 | if [ -z "$SKIP_MIGRATIONS" ]; then 31 | su-exec django django-admin migrate --noinput 32 | fi 33 | 34 | # Allow running of collectstatic command because it might require env vars 35 | if [ -n "$RUN_COLLECTSTATIC" ]; then 36 | su-exec django django-admin collectstatic --noinput 37 | fi 38 | 39 | if [ -n "$SUPERUSER_PASSWORD" ]; then 40 | echo "from django.contrib.auth.models import User 41 | if not User.objects.filter(username='admin').exists(): 42 | User.objects.create_superuser('admin', 'admin@example.com', '$SUPERUSER_PASSWORD') 43 | " | su-exec django django-admin shell 44 | echo "Created superuser with username 'admin' and password '$SUPERUSER_PASSWORD'" 45 | fi 46 | 47 | nginx -g 'daemon off;' & 48 | 49 | # Celery 50 | ensure_celery_app() { 51 | [ -n "$CELERY_APP" ] || \ 52 | { echo 'If $CELERY_WORKER or $CELERY_BEAT are set then $CELERY_APP must be provided'; exit 1; } 53 | } 54 | 55 | if [ -n "$CELERY_WORKER" ]; then 56 | ensure_celery_app 57 | celery-entrypoint.sh worker --pool=solo --pidfile worker.pid & 58 | fi 59 | 60 | if [ -n "$CELERY_BEAT" ]; then 61 | ensure_celery_app 62 | celery-entrypoint.sh beat --pidfile beat.pid & 63 | fi 64 | 65 | if [ -n "$APP_MODULE" ]; then 66 | echo 'DEPRECATED: Providing APP_MODULE via an environment variable is deprecated. 67 | Please provide it using the container command rather.' 1>&2 68 | set -- "$@" "$APP_MODULE" 69 | fi 70 | 71 | # Create the Gunicorn runtime directory at runtime in case /run is a tmpfs 72 | if mkdir /run/gunicorn 2> /dev/null; then 73 | chown django:django /run/gunicorn 74 | fi 75 | 76 | set -- su-exec django "$@" --config /etc/gunicorn/config.py 77 | fi 78 | 79 | exec "$@" 80 | -------------------------------------------------------------------------------- /gunicorn/config.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import os 3 | 4 | from gunicorn.workers.sync import SyncWorker 5 | 6 | # See http://docs.gunicorn.org/en/latest/settings.html for a list of available 7 | # settings. Note that the setting names are used here and not the CLI option 8 | # names (e.g. "pidfile", not "pid"). 9 | 10 | # Set some sensible Gunicorn options, needed for things to work with Nginx 11 | pidfile = "/run/gunicorn/gunicorn.pid" 12 | bind = "unix:/run/gunicorn/gunicorn.sock" 13 | # umask working files (worker tmp files & unix socket) as 0o117 (i.e. chmod as 14 | # 0o660) so that they are only read/writable by django and nginx users. 15 | umask = 0o117 16 | # Set the worker temporary file directory to /run/gunicorn (rather than default 17 | # of /tmp) so that all of Gunicorn's files are in one place and a tmpfs can be 18 | # mounted at /run for better performance. 19 | # http://docs.gunicorn.org/en/latest/faq.html#blocking-os-fchmod 20 | worker_tmp_dir = "/run/gunicorn" 21 | 22 | if os.environ.get("GUNICORN_ACCESS_LOGS"): 23 | accesslog = "-" 24 | 25 | 26 | DEFAULT_PROMETHEUS_MULTIPROC_DIR = "/run/gunicorn/prometheus" 27 | 28 | 29 | def nworkers_changed(server, new_value, old_value): 30 | # Configure the prometheus_multiproc_dir value. This may seem like a 31 | # strange place to do that, but it's the only callback that gets called 32 | # before the WSGI app is setup if app preloading is enabled. 33 | 34 | # We only care about the first time the number of workers is set--during 35 | # setup. At this point the old_value is None. 36 | if old_value is not None: 37 | return 38 | 39 | # If there are multiple processes (num_workers > 1) or the workers are 40 | # synchronous (in which case in production the num_workers will need to be 41 | # >1), enable multiprocess mode by default. 42 | if server.num_workers > 1 or server.worker_class == SyncWorker: 43 | # Don't override an existing value 44 | if "prometheus_multiproc_dir" not in os.environ: 45 | os.environ["prometheus_multiproc_dir"] = ( 46 | DEFAULT_PROMETHEUS_MULTIPROC_DIR) 47 | 48 | # Try to create the prometheus_multiproc_dir if set but fail gracefully 49 | if "prometheus_multiproc_dir" in os.environ: 50 | path = os.environ["prometheus_multiproc_dir"] 51 | # mkdir -p equivalent: https://stackoverflow.com/a/600612/3077893 52 | try: 53 | os.makedirs(path) 54 | except OSError as e: 55 | if e.errno != errno.EEXIST or not os.path.isdir(path): 56 | server.log.warning( 57 | ("Unable to create prometheus_multiproc_dir directory at " 58 | "'%s'"), path, exc_info=e) 59 | 60 | 61 | def worker_exit(server, worker): 62 | # Do bookkeeping for Prometheus collectors for each worker process as they 63 | # exit, as described in the prometheus_client documentation: 64 | # https://github.com/prometheus/client_python#multiprocess-mode-gunicorn 65 | if "prometheus_multiproc_dir" in os.environ: 66 | # Don't error if the environment variable has been set but 67 | # prometheus_client isn't installed 68 | try: 69 | from prometheus_client import multiprocess 70 | except ImportError: 71 | return 72 | 73 | multiprocess.mark_process_dead(worker.pid) 74 | -------------------------------------------------------------------------------- /gunicorn/requirements.txt: -------------------------------------------------------------------------------- 1 | gunicorn==20.1.0 2 | -------------------------------------------------------------------------------- /images/pod.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nginx/conf.d/django.conf: -------------------------------------------------------------------------------- 1 | include conf.d/django.conf.d/upstream.conf; 2 | include conf.d/django.conf.d/maps/*.conf; 3 | 4 | server { 5 | listen 8000; 6 | 7 | root /app; 8 | 9 | include conf.d/django.conf.d/locations/*.conf; 10 | } 11 | -------------------------------------------------------------------------------- /nginx/conf.d/django.conf.d/locations/media.conf: -------------------------------------------------------------------------------- 1 | location ~ ^/media/?(.*)$ { 2 | # Fallback for projects still using MEDIA_ROOT = BASE_DIR/mediafiles 3 | try_files /media/$1 /mediafiles/$1 =404; 4 | } 5 | -------------------------------------------------------------------------------- /nginx/conf.d/django.conf.d/locations/root.conf: -------------------------------------------------------------------------------- 1 | location / { 2 | client_max_body_size 20m; 3 | proxy_pass http://gunicorn; 4 | 5 | proxy_set_header Host $http_host; 6 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 7 | 8 | # We only use the 'X-Forwarded-Proto' header from our load-balancer to 9 | # indicate the original connection used HTTPS, but Gunicorn by default 10 | # accepts more headers than that: 11 | # http://docs.gunicorn.org/en/19.7.1/settings.html#secure-scheme-headers 12 | # Overriding that config in Gunicorn is a bit complicated, and could 13 | # easily be overriden by accident by the user, so just delete those 14 | # other headers here so that a client can't set them 15 | # incorrectly/maliciously. 16 | proxy_set_header X-Forwarded-Protocol ""; 17 | proxy_set_header X-Forwarded-Ssl ""; 18 | } 19 | -------------------------------------------------------------------------------- /nginx/conf.d/django.conf.d/locations/static.conf: -------------------------------------------------------------------------------- 1 | location ~ ^/static/?(.*)$ { 2 | # Fallback for projects still using STATIC_ROOT = BASE_DIR/staticfiles 3 | # as recommended by WhiteNoise 4 | try_files /static/$1 /staticfiles/$1 =404; 5 | add_header Cache-Control $static_cache_control; 6 | } 7 | -------------------------------------------------------------------------------- /nginx/conf.d/django.conf.d/maps/static_cache_control.conf: -------------------------------------------------------------------------------- 1 | # Detect filenames for static files that look like they contain MD5 hashes as 2 | # these can be cached indefinitely. 3 | 4 | # Nginx's 'expires max' directive sets the Cache-Control header to have a max- 5 | # age of 10 years. It also sets the Expires header to a certain date in 2037: 6 | # http://nginx.org/en/docs/http/ngx_http_headers_module.html#expires 7 | 8 | # We also want to add the 'public' and 'immutable' values to the Cache-Control 9 | # header to prevent clients from revalidating files that we know are immutable: 10 | # https://bitsup.blogspot.co.za/2016/05/cache-control-immutable.html 11 | 12 | # Using 'expires max' makes adding other Cache-Control values tricky and the 13 | # The Expires header should be ignored when a max-age is set for Cache-Control: 14 | # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expires 15 | 16 | # So, avoid 'expires max' & just add the header manually with a 10-year max-age. 17 | map $uri $static_cache_control { 18 | # ManifestStaticFilesStorage files have a hash in the middle of the filename 19 | "~^/static/.*/[^/\.]+\.[a-f0-9]{12}\.\w+$" "max-age=315360000, public, immutable"; 20 | 21 | # django-compressor cached files are js/css with a hash filename 22 | "~^/static/CACHE/(js|css)/[a-f0-9]{12}\.(js|css)$" "max-age=315360000, public, immutable"; 23 | 24 | # For the default, copy what WhiteNoise does and set a short max-age 25 | default "max-age=60, public"; 26 | } 27 | -------------------------------------------------------------------------------- /nginx/conf.d/django.conf.d/upstream.conf: -------------------------------------------------------------------------------- 1 | upstream gunicorn { 2 | # Proxy to Gunicorn socket and always retry, as recommended by deployment 3 | # guide: http://docs.gunicorn.org/en/stable/deploy.html 4 | server unix:/run/gunicorn/gunicorn.sock max_fails=0; 5 | } 6 | -------------------------------------------------------------------------------- /nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | # Config is the default provided in the Debian Nginx 1.14.2 package--as used by 2 | # the official Nginx Docker images. Adjustments made as described in the README. 3 | # https://github.com/nginxinc/docker-nginx/blob/1.14.2/stable/alpine/nginx.conf 4 | 5 | user nginx; 6 | worker_processes 1; 7 | 8 | error_log /dev/stderr warn; 9 | pid /run/nginx.pid; 10 | 11 | 12 | events { 13 | worker_connections 1024; 14 | } 15 | 16 | 17 | http { 18 | include /etc/nginx/mime.types; 19 | default_type application/octet-stream; 20 | 21 | # Use a JSON-compatible log format. The default Nginx log format has 22 | # unlabeled fields that makes it tricky to parse. Since Nginx 1.11.8, 23 | # `escape=json` is available to escape variables. 24 | log_format main escape=json 25 | '{ ' 26 | '"time": "$time_iso8601", ' 27 | '"remote_addr": "$remote_addr", ' 28 | '"remote_user": "$remote_user", ' 29 | '"request": "$request", ' 30 | '"status": $status, ' 31 | '"body_bytes_sent": $body_bytes_sent, ' 32 | '"request_time": $request_time, ' 33 | '"http_host": "$http_host", ' 34 | '"http_referer": "$http_referer", ' 35 | '"http_user_agent": "$http_user_agent", ' 36 | '"http_via": "$http_via", ' 37 | '"http_x_forwarded_proto": "$http_x_forwarded_proto", ' 38 | '"http_x_forwarded_for": "$http_x_forwarded_for" ' 39 | '}'; 40 | 41 | access_log /dev/stdout main; 42 | 43 | sendfile on; 44 | #tcp_nopush on; 45 | 46 | keepalive_timeout 65; 47 | 48 | gzip on; 49 | gzip_disable "msie6"; 50 | # Allow gzip responses for proxied requests. Our load-balancers don't set 51 | # this header, but some other things do. We can reasonably trust any vaguely 52 | # modern proxy to handle gzip correctly, and change this if not. 53 | gzip_proxied any; 54 | # Don't gzip tiny requests-- anything smaller than this should roughly fit 55 | # inside the MTU. 56 | gzip_min_length 1024; 57 | # Pick a balance of compression vs CPU time. 6 is the default for the gzip 58 | # library and leans slightly more towards better compression. 59 | gzip_comp_level 6; 60 | # Include 'Accept-Encoding' in the 'Vary' header so downstream caches work 61 | gzip_vary on; 62 | 63 | # Roughly based on https://www.fastly.com/blog/new-gzip-settings-and-deciding-what-compress 64 | # ...with a few adjustments because we control what content-types we send 65 | gzip_types 66 | # Applications 67 | application/javascript 68 | application/json 69 | application/xml 70 | 71 | # Fonts (WOFF already compressed) 72 | application/vnd.ms-fontobject 73 | # No .otf/.ttf in default Nginx mimetype mapping 74 | 75 | # Images (most already compressed) 76 | image/svg+xml 77 | image/x-icon # 'x-icon' seems to be used more commonly than 'vnd.microsoft.icon' 78 | 79 | # Text types 80 | # text/html is compressed by default: 81 | # https://nginx.org/en/docs/http/ngx_http_gzip_module.html#gzip_types 82 | text/css 83 | text/plain; 84 | 85 | include conf.d/*.conf; 86 | } 87 | -------------------------------------------------------------------------------- /tests/.dockerignore: -------------------------------------------------------------------------------- 1 | # Docker-specific things: 2 | # Docker files 3 | Dockerfile 4 | *.dockerfile 5 | .dockerignore 6 | 7 | # Git files 8 | .git/ 9 | **/.gitignore 10 | 11 | # Adapted from the standard Python .gitignore 12 | # https://github.com/github/gitignore/blob/master/Python.gitignore 13 | # 455a69dd48ce041f6ac2aa7aeeb9560957311e2f 14 | 15 | # Byte-compiled / optimized / DLL files 16 | **/__pycache__/ 17 | **/*.py[cod] 18 | **/*$py.class 19 | 20 | # C extensions 21 | *.so 22 | 23 | # Distribution / packaging 24 | .Python 25 | env/ 26 | build/ 27 | develop-eggs/ 28 | dist/ 29 | downloads/ 30 | eggs/ 31 | .eggs/ 32 | lib/ 33 | lib64/ 34 | parts/ 35 | sdist/ 36 | var/ 37 | *.egg-info/ 38 | .installed.cfg 39 | *.egg 40 | 41 | # PyInstaller 42 | # Usually these files are written by a python script from a template 43 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 44 | *.manifest 45 | *.spec 46 | 47 | # Installer logs 48 | pip-log.txt 49 | pip-delete-this-directory.txt 50 | 51 | # Unit test / coverage reports 52 | htmlcov/ 53 | .tox/ 54 | .coverage 55 | .coverage.* 56 | .cache 57 | nosetests.xml 58 | coverage.xml 59 | *,cover 60 | .hypothesis/ 61 | 62 | # Translations 63 | **/*.mo 64 | **/*.pot 65 | 66 | # Django stuff: 67 | *.log 68 | local_settings.py 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | target/ 82 | 83 | # IPython Notebook 84 | .ipynb_checkpoints 85 | 86 | # pyenv 87 | .python-version 88 | 89 | # celery beat schedule file 90 | celerybeat-schedule 91 | 92 | # dotenv 93 | .env 94 | 95 | # virtualenv 96 | venv/ 97 | ENV/ 98 | 99 | # Spyder project settings 100 | .spyderproject 101 | 102 | # Rope project settings 103 | .ropeproject 104 | -------------------------------------------------------------------------------- /tests/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | # `django-admin startproject` generates some files with lines longer than 80 3 | # characters...let's just let it do that. 4 | max-line-length = 100 5 | ignore = 6 | # E741: We use "l" as a variable in listcomps, this is fine. 7 | E741 8 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | # Standard Python .gitignore 2 | # https://github.com/github/gitignore/blob/master/Python.gitignore 3 | # 455a69dd48ce041f6ac2aa7aeeb9560957311e2f 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | env/ 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # IPython Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | 93 | # Rope project settings 94 | .ropeproject 95 | -------------------------------------------------------------------------------- /tests/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=praekeltfoundation/django-bootstrap:py3-stretch 2 | FROM $BASE_IMAGE 3 | 4 | RUN command -v ps > /dev/null || apt-get-install.sh procps 5 | 6 | ARG PROJECT=django2 7 | COPY ${PROJECT} /app/ 8 | 9 | RUN pip install -e . 10 | 11 | ENV DJANGO_SETTINGS_MODULE mysite.docker_settings 12 | ENV CELERY_APP mysite 13 | 14 | RUN django-admin collectstatic --noinput \ 15 | && django-admin compress 16 | 17 | CMD ["mysite.wsgi:application"] 18 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Tests 2 | Example Django projects for testing. Generated using 3 | ```shell 4 | django-admin startproject mysite 5 | ``` 6 | 7 | Two example projects were set up: 8 | * `django1` with Django 1.11, the final Django version to support Python 2.7. 9 | * `django2` with Django 2+. This should track the latest Django version and work with Python 3 images. 10 | 11 | A `setup.py` was added to install dependencies. An example [Django settings file](mysite/docker_settings.py) was also added to make configuration in a Docker container easier. An example Celery setup (see [`celery.py`](mysite/celery.py)) was added as well. 12 | 13 | [django-compressor](https://django-compressor.readthedocs.io) is set up to compress some JavaScript and CSS in a dummy template. 14 | 15 | ## Usage 16 | To build the example site Docker image and run tests on it, use commands like this: 17 | ``` 18 | docker build --tag mysite --build-arg VARIANT=py2-stretch --build-arg PROJECT=django1 . 19 | pytest test.py --django-bootstrap-image=mysite 20 | ``` 21 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def pytest_addoption(parser): 5 | parser.addoption( 6 | '--django-bootstrap-image', action='store', 7 | default=os.environ.get('DJANGO_BOOTSTRAP_IMAGE', 'mysite:py3'), 8 | help='django-bootstrap docker image to test') 9 | 10 | 11 | def pytest_report_header(config): 12 | return 'django-bootstrap docker image: {}'.format( 13 | config.getoption('--django-bootstrap-image')) 14 | -------------------------------------------------------------------------------- /tests/definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | import pytest 5 | 6 | from seaworthy.client import wait_for_response 7 | from seaworthy.containers.postgresql import PostgreSQLContainer 8 | from seaworthy.containers.rabbitmq import RabbitMQContainer 9 | from seaworthy.definitions import ContainerDefinition 10 | from seaworthy.ps import list_container_processes 11 | from seaworthy.utils import output_lines 12 | 13 | 14 | DDB_IMAGE = pytest.config.getoption('--django-bootstrap-image') 15 | 16 | DEFAULT_WAIT_TIMEOUT = int(os.environ.get('DEFAULT_WAIT_TIMEOUT', '30')) 17 | 18 | db_container = PostgreSQLContainer(wait_timeout=DEFAULT_WAIT_TIMEOUT) 19 | 20 | amqp_container = RabbitMQContainer( 21 | vhost='/mysite', wait_timeout=DEFAULT_WAIT_TIMEOUT) 22 | 23 | default_env = { 24 | 'SECRET_KEY': 'secret', 25 | 'ALLOWED_HOSTS': 'localhost,127.0.0.1,0.0.0.0', 26 | 'CELERY_BROKER_URL': amqp_container.broker_url(), 27 | 'DATABASE_URL': db_container.database_url(), 28 | } 29 | 30 | 31 | class _BaseContainerDefinition(ContainerDefinition): 32 | WAIT_TIMEOUT = DEFAULT_WAIT_TIMEOUT 33 | 34 | def list_processes(self): 35 | return list_container_processes(self.inner()) 36 | 37 | def exec_run(self, args, **kwargs): 38 | return output_lines(self.inner().exec_run(args, **kwargs)) 39 | 40 | def exec_find(self, params): 41 | return self.exec_run(['find'] + params) 42 | 43 | def exec_stat(self, *paths, format='%a %U:%G'): 44 | return self.exec_run(['stat'] + ['--format', format] + list(paths)) 45 | 46 | def django_maj_version(self): 47 | [version] = self.exec_run( 48 | ['python', '-c', 'import django; print(django.__version__)']) 49 | return int(version.split('.')[0]) 50 | 51 | 52 | class GunicornContainer(_BaseContainerDefinition): 53 | def wait_for_start(self): 54 | # Override wait_for_start to wait for the health check to succeed. 55 | # Still wait for log lines to match because we also need to wait for 56 | # Celery to start in the single-container setup. 57 | start = time.monotonic() 58 | super().wait_for_start() 59 | 60 | remaining = self.wait_timeout - (time.monotonic() - start) 61 | wait_for_response(self.http_client(), remaining, path='/health/', 62 | expected_status_code=200) 63 | 64 | @classmethod 65 | def for_fixture(cls, name, wait_lines, command=None, env_extra={}): 66 | env = dict(default_env) 67 | env.update(env_extra) 68 | kwargs = { 69 | 'command': command, 70 | 'environment': env, 71 | 'tmpfs': { 72 | # Everything in /run should be ephemeral and created at runtime 73 | '/run': '', 74 | # Add a tmpfs mount at /app/media so that we can test ownership 75 | # of the directory is set. Normally a proper volume would be 76 | # mounted but the effect is the same. 77 | '/app/media': 'uid=0' 78 | }, 79 | 'ports': {'8000/tcp': ('127.0.0.1',)} 80 | } 81 | 82 | return cls(name, DDB_IMAGE, wait_lines, create_kwargs=kwargs) 83 | 84 | def pytest_fixture(self, name): 85 | return super().pytest_fixture( 86 | name, dependencies=('db_container', 'amqp_container')) 87 | 88 | @classmethod 89 | def make_fixture(cls, fixture_name, name, *args, **kw): 90 | return cls.for_fixture(name, *args, **kw).pytest_fixture(fixture_name) 91 | 92 | 93 | class CeleryContainer(_BaseContainerDefinition): 94 | @classmethod 95 | def for_fixture(cls, celery_command, wait_lines): 96 | kwargs = { 97 | 'command': ['celery', celery_command], 98 | 'environment': default_env, 99 | } 100 | return cls(celery_command, DDB_IMAGE, wait_lines, create_kwargs=kwargs) 101 | 102 | 103 | single_container = GunicornContainer.for_fixture( 104 | 'web', [r'Booting worker', r'celery@\w+ ready', r'beat: Starting\.\.\.'], 105 | env_extra={'CELERY_WORKER': '1', 'CELERY_BEAT': '1'}) 106 | 107 | 108 | web_container = GunicornContainer.for_fixture('web', [r'Booting worker']) 109 | 110 | 111 | worker_container = CeleryContainer.for_fixture('worker', [r'celery@\w+ ready']) 112 | 113 | 114 | beat_container = CeleryContainer.for_fixture('beat', [r'beat: Starting\.\.\.']) 115 | 116 | 117 | def make_combined_fixture(base): 118 | """ 119 | This creates a parameterised fixture that allows us to run a single test 120 | with both a special-purpose container and the all-in-one container. 121 | """ 122 | name = '{}_container'.format(base) 123 | fixtures = ['single_container', '{}_only_container'.format(base)] 124 | 125 | @pytest.fixture(name=name, params=fixtures) 126 | def containers(request): 127 | yield request.getfixturevalue(request.param) 128 | 129 | return containers 130 | -------------------------------------------------------------------------------- /tests/django2/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == '__main__': 6 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError as exc: 10 | raise ImportError( 11 | "Couldn't import Django. Are you sure it's installed and " 12 | "available on your PYTHONPATH environment variable? Did you " 13 | "forget to activate a virtual environment?" 14 | ) from exc 15 | execute_from_command_line(sys.argv) 16 | -------------------------------------------------------------------------------- /tests/django2/mysite/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/praekeltfoundation/docker-django-bootstrap/9cac8d7dfb259e7ac74b6d560ca82f9db15e4491/tests/django2/mysite/__init__.py -------------------------------------------------------------------------------- /tests/django2/mysite/celery.py: -------------------------------------------------------------------------------- 1 | # Standard example Celery setup: 2 | # http://docs.celeryproject.org/en/latest/django/first-steps-with-django.html 3 | from __future__ import absolute_import, unicode_literals 4 | 5 | import os 6 | 7 | from celery import Celery 8 | 9 | # set the default Django settings module for the 'celery' program. 10 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') 11 | 12 | app = Celery('mysite') 13 | 14 | # Using a string here means the worker doesn't have to serialize 15 | # the configuration object to child processes. 16 | # - namespace='CELERY' means all celery-related configuration keys 17 | # should have a `CELERY_` prefix. 18 | app.config_from_object('django.conf:settings', namespace='CELERY') 19 | 20 | # Load task modules from all registered Django app configs. 21 | app.autodiscover_tasks() 22 | 23 | 24 | @app.task(bind=True) 25 | def debug_task(self): 26 | print('Request: {0!r}'.format(self.request)) 27 | -------------------------------------------------------------------------------- /tests/django2/mysite/docker_settings.py: -------------------------------------------------------------------------------- 1 | # An example settings file for deploying a Django app in a Docker container. 2 | # Uses environment variables to configure the majority of settings. This 3 | # pattern is sometimes attributed to the '12factor' app guidelines: 4 | # https://12factor.net/config 5 | 6 | from __future__ import absolute_import 7 | 8 | # We use django-environ here to make working with environment variables a bit 9 | # easier: https://github.com/joke2k/django-environ. To use this, you'll need to 10 | # add 'django-environ' to your install_requires. 11 | import environ 12 | 13 | # Import the existing settings file, we'll work from there... 14 | from .settings import ALLOWED_HOSTS, SECRET_KEY 15 | from .settings import * # noqa 16 | 17 | env = environ.Env() 18 | 19 | SECRET_KEY = env.str('SECRET_KEY', default=SECRET_KEY) 20 | DEBUG = env.bool('DEBUG', default=False) 21 | ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default=ALLOWED_HOSTS) 22 | 23 | DATABASES = { 24 | # django-environ builds in the functionality of dj-database-url 25 | # (https://github.com/kennethreitz/dj-database-url). This allows you to 26 | # fully configure the database connection using a single environment 27 | # variable that defines a 'database URL', for example: 28 | # `DATABASE_URL=postgres://username:password@db-host/db-name` 29 | 'default': env.db(default='sqlite:///db.sqlite3') 30 | } 31 | 32 | # Set up static file storage as described in the README 33 | STATIC_ROOT = '/app/static' 34 | STATIC_URL = '/static/' 35 | # Using ManifestStaticFilesStorage results in a larger Docker image but means 36 | # that Nginx can set long 'expires' headers for the files. 37 | # https://github.com/praekeltfoundation/docker-django-bootstrap/pull/11 38 | STATICFILES_STORAGE = ( 39 | 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage') 40 | 41 | MEDIA_ROOT = '/app/media' 42 | MEDIA_URL = '/media/' 43 | 44 | # Logs are dealt with a bit differently in Docker-land. We don't really want to 45 | # log to files as Docker containers should be ephemeral and so the files may 46 | # be lost as soon as the container stops. Instead, we want to log to 47 | # stdout/stderr and have those streams handled by the Docker daemon. 48 | # https://docs.djangoproject.com/en/1.10/topics/logging/#configuring-logging 49 | LOGGING = { 50 | 'version': 1, 51 | 'disable_existing_loggers': False, 52 | 'formatters': { 53 | 'simple': { 54 | 'format': '%(name)s %(levelname)s %(message)s', 55 | }, 56 | }, 57 | 'handlers': { 58 | 'console': { 59 | 'class': 'logging.StreamHandler', 60 | 'formatter': 'simple', 61 | }, 62 | }, 63 | 'loggers': { 64 | 'django': { 65 | 'handlers': ['console'], 66 | 'level': env.str('DJANGO_LOG_LEVEL', default='INFO'), 67 | }, 68 | 'celery': { 69 | 'handlers': ['console'], 70 | 'level': env.str('CELERY_LOG_LEVEL', default='INFO'), 71 | }, 72 | }, 73 | } 74 | 75 | # Configure Celery using environment variables. These require that there is a 76 | # `celery.py` file in the project that tells Celery to read config from your 77 | # Django settings: 78 | # http://docs.celeryproject.org/en/latest/django/first-steps-with-django.html 79 | CELERY_BROKER_URL = env.str('CELERY_BROKER_URL', default='amqp://') 80 | # django-health-check's Celery health check requires a result backend to be 81 | # configured. 82 | CELERY_RESULT_BACKEND = "rpc://" 83 | # *** This line is important! We want the worker concurrency to default to 1. 84 | # If we don't do this it will default to the number of CPUs on the particular 85 | # machine that we run the container on, which means unpredictable resource 86 | # usage on a cluster of mixed hosts. 87 | CELERY_WORKER_CONCURRENCY = env.int('CELERY_WORKER_CONCURRENCY', default=1) 88 | 89 | # Celery 3.1 compatibility 90 | BROKER_URL = CELERY_BROKER_URL 91 | CELERYD_CONCURRENCY = CELERY_WORKER_CONCURRENCY 92 | -------------------------------------------------------------------------------- /tests/django2/mysite/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for mysite project. 3 | 4 | Generated by 'django-admin startproject' using Django 2.1.1. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.1/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/2.1/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = 'bksrd%vl1)+(if0r)8$*q-gz+$g=32%^%!wc(et4+0ka)i1kvq' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = [] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = [ 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | 'compressor', 41 | 'health_check', 42 | # These more in-depth checks could be used in real-life but in our tests 43 | # they get in the way when we're testing edge cases. 44 | # 'health_check.db', 45 | # 'health_check.contrib.celery', 46 | 'django_prometheus', 47 | ] 48 | 49 | MIDDLEWARE = [ 50 | 'django_prometheus.middleware.PrometheusBeforeMiddleware', 51 | 'django.middleware.security.SecurityMiddleware', 52 | 'django.contrib.sessions.middleware.SessionMiddleware', 53 | 'django.middleware.common.CommonMiddleware', 54 | 'django.middleware.csrf.CsrfViewMiddleware', 55 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 56 | 'django.contrib.messages.middleware.MessageMiddleware', 57 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 58 | 'django_prometheus.middleware.PrometheusAfterMiddleware', 59 | ] 60 | 61 | ROOT_URLCONF = 'mysite.urls' 62 | 63 | TEMPLATES = [ 64 | { 65 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 66 | 'DIRS': ['mysite/templates'], 67 | 'APP_DIRS': True, 68 | 'OPTIONS': { 69 | 'context_processors': [ 70 | 'django.template.context_processors.debug', 71 | 'django.template.context_processors.request', 72 | 'django.contrib.auth.context_processors.auth', 73 | 'django.contrib.messages.context_processors.messages', 74 | ], 75 | }, 76 | }, 77 | ] 78 | 79 | WSGI_APPLICATION = 'mysite.wsgi.application' 80 | 81 | 82 | # Database 83 | # https://docs.djangoproject.com/en/2.1/ref/settings/#databases 84 | 85 | DATABASES = { 86 | 'default': { 87 | 'ENGINE': 'django.db.backends.sqlite3', 88 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 89 | } 90 | } 91 | 92 | 93 | # Password validation 94 | # https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators 95 | 96 | AUTH_PASSWORD_VALIDATORS = [ 97 | { 98 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 99 | }, 100 | { 101 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 102 | }, 103 | { 104 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 105 | }, 106 | { 107 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 108 | }, 109 | ] 110 | 111 | 112 | # Internationalization 113 | # https://docs.djangoproject.com/en/2.1/topics/i18n/ 114 | 115 | LANGUAGE_CODE = 'en-us' 116 | 117 | TIME_ZONE = 'UTC' 118 | 119 | USE_I18N = True 120 | 121 | USE_L10N = True 122 | 123 | USE_TZ = True 124 | 125 | 126 | # Static files (CSS, JavaScript, Images) 127 | # https://docs.djangoproject.com/en/2.1/howto/static-files/ 128 | 129 | STATIC_URL = '/static/' 130 | 131 | # Set up django-compressor 132 | STATICFILES_FINDERS = [ 133 | 'django.contrib.staticfiles.finders.FileSystemFinder', 134 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 135 | 'compressor.finders.CompressorFinder', 136 | ] 137 | COMPRESS_OFFLINE = True 138 | -------------------------------------------------------------------------------- /tests/django2/mysite/templates/template.html: -------------------------------------------------------------------------------- 1 | {# Dummy template with gibberish CSS and JS to test django-compressor #} 2 | {% load compress %} 3 | 4 | {% compress css %} 5 | 6 | {% endcompress %} 7 | 8 | {% compress js %} 9 | 10 | {% endcompress %} 11 | -------------------------------------------------------------------------------- /tests/django2/mysite/urls.py: -------------------------------------------------------------------------------- 1 | """mysite URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/2.1/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.contrib import admin 17 | from django.urls import include, path 18 | 19 | import django_prometheus 20 | 21 | urlpatterns = [ 22 | path('admin/', admin.site.urls), 23 | path('health/', include('health_check.urls')), 24 | path('metrics/', django_prometheus.exports.ExportToDjangoView, 25 | name='prometheus-django-metrics'), 26 | ] 27 | -------------------------------------------------------------------------------- /tests/django2/mysite/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for mysite project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /tests/django2/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='mysite', 5 | version='0.1', 6 | author='Praekelt.org', 7 | author_email='sre@praekelt.org', 8 | packages=['mysite'], 9 | install_requires=[ 10 | 'celery >=5.2.2, <6', 11 | 'Django >=2.2.2, <2.3', 12 | 'django_compressor >=2.1', 13 | 'django-environ', 14 | 'django-health-check', 15 | 'django-prometheus <2.3', 16 | 'psycopg2-binary >=2.7', 17 | # For compat with older celery in Python 3.7 18 | 'importlib_metadata < 5.0', 19 | ], 20 | ) 21 | -------------------------------------------------------------------------------- /tests/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore:unclosed:ResourceWarning 4 | 5 | log_format = %(name)-25s %(levelname)-8s %(message)s 6 | -------------------------------------------------------------------------------- /tests/requirements.txt: -------------------------------------------------------------------------------- 1 | # Pin to older requests version for seaworthy deps 2 | requests < 2.29 3 | 4 | seaworthy >= 0.4.2 5 | iso8601 6 | prometheus_client 7 | pytest >= 3.3, < 5.0 8 | testtools 9 | 10 | flake8 11 | flake8-import-order 12 | -------------------------------------------------------------------------------- /tests/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import json 3 | import logging 4 | import re 5 | import time 6 | from datetime import datetime, timedelta, timezone 7 | 8 | import iso8601 9 | 10 | from prometheus_client import parser as prom_parser 11 | 12 | import pytest 13 | 14 | from seaworthy.ps import build_process_tree 15 | from seaworthy.stream.matchers import OrderedMatcher, RegexMatcher 16 | from seaworthy.testtools import MatchesPsTree 17 | from seaworthy.utils import output_lines 18 | 19 | from testtools.assertions import assert_that 20 | from testtools.matchers import ( 21 | AfterPreprocessing as After, Contains, Equals, GreaterThan, HasLength, 22 | LessThan, MatchesAll, MatchesAny, MatchesDict, MatchesListwise, 23 | MatchesRegex, MatchesSetwise, Not, StartsWith) 24 | 25 | from definitions import ( # noqa: I100,I101 26 | # dependencies 27 | amqp_container, db_container, 28 | # our definitions 29 | beat_container, single_container, web_container, worker_container, 30 | # helper function 31 | make_combined_fixture) 32 | 33 | 34 | # Turn off spam from all the random loggers that set themselves up behind us. 35 | for logger in logging.Logger.manager.loggerDict.values(): 36 | if isinstance(logger, logging.Logger): 37 | logger.setLevel(logging.WARNING) 38 | # Turn on spam from the loggers we're interested in. 39 | logging.getLogger('docker_helper.helper').setLevel(logging.DEBUG) 40 | 41 | 42 | # Set up fixtures. The fixtures are identified by the names passed into the 43 | # factory functions, but we need references to them at module level so pytest 44 | # can find them. 45 | single_container_fixture = single_container.pytest_fixture('single_container') 46 | web_only_fixture = web_container.pytest_fixture('web_only_container') 47 | worker_only_fixture = worker_container.pytest_fixture('worker_only_container') 48 | beat_only_fixture = beat_container.pytest_fixture('beat_only_container') 49 | 50 | web_fixture = make_combined_fixture('web') 51 | worker_fixture = make_combined_fixture('worker') 52 | beat_fixture = make_combined_fixture('beat') 53 | 54 | raw_db_fixture, db_fixture = db_container.pytest_clean_fixtures( 55 | 'db_container', scope='module') 56 | raw_amqp_fixture, amqp_fixture = amqp_container.pytest_clean_fixtures( 57 | 'amqp_container', scope='module') 58 | 59 | 60 | def public_tables(db_container): 61 | return [r[1] for r in db_container.list_tables() if r[0] == 'public'] 62 | 63 | 64 | def filter_ldconfig_process(ps_rows): 65 | """ 66 | Sometimes an ldconfig process running under the django user shows up. 67 | Filter it out. 68 | :param ps_rows: A list of PsRow objects. 69 | """ 70 | return [row for row in ps_rows 71 | if not (row.ruser == 'django' and 'ldconfig' in row.args)] 72 | 73 | 74 | def http_requests_total_for_view(metrics_text, view): 75 | """ 76 | Get the samples available for a view given the response text from the 77 | metrics endpoint. 78 | """ 79 | # https://github.com/prometheus/client_python/#parser 80 | samples = [] 81 | for family in prom_parser.text_string_to_metric_families(metrics_text): 82 | if family.name == ( 83 | 'django_http_requests_total_by_view_transport_method'): 84 | for sample in family.samples: 85 | if sample.labels['view'] == view: 86 | samples.append(sample) 87 | break 88 | 89 | return samples 90 | 91 | 92 | class TestWeb(object): 93 | def test_expected_processes(self, web_only_container): 94 | """ 95 | When the container is running, there should be 5 running processes: 96 | tini, the Nginx master and worker, and the Gunicorn master and worker. 97 | """ 98 | ps_rows = filter_ldconfig_process(web_only_container.list_processes()) 99 | 100 | # Sometimes it takes a little while for the processes to settle so try 101 | # a few times with a delay inbetween. 102 | retries = 3 103 | delay = 0.5 104 | for _ in range(retries): 105 | if len(ps_rows) == 5: 106 | break 107 | time.sleep(delay) 108 | ps_rows = filter_ldconfig_process( 109 | web_only_container.list_processes()) 110 | 111 | ps_tree = build_process_tree(ps_rows) 112 | 113 | tini_args = 'tini -- django-entrypoint.sh mysite.wsgi:application' 114 | gunicorn_master_args = ( 115 | '/usr/local/bin/python /usr/local/bin/gunicorn ' 116 | 'mysite.wsgi:application --config /etc/gunicorn/config.py') 117 | gunicorn_worker_args = ( 118 | '/usr/local/bin/python /usr/local/bin/gunicorn ' 119 | 'mysite.wsgi:application --config /etc/gunicorn/config.py') 120 | nginx_master_args = 'nginx: master process nginx -g daemon off;' 121 | nginx_worker_args = 'nginx: worker process' 122 | 123 | assert_that( 124 | ps_tree, 125 | MatchesPsTree('root', tini_args, pid=1, children=[ 126 | MatchesPsTree('django', gunicorn_master_args, children=[ 127 | MatchesPsTree('django', gunicorn_worker_args), 128 | # FIXME: Nginx should not be parented by Gunicorn 129 | MatchesPsTree('root', nginx_master_args, children=[ 130 | MatchesPsTree('nginx', nginx_worker_args), 131 | ]), 132 | ]), 133 | ])) 134 | 135 | def test_expected_processes_single_container(self, single_container): 136 | """ 137 | When the container is running, there should be 7 running processes: 138 | tini, the Nginx master and worker, the Gunicorn master and worker, and 139 | the Celery worker ("solo", non-forking) and beat processes. 140 | """ 141 | ps_rows = single_container.list_processes() 142 | ps_tree = build_process_tree(ps_rows) 143 | 144 | tini_args = 'tini -- django-entrypoint.sh mysite.wsgi:application' 145 | gunicorn_master_args = ( 146 | '/usr/local/bin/python /usr/local/bin/gunicorn ' 147 | 'mysite.wsgi:application --config /etc/gunicorn/config.py') 148 | gunicorn_worker_args = ( 149 | '/usr/local/bin/python /usr/local/bin/gunicorn ' 150 | 'mysite.wsgi:application --config /etc/gunicorn/config.py') 151 | nginx_master_args = 'nginx: master process nginx -g daemon off;' 152 | nginx_worker_args = 'nginx: worker process' 153 | celery_worker_args = ( 154 | '/usr/local/bin/python /usr/local/bin/celery worker --pool=solo ' 155 | '--pidfile worker.pid --concurrency 1') 156 | celery_beat_args = ( 157 | '/usr/local/bin/python /usr/local/bin/celery beat --pidfile ' 158 | 'beat.pid') 159 | 160 | assert_that( 161 | ps_tree, 162 | MatchesPsTree('root', tini_args, pid=1, children=[ 163 | MatchesPsTree('django', gunicorn_master_args, children=[ 164 | MatchesPsTree('django', gunicorn_worker_args), 165 | # FIXME: Celery worker should not be parented by Gunicorn 166 | MatchesPsTree('django', celery_worker_args), 167 | # FIXME: Celery beat should not be parented by Gunicorn 168 | MatchesPsTree('django', celery_beat_args), 169 | # FIXME: Nginx should not be parented by Gunicorn 170 | MatchesPsTree('root', nginx_master_args, children=[ 171 | MatchesPsTree('nginx', nginx_worker_args), 172 | ]), 173 | ]), 174 | ])) 175 | 176 | def test_expected_files(self, web_only_container): 177 | """ 178 | When the container is running, there should be PID files for Nginx and 179 | Gunicorn, a working directory for Gunicorn, and Gunicorn's Unix socket 180 | file. 181 | """ 182 | stat = web_only_container.exec_stat( 183 | '/run/nginx.pid', 184 | '/run/gunicorn', 185 | '/run/gunicorn/gunicorn.sock', 186 | '/run/gunicorn/gunicorn.pid', 187 | '/run/gunicorn/prometheus', 188 | ) 189 | 190 | assert_that(stat, Equals([ 191 | '644 root:root', 192 | '755 django:django', 193 | '660 django:django', 194 | '644 django:django', 195 | '755 django:django', 196 | ])) 197 | 198 | self._assert_gunicorn_worker_tmp_file(web_only_container) 199 | 200 | self._assert_prometheus_dbs(web_only_container) 201 | 202 | def test_expected_files_single_container(self, single_container): 203 | """ 204 | When the container is running, there should be PID files for Nginx, 205 | Gunicorn, and Celery, working directories for Gunicorn and Celery, and 206 | Gunicorn's Unix socket file. 207 | """ 208 | stat = single_container.exec_stat( 209 | '/run/nginx.pid', 210 | '/run/gunicorn', 211 | '/run/gunicorn/gunicorn.sock', 212 | '/run/gunicorn/gunicorn.pid', 213 | '/run/gunicorn/prometheus', 214 | '/run/celery', 215 | '/run/celery/worker.pid', 216 | '/run/celery/beat.pid', 217 | ) 218 | 219 | assert_that(stat, Equals([ 220 | '644 root:root', 221 | '755 django:django', 222 | '660 django:django', 223 | '644 django:django', 224 | '755 django:django', 225 | '755 django:django', 226 | '644 django:django', 227 | '644 django:django', 228 | ])) 229 | 230 | self._assert_gunicorn_worker_tmp_file(single_container) 231 | 232 | self._assert_prometheus_dbs(single_container) 233 | 234 | def _assert_gunicorn_worker_tmp_file(self, container): 235 | # Find the worker temporary file...this is quite involved because it's 236 | # a temp file without a predictable name and Gunicorn unlinks it so we 237 | # can't just stat it 238 | ps_rows = container.list_processes() 239 | gunicorns = [r for r in ps_rows if '/usr/local/bin/gunicorn' in r.args] 240 | worker = gunicorns[-1] # Already sorted by PID, pick the highest PID 241 | proc_fd_path = '/proc/{}/fd'.format(worker.pid) 242 | 243 | # Due to container restrictions, the root user doesn't have access to 244 | # /proc/pid/fd/*--we must run as the django user. 245 | fd_targets = container.exec_run([ 246 | 'bash', '-c', 247 | 'for fd in {}/*; do readlink -f "$fd"; done'.format(proc_fd_path) 248 | ], user='django') 249 | 250 | # Filter out pipes and sockets 251 | pipe_path = '{}/pipe'.format(proc_fd_path) 252 | ceci_nest_pas_une_pipe = ( 253 | [t for t in fd_targets if not t.startswith(pipe_path)]) 254 | sock_path = '{}/socket'.format(proc_fd_path) 255 | neither_pipes_nor_socks = ( 256 | [t for t in ceci_nest_pas_une_pipe if not t.startswith(sock_path)]) 257 | 258 | # Sometimes things like /dev/urandom are in there too 259 | files = ( 260 | [t for t in neither_pipes_nor_socks if not t.startswith('/dev/')]) 261 | not_prom_files = ( 262 | [t for t in files if not t.startswith('/run/gunicorn/prometheus/')] 263 | ) 264 | 265 | # Finally, assert the worker temp file is in the place we expect 266 | assert_that(not_prom_files, MatchesListwise([ 267 | StartsWith('/run/gunicorn/wgunicorn-') 268 | ])) 269 | 270 | def _assert_prometheus_dbs(self, container): 271 | # The worker Gunicorn process has some process-specific db-files. Get 272 | # the PID and assert there are files for it. 273 | ps_rows = container.list_processes() 274 | gunicorns = [r for r in ps_rows if '/usr/local/bin/gunicorn' in r.args] 275 | pid = gunicorns[-1].pid # Already sorted by PID, pick the highest PID 276 | 277 | prom_files = container.exec_find( 278 | ['/run/gunicorn/prometheus/', '-type', 'f']) 279 | assert_that(prom_files, MatchesSetwise( 280 | Equals('/run/gunicorn/prometheus/counter_{}.db'.format(pid)), 281 | Equals('/run/gunicorn/prometheus/gauge_all_{}.db'.format(pid)), 282 | Equals('/run/gunicorn/prometheus/histogram_{}.db'.format(pid)), 283 | )) 284 | 285 | @pytest.mark.clean_db_container 286 | def test_database_tables_created(self, db_container, web_container): 287 | """ 288 | When the web container is running, a migration should have completed 289 | and there should be some tables in the database. 290 | """ 291 | assert_that(len(public_tables(db_container)), GreaterThan(0)) 292 | 293 | @pytest.mark.clean_db_container 294 | def test_database_tables_not_created(self, docker_helper, db_container): 295 | """ 296 | When the web container is running with the `SKIP_MIGRATIONS` 297 | environment variable set, there should be no tables in the database. 298 | """ 299 | self._test_database_tables_not_created( 300 | docker_helper, db_container, web_container) 301 | 302 | @pytest.mark.clean_db_container 303 | def test_database_tables_not_created_single( 304 | self, docker_helper, db_container, amqp_container): 305 | """ 306 | When the single container is running with the `SKIP_MIGRATIONS` 307 | environment variable set, there should be no tables in the database. 308 | """ 309 | self._test_database_tables_not_created( 310 | docker_helper, db_container, single_container) 311 | 312 | def _test_database_tables_not_created( 313 | self, docker_helper, db_container, django_container): 314 | assert_that(public_tables(db_container), Equals([])) 315 | 316 | django_container.set_helper(docker_helper) 317 | with django_container.setup(environment={'SKIP_MIGRATIONS': '1'}): 318 | if django_container.django_maj_version() >= 2: 319 | assert_that(public_tables(db_container), Equals([])) 320 | else: 321 | # On Django 1, if our app has any models (and the 322 | # django-prometheus package adds one), then an empty table 323 | # called "django_migrations" is created, even if migrations 324 | # aren't run. 325 | assert_that( 326 | public_tables(db_container), Equals(['django_migrations'])) 327 | [count] = output_lines(db_container.exec_psql( 328 | 'SELECT COUNT(*) FROM django_migrations;')) 329 | assert_that(int(count), Equals(0)) 330 | 331 | def test_admin_site_live(self, web_container): 332 | """ 333 | When we get the /admin/ path, we should receive some HTML for the 334 | Django admin interface. 335 | """ 336 | web_client = web_container.http_client() 337 | response = web_client.get('/admin/') 338 | 339 | assert_that(response.headers['Content-Type'], 340 | Equals('text/html; charset=utf-8')) 341 | assert_that(response.text, 342 | Contains('Log in | Django site admin')) 343 | 344 | def test_prometheus_metrics_live(self, web_container): 345 | """ 346 | When we get the /metrics path, we should receive Prometheus metrics. 347 | """ 348 | web_client = web_container.http_client() 349 | response = web_client.get('/metrics') 350 | 351 | assert_that(response.headers['Content-Type'], 352 | Equals('text/plain; version=0.0.4; charset=utf-8')) 353 | 354 | [sample] = http_requests_total_for_view( 355 | response.text, 'prometheus-django-metrics') 356 | assert_that(sample.labels['transport'], Equals('http')) 357 | assert_that(sample.labels['method'], Equals('GET')) 358 | assert_that(sample.value, Equals(1.0)) 359 | 360 | def test_prometheus_metrics_worker_restart(self, web_container): 361 | """ 362 | When a worker process is restarted, Prometheus counters should be 363 | preserved, such that a call to the metrics endpoint before a worker 364 | restart can be observed after the worker restart. 365 | """ 366 | web_client = web_container.http_client() 367 | response = web_client.get('/metrics', headers={'Connection': 'close'}) 368 | 369 | [sample] = http_requests_total_for_view( 370 | response.text, 'prometheus-django-metrics') 371 | assert_that(sample.value, Equals(1.0)) 372 | 373 | # Signal Gunicorn so that it restarts the worker(s) 374 | # http://docs.gunicorn.org/en/latest/signals.html 375 | web_container.inner().kill("SIGHUP") 376 | 377 | matcher = OrderedMatcher(*(RegexMatcher(r) for r in ( 378 | r'Booting worker', # Original worker start on startup 379 | r'Booting worker', # Worker start after SIGHUP restart 380 | ))) 381 | web_container.wait_for_logs_matching( 382 | matcher, web_container.wait_timeout) 383 | # Wait a little longer to make sure everything's up 384 | time.sleep(0.2) 385 | 386 | # Now try make another request and ensure the counter was incremented 387 | response = web_client.get('/metrics') 388 | [sample] = http_requests_total_for_view( 389 | response.text, 'prometheus-django-metrics') 390 | assert_that(sample.value, Equals(2.0)) 391 | 392 | def test_prometheus_metrics_web_concurrency( 393 | self, docker_helper, db_container): 394 | """ 395 | When the web container is running with multiple worker processes, 396 | multiple requests to the admin or metrics endpoints should result in 397 | the expected metrics. 398 | """ 399 | self._test_prometheus_metrics_web_concurrency( 400 | docker_helper, web_container) 401 | 402 | def test_prometheus_metrics_web_concurrency_single( 403 | self, docker_helper, db_container, amqp_container): 404 | """ 405 | When the single container is running with multiple worker processes, 406 | multiple requests to the admin or metrics endpoints should result in 407 | the expected metrics. 408 | """ 409 | self._test_prometheus_metrics_web_concurrency( 410 | docker_helper, single_container) 411 | 412 | def _test_prometheus_metrics_web_concurrency( 413 | self, docker_helper, django_container): 414 | django_container.set_helper(docker_helper) 415 | 416 | # Mo' workers mo' problems 417 | # FIXME: Make this a bit less hacky somehow... 418 | django_container.wait_matchers *= 4 419 | 420 | with django_container.setup(environment={'WEB_CONCURRENCY': '4'}): 421 | client = django_container.http_client() 422 | 423 | # Make a bunch of requests 424 | for _ in range(20): 425 | response = client.get('/admin') 426 | assert_that(response.status_code, Equals(200)) 427 | 428 | # Check the metrics a bunch of times for a few things... 429 | # We don't know which worker serves which request, so we just 430 | # make a few requests hoping we'll hit multiple/all of them. 431 | for i in range(20): 432 | response = client.get('/metrics') 433 | 434 | # Requests to the admin page are counted 435 | [admin_sample] = http_requests_total_for_view( 436 | response.text, view='admin:index') 437 | assert_that(admin_sample.value, Equals(20.0)) 438 | 439 | # Requests to the metrics endpoint increment 440 | [metrics_sample] = http_requests_total_for_view( 441 | response.text, 'prometheus-django-metrics') 442 | assert_that(metrics_sample.value, Equals(i + 1)) 443 | 444 | # django_migrations_ samples are per-process, with a pid label. 445 | # Check that there are 4 different pids. 446 | fs = prom_parser.text_string_to_metric_families(response.text) 447 | [family] = [f for f in fs 448 | if f.name == 'django_migrations_applied_total'] 449 | pids = set() 450 | for sample in family.samples: 451 | pids.add(sample.labels['pid']) 452 | assert_that(pids, HasLength(4)) 453 | 454 | def test_nginx_access_logs(self, web_container): 455 | """ 456 | When a request has been made to the container, Nginx logs access logs 457 | to stdout 458 | """ 459 | # Wait a little bit so that previous tests' requests have been written 460 | # to the log. 461 | time.sleep(0.2) 462 | before_lines = output_lines(web_container.get_logs(stderr=False)) 463 | 464 | # Make a request to see the logs for it 465 | web_client = web_container.http_client() 466 | web_client.get('/') 467 | 468 | # Wait a little bit so that our request has been written to the log. 469 | time.sleep(0.2) 470 | after_lines = output_lines(web_container.get_logs(stderr=False)) 471 | 472 | new_lines = after_lines[len(before_lines):] 473 | assert_that(len(new_lines), GreaterThan(0)) 474 | 475 | # Find the Nginx log line 476 | nginx_lines = [l for l in new_lines if re.match(r'^\{ "time": .+', l)] 477 | assert_that(nginx_lines, HasLength(1)) 478 | 479 | now = datetime.now(timezone.utc) 480 | assert_that(json.loads(nginx_lines[0]), MatchesDict({ 481 | # Assert time is valid and recent 482 | 'time': After(iso8601.parse_date, MatchesAll( 483 | MatchesAny(LessThan(now), Equals(now)), 484 | MatchesAny(GreaterThan(now - timedelta(seconds=5))) 485 | )), 486 | 487 | 'request': Equals('GET / HTTP/1.1'), 488 | 'status': Equals(404), 489 | 'body_bytes_sent': GreaterThan(0), 490 | 'request_time': LessThan(1.0), 491 | 'http_referer': Equals(''), 492 | 493 | # Assert remote_addr is an IPv4 (roughly) 494 | 'remote_addr': MatchesRegex( 495 | r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$'), 496 | 'http_host': MatchesRegex(r'^127.0.0.1:\d{4,5}$'), 497 | 'http_user_agent': MatchesRegex(r'^python-requests/'), 498 | 499 | # Not very interesting empty fields 500 | 'remote_user': Equals(''), 501 | 'http_via': Equals(''), 502 | 'http_x_forwarded_proto': Equals(''), 503 | 'http_x_forwarded_for': Equals(''), 504 | })) 505 | 506 | def test_gunicorn_access_logs(self, docker_helper, db_container): 507 | """ 508 | When the web container is running with the `GUNICORN_ACCESS_LOGS` 509 | environment variable set, Gunicorn access logs should be output. 510 | """ 511 | self._test_gunicorn_access_logs(docker_helper, web_container) 512 | 513 | def test_gunicorn_access_logs_single( 514 | self, docker_helper, db_container, amqp_container): 515 | """ 516 | When the single container is running with the `GUNICORN_ACCESS_LOGS` 517 | environment variable set, Gunicorn access logs should be output. 518 | """ 519 | self._test_gunicorn_access_logs(docker_helper, single_container) 520 | 521 | def _test_gunicorn_access_logs(self, docker_helper, web_container): 522 | web_container.set_helper(docker_helper) 523 | with web_container.setup(environment={'GUNICORN_ACCESS_LOGS': '1'}): 524 | # Wait a little bit so that previous tests' requests have been 525 | # written to the log. 526 | time.sleep(0.2) 527 | before_lines = output_lines(web_container.get_logs(stderr=False)) 528 | 529 | # Make a request to see the logs for it 530 | web_client = web_container.http_client() 531 | web_client.get('/') 532 | 533 | # Wait a little bit so that our request has been written to the log. 534 | time.sleep(0.2) 535 | after_lines = output_lines(web_container.get_logs(stderr=False)) 536 | 537 | new_lines = after_lines[len(before_lines):] 538 | assert_that(len(new_lines), GreaterThan(0)) 539 | 540 | # Find the Gunicorn log line (the not-Nginx-JSON line) 541 | gunicorn_lines = [ 542 | l for l in new_lines if not re.match(r'^\{ .+', l)] 543 | assert_that(gunicorn_lines, HasLength(1)) 544 | assert_that(gunicorn_lines[0], Contains('"GET / HTTP/1.0"')) 545 | 546 | def test_static_file(self, web_container): 547 | """ 548 | When a static file is requested, Nginx should serve the file with the 549 | correct mime type. 550 | """ 551 | web_client = web_container.http_client() 552 | response = web_client.get('/static/admin/css/base.css') 553 | 554 | assert_that(response.headers['Content-Type'], Equals('text/css')) 555 | assert_that(response.text, Contains('DJANGO Admin styles')) 556 | 557 | def test_manifest_static_storage_file(self, web_container): 558 | """ 559 | When a static file that was processed by Django's 560 | ManifestStaticFilesStorage system is requested, that file should be 561 | served with a far-future 'Cache-Control' header. 562 | """ 563 | hashed_svg = web_container.exec_find( 564 | ['static/admin/img', '-regextype', 'posix-egrep', '-regex', 565 | r'.*\.[a-f0-9]{12}\.svg$']) 566 | test_file = hashed_svg[0] 567 | 568 | web_client = web_container.http_client() 569 | response = web_client.get('/' + test_file) 570 | 571 | assert_that(response.headers['Content-Type'], Equals('image/svg+xml')) 572 | assert_that(response.headers['Cache-Control'], 573 | Equals('max-age=315360000, public, immutable')) 574 | 575 | def test_django_compressor_js_file(self, web_container): 576 | """ 577 | When a static JavaScript file that was processed by django_compressor 578 | is requested, that file should be served with a far-future 579 | 'Cache-Control' header. 580 | """ 581 | compressed_js = web_container.exec_find( 582 | ['static/CACHE/js', '-name', '*.js']) 583 | test_file = compressed_js[0] 584 | 585 | web_client = web_container.http_client() 586 | response = web_client.get('/' + test_file) 587 | 588 | assert_that(response.headers['Content-Type'], 589 | Equals('application/javascript')) 590 | assert_that(response.headers['Cache-Control'], 591 | Equals('max-age=315360000, public, immutable')) 592 | 593 | def test_django_compressor_css_file(self, web_container): 594 | """ 595 | When a static CSS file that was processed by django_compressor is 596 | requested, that file should be served with a far-future 'Cache-Control' 597 | header. 598 | """ 599 | compressed_js = web_container.exec_find( 600 | ['static/CACHE/css', '-name', '*.css']) 601 | test_file = compressed_js[0] 602 | 603 | web_client = web_container.http_client() 604 | response = web_client.get('/' + test_file) 605 | 606 | assert_that(response.headers['Content-Type'], Equals('text/css')) 607 | assert_that(response.headers['Cache-Control'], 608 | Equals('max-age=315360000, public, immutable')) 609 | 610 | def test_gzip_css_compressed(self, web_container): 611 | """ 612 | When a CSS file larger than 1024 bytes is requested and the 613 | 'Accept-Encoding' header lists gzip as an accepted encoding, the file 614 | should be served gzipped. 615 | """ 616 | css_to_gzip = web_container.exec_find( 617 | ['static', '-name', '*.css', '-size', '+1024c']) 618 | test_file = css_to_gzip[0] 619 | 620 | web_client = web_container.http_client() 621 | response = web_client.get( 622 | '/' + test_file, headers={'Accept-Encoding': 'gzip'}) 623 | 624 | assert_that(response.headers['Content-Type'], Equals('text/css')) 625 | assert_that(response.headers['Content-Encoding'], Equals('gzip')) 626 | assert_that(response.headers['Vary'], Equals('Accept-Encoding')) 627 | 628 | def test_gzip_woff_not_compressed(self, web_container): 629 | """ 630 | When a .woff file larger than 1024 bytes is requested and the 631 | 'Accept-Encoding' header lists gzip as an accepted encoding, the file 632 | should not be served gzipped as it is already a compressed format. 633 | """ 634 | woff_to_not_gzip = web_container.exec_find( 635 | ['static', '-name', '*.woff', '-size', '+1024c']) 636 | test_file = woff_to_not_gzip[0] 637 | 638 | web_client = web_container.http_client() 639 | response = web_client.get( 640 | '/' + test_file, headers={'Accept-Encoding': 'gzip'}) 641 | 642 | assert_that(response.headers['Content-Type'], 643 | Equals('font/woff')) 644 | assert_that(response.headers, MatchesAll( 645 | Not(Contains('Content-Encoding')), 646 | Not(Contains('Vary')), 647 | )) 648 | 649 | def test_gzip_accept_encoding_respected(self, web_container): 650 | """ 651 | When a CSS file larger than 1024 bytes is requested and the 652 | 'Accept-Encoding' header does not list gzip as an accepted encoding, 653 | the file should not be served gzipped, but the 'Vary' header should be 654 | set to 'Accept-Encoding'. 655 | """ 656 | css_to_gzip = web_container.exec_find( 657 | ['static', '-name', '*.css', '-size', '+1024c']) 658 | test_file = css_to_gzip[0] 659 | 660 | web_client = web_container.http_client() 661 | response = web_client.get( 662 | '/' + test_file, headers={'Accept-Encoding': ''}) 663 | 664 | assert_that(response.headers['Content-Type'], Equals('text/css')) 665 | assert_that(response.headers, Not(Contains('Content-Encoding'))) 666 | # The Vary header should be set if there is a *possibility* that this 667 | # file will be served with a different encoding. 668 | assert_that(response.headers['Vary'], Equals('Accept-Encoding')) 669 | 670 | def test_gzip_via_compressed(self, web_container): 671 | """ 672 | When a CSS file larger than 1024 bytes is requested and the 673 | 'Accept-Encoding' header lists gzip as an accepted encoding and the 674 | 'Via' header is set, the file should be served gzipped. 675 | """ 676 | css_to_gzip = web_container.exec_find( 677 | ['static', '-name', '*.css', '-size', '+1024c']) 678 | test_file = css_to_gzip[0] 679 | 680 | web_client = web_container.http_client() 681 | response = web_client.get( 682 | '/' + test_file, 683 | headers={'Accept-Encoding': 'gzip', 'Via': 'Internet.org'}) 684 | 685 | assert_that(response.headers['Content-Type'], Equals('text/css')) 686 | assert_that(response.headers['Content-Encoding'], Equals('gzip')) 687 | assert_that(response.headers['Vary'], Equals('Accept-Encoding')) 688 | 689 | def test_gzip_small_file_not_compressed(self, web_container): 690 | """ 691 | When a CSS file smaller than 1024 bytes is requested and the 692 | 'Accept-Encoding' header lists gzip as an accepted encoding, the file 693 | should not be served gzipped. 694 | """ 695 | css_to_gzip = web_container.exec_find( 696 | ['static', '-name', '*.css', '-size', '-1024c']) 697 | test_file = css_to_gzip[0] 698 | 699 | web_client = web_container.http_client() 700 | response = web_client.get( 701 | '/' + test_file, headers={'Accept-Encoding': 'gzip'}) 702 | 703 | assert_that(response.headers['Content-Type'], Equals('text/css')) 704 | assert_that(response.headers, MatchesAll( 705 | Not(Contains('Content-Encoding')), 706 | Not(Contains('Vary')), 707 | )) 708 | 709 | def test_media_volume_ownership(self, web_container): 710 | """ 711 | The /app/media directory should have the correct ownership set at 712 | runtime if it is a mounted volume. 713 | """ 714 | [app_media_ownership] = web_container.exec_run( 715 | ['stat', '-c', '%U:%G', '/app/media']) 716 | 717 | assert_that(app_media_ownership, Equals('django:django')) 718 | 719 | 720 | class TestCeleryWorker(object): 721 | def test_expected_processes(self, worker_only_container): 722 | """ 723 | When the container is running, there should be 3 running processes: 724 | tini, and the Celery worker master and worker. 725 | """ 726 | ps_rows = worker_only_container.list_processes() 727 | ps_tree = build_process_tree(ps_rows) 728 | 729 | tini_args = 'tini -- django-entrypoint.sh celery worker' 730 | celery_master_args = ( 731 | '/usr/local/bin/python /usr/local/bin/celery worker ' 732 | '--concurrency 1') 733 | celery_worker_args = ( 734 | '/usr/local/bin/python /usr/local/bin/celery worker ' 735 | '--concurrency 1') 736 | 737 | assert_that( 738 | ps_tree, 739 | MatchesPsTree('root', tini_args, pid=1, children=[ 740 | MatchesPsTree('django', celery_master_args, children=[ 741 | MatchesPsTree('django', celery_worker_args), 742 | ]), 743 | ])) 744 | 745 | @pytest.mark.clean_amqp 746 | def test_amqp_queues_created(self, amqp_container, worker_container): 747 | """ 748 | When the worker container is running, the three default Celery queues 749 | should have been created in RabbitMQ. 750 | """ 751 | queue_data = amqp_container.list_queues() 752 | 753 | assert_that(queue_data, MatchesSetwise(*map(MatchesListwise, ( 754 | [Equals('celery'), Equals('0')], 755 | [MatchesRegex(r'^celeryev\..+'), Equals('0')], 756 | [MatchesRegex(r'^celery@.+\.celery\.pidbox$'), Equals('0')], 757 | )))) 758 | 759 | 760 | class TestCeleryBeat(object): 761 | def test_expected_processes(self, beat_only_container): 762 | """ 763 | When the container is running, there should be 2 running processes: 764 | tini, and the Celery beat process. 765 | """ 766 | ps_rows = beat_only_container.list_processes() 767 | ps_tree = build_process_tree(ps_rows) 768 | 769 | tini_args = 'tini -- django-entrypoint.sh celery beat' 770 | celery_beat_args = '/usr/local/bin/python /usr/local/bin/celery beat' 771 | 772 | assert_that( 773 | ps_tree, 774 | MatchesPsTree('root', tini_args, pid=1, children=[ 775 | MatchesPsTree('django', celery_beat_args), 776 | ])) 777 | --------------------------------------------------------------------------------