├── src ├── tweet │ ├── __init__.py │ ├── migrations │ │ ├── __init__.py │ │ └── 0001_initial.py │ ├── admin.py │ ├── apps.py │ ├── models.py │ ├── urls.py │ ├── fixtures │ │ └── tweet.json │ ├── views.py │ └── tests.py ├── healthcheck │ ├── __init__.py │ ├── migrations │ │ └── __init__.py │ ├── models.py │ ├── admin.py │ ├── apps.py │ ├── urls.py │ ├── tests.py │ └── views.py ├── templatesite │ ├── __init__.py │ ├── test │ │ ├── __init__.py │ │ └── test_template.py │ ├── wsgi.py │ ├── urls.py │ └── settings.py ├── pytest.ini ├── start_dev_server.sh └── manage.py ├── system-test ├── requirements.txt ├── pytest.ini ├── tweet │ └── test_tweet.py └── healthcheck │ └── test_healthcheck.py ├── .dockerignore ├── docker ├── db │ ├── stop_postgres.sh │ ├── start_postgres.sh │ ├── prepare_django_db.sh │ ├── postgres-setup.sh │ └── Dockerfile ├── deps │ ├── copy_deps.sh │ ├── Dockerfile │ ├── build_deps.sh │ └── search_wheels.py ├── log │ ├── Dockerfile │ └── syslog-ng.conf ├── server │ ├── uwsgi.ini.template │ ├── start_server.sh │ ├── stack-fix.c │ └── nginx.conf.template ├── system-test │ └── Dockerfile └── metrics │ ├── prometheus.yaml │ ├── Dockerfile │ ├── django.rules │ ├── django.rules.yml │ └── consoles │ └── django.html ├── .gitmodules ├── requirements.txt ├── environment.env ├── LICENSE ├── vendor └── README.md ├── .gitignore ├── deps └── README.md ├── Dockerfile ├── docker-compose.yaml └── README.md /src/tweet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/healthcheck/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/templatesite/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/templatesite/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/tweet/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/healthcheck/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /system-test/requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.18.1 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # python 2 | **/.python-version 3 | **/__pycache__ 4 | **/*.pyc 5 | -------------------------------------------------------------------------------- /src/tweet/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /src/healthcheck/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /docker/db/stop_postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Stop grafecully postgres 3 | pkill postgres 4 | sleep 1 5 | -------------------------------------------------------------------------------- /src/healthcheck/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /src/tweet/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class TweetConfig(AppConfig): 5 | name = 'tweet' 6 | -------------------------------------------------------------------------------- /system-test/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | # Configuration for system tests 3 | python_files = tests.py test_*.py *_tests.py 4 | -------------------------------------------------------------------------------- /src/templatesite/test/test_template.py: -------------------------------------------------------------------------------- 1 | # Tests in pytest format, if you prefer 2 | 3 | 4 | def test_pass(): 5 | pass 6 | -------------------------------------------------------------------------------- /src/healthcheck/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class HealthtestsConfig(AppConfig): 5 | name = 'healthcheck' 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/django-prometheus"] 2 | path = deps/django-prometheus 3 | url = https://github.com/korfuri/django-prometheus.git 4 | -------------------------------------------------------------------------------- /src/healthcheck/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | from . import views 4 | 5 | urlpatterns = [ 6 | url(r'^$', views.healthcheck, name='healthcheck'), 7 | ] 8 | -------------------------------------------------------------------------------- /src/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | # Configuration for unit tests 3 | DJANGO_SETTINGS_MODULE = templatesite.settings 4 | python_files = tests.py test_*.py *_tests.py 5 | addopts = --reuse-db 6 | -------------------------------------------------------------------------------- /src/start_dev_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | python3 manage.py migrate -v 3 3 | # Preload hack to avoid sigfault 4 | export LD_PRELOAD=/opt/server/stack-fix.so 5 | python3 manage.py runserver 0.0.0.0:80 6 | -------------------------------------------------------------------------------- /docker/deps/copy_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | echo 'Deleting wheels and copy new ones' 3 | rm /opt/ext_vendor/*.whl 4 | echo "Dependencies are created at build. Run build --no-cache to recreate" 5 | cp /opt/vendor/* /opt/ext_vendor 6 | 7 | -------------------------------------------------------------------------------- /src/tweet/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | # Create your models here. 5 | class Tweet(models.Model): 6 | text = models.CharField(max_length=140) 7 | timestamp = models.DateTimeField(auto_now_add=True) 8 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | uwsgi==2.0.17 2 | Django==1.11.11 3 | pytest-django==3.1.2 4 | psycopg2==2.7.4 5 | djangorestframework==3.7.7 6 | django-log-request-id==1.3.2 7 | 8 | 9 | # dependencies 10 | /opt/deps/django-prometheus 11 | -------------------------------------------------------------------------------- /src/tweet/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | from . import views 4 | 5 | urlpatterns = [ 6 | url(r'^$', views.TweetListView.as_view(), name='list_tweets'), 7 | url(r'^(?P\d+)$', views.TweetView.as_view(), name='get_tweet'), 8 | ] 9 | -------------------------------------------------------------------------------- /docker/log/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | RUN apk add --update syslog-ng 4 | 5 | ADD ./docker/log/syslog-ng.conf /etc/syslog-ng/syslog-ng.conf 6 | 7 | EXPOSE 514/tcp 514/udp 8 | 9 | ENTRYPOINT ["/usr/sbin/syslog-ng", "-F", "-f", "/etc/syslog-ng/syslog-ng.conf"] 10 | -------------------------------------------------------------------------------- /docker/db/start_postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Start gracefully postgres 3 | su-exec postgres postgres & 4 | 5 | set -e 6 | echo "Waiting till up" 7 | 8 | host="$1" 9 | shift 10 | cmd="$@" 11 | 12 | until PGPASSWORD=$PGPASSWORD su-exec postgres psql -c '\l'; do 13 | >&2 echo "Postgres is unavailable - sleeping" 14 | sleep 1 15 | done 16 | -------------------------------------------------------------------------------- /docker/server/uwsgi.ini.template: -------------------------------------------------------------------------------- 1 | [uwsgi] 2 | chdir=/opt/code 3 | wsgi-file = templatesite/wsgi.py 4 | master=True 5 | pidfile=/tmp/uwsgi.pid 6 | uid=nginx 7 | socket=/tmp/uwsgi.sock 8 | vacuum=True 9 | uid=1000 10 | processes=1 11 | max-requests=5000 12 | logger=rsyslog:log:514,uwsgi 13 | # Used to send commands to uWSGI 14 | master-fifo=/tmp/uwsgi-fifo 15 | -------------------------------------------------------------------------------- /docker/system-test/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | RUN apk add --update python3 py3-pip pytest 4 | RUN mkdir -p /opt/system-test 5 | 6 | ADD ./system-test/requirements.txt /opt/system-test 7 | 8 | WORKDIR /opt/system-test 9 | 10 | # So far, no compilation requirements 11 | RUN pip3 install -r requirements.txt 12 | 13 | ADD ./system-test/ /opt/system-test 14 | -------------------------------------------------------------------------------- /src/tweet/fixtures/tweet.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model": "tweet.tweet", 4 | "pk": 1, 5 | "fields": { 6 | "text": "This is a test", 7 | "timestamp": "2017-07-22T13:36:02.694Z" 8 | } 9 | }, 10 | { 11 | "model": "tweet.tweet", 12 | "pk": 2, 13 | "fields": { 14 | "text": "This is another test", 15 | "timestamp": "2017-07-22T13:36:09.752Z" 16 | } 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /docker/metrics/prometheus.yaml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 2s 3 | evaluation_interval: 10s 4 | 5 | external_labels: 6 | monitor: django-monitor 7 | 8 | rule_files: 9 | - '/etc/prometheus/django.rules.yml' 10 | 11 | scrape_configs: 12 | # The job name is added as a label `job=` to any timeseries scraped from this config. 13 | - job_name: 'templatesite' 14 | static_configs: 15 | - targets: ['server:80'] 16 | -------------------------------------------------------------------------------- /src/templatesite/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for templatesite project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "templatesite.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /docker/log/syslog-ng.conf: -------------------------------------------------------------------------------- 1 | @version: 3.7 2 | 3 | options { 4 | use_dns(no); 5 | keep_hostname(yes); 6 | create_dirs(yes); 7 | ts_format(iso); 8 | }; 9 | 10 | source s_net { 11 | tcp(ip(0.0.0.0), port(514)); 12 | udp(ip(0.0.0.0), port(514)); 13 | unix-stream("/var/run/syslog-ng/syslog-ng.sock"); 14 | }; 15 | 16 | destination logfiles { 17 | file("/var/log/templatesite.log"); 18 | }; 19 | 20 | log { 21 | source(s_net); 22 | destination(logfiles); 23 | }; 24 | -------------------------------------------------------------------------------- /docker/db/prepare_django_db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | /opt/code/db/start_postgres.sh 3 | echo 'Migrating DB' 4 | python3 manage.py migrate -v 3 5 | 6 | echo 'Migrating to test DB' 7 | # Copy the database, so we don't run migrations twice 8 | su-exec postgres psql -c "CREATE DATABASE test_$POSTGRES_DB WITH TEMPLATE $POSTGRES_DB" 9 | 10 | echo 'Loading fixtures' 11 | # Note the fixtures are not loaded into the test DB 12 | python3 manage.py loaddata ./*/fixtures/* 13 | 14 | /opt/code/db/stop_postgres.sh 15 | -------------------------------------------------------------------------------- /system-test/tweet/test_tweet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import requests 3 | 4 | HOSTPORT = os.environ.get('SYSTEM_TEST_HOSTPORT') 5 | TWEET_URL = HOSTPORT + 'tweet/' 6 | 7 | 8 | def test_tweets(): 9 | result = requests.get(TWEET_URL) 10 | assert result.status_code == 200 11 | tweets = result.json() 12 | assert len(tweets) == 2 13 | for tweet in tweets: 14 | # Get all the linked urls 15 | url = tweet['href'] 16 | result = requests.get(url) 17 | assert result.status_code == 200 18 | -------------------------------------------------------------------------------- /docker/deps/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | RUN mkdir -p /opt/vendor 3 | WORKDIR /opt/vendor 4 | RUN apk update 5 | # Basic python usage 6 | RUN apk add python3 7 | RUN apk add py3-pip 8 | 9 | # Required for compiling 10 | RUN apk add python3-dev build-base linux-headers gcc postgresql-dev 11 | RUN pip3 install cython wheel 12 | 13 | ADD ./deps /opt/deps 14 | RUN mkdir -p /opt/vendor 15 | ADD requirements.txt /opt/deps 16 | ADD ./docker/deps/build_deps.sh /opt/ 17 | ADD ./docker/deps/copy_deps.sh /opt/ 18 | ADD ./docker/deps/search_wheels.py /opt/ 19 | 20 | WORKDIR /opt/ 21 | RUN ./build_deps.sh 22 | -------------------------------------------------------------------------------- /docker/metrics/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM prom/prometheus:v2.2.0 2 | 3 | 4 | ADD ./docker/metrics/prometheus.yaml /etc/prometheus/prometheus.yml 5 | ADD ./docker/metrics/django.rules.yml /etc/prometheus/django.rules.yml 6 | # Consoles can be found in localhost:9090/consoles/django.html 7 | ADD ./docker/metrics/consoles/django.html /usr/share/prometheus/consoles/ 8 | 9 | ENTRYPOINT [] 10 | 11 | # Strangely, replacing the entrypoint doesn't properly work. Deleting it and adding everything as CMD does the trick 12 | CMD /bin/prometheus --config.file=/etc/prometheus/prometheus.yml --web.console.libraries=/usr/share/prometheus/console_libraries --web.console.templates=/usr/share/prometheus/consoles 13 | -------------------------------------------------------------------------------- /environment.env: -------------------------------------------------------------------------------- 1 | # Comment this line for not run in DEBUG_MODE 2 | DEBUG_MODE=True 3 | # Be careful to update this value when out of debug mode 4 | ALLOWED_HOSTS=localhost,server 5 | 6 | # Syslog parameters 7 | SYSLOG_HOST=log 8 | SYSLOG_PORT=514 9 | 10 | # Database parameters 11 | # If you change this variables for development 12 | # be sure to change the equivalente args in docker-compose.yaml 13 | # and rebuild the db service container 14 | # from scratch using 15 | # docker-compose build --no-cache db 16 | POSTGRES_HOST=db 17 | POSTGRES_PORT=5432 18 | POSTGRES_DB=templatesitedb 19 | POSTGRES_USER=postgres 20 | POSTGRES_PASSWORD=somepassword 21 | 22 | 23 | # For system tests only 24 | SYSTEM_TEST_HOSTPORT=http://server:80/ 25 | -------------------------------------------------------------------------------- /src/tweet/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11 on 2017-07-22 10:13 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='Tweet', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('text', models.CharField(max_length=140)), 21 | ('timestamp', models.DateTimeField(auto_now_add=True)), 22 | ], 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /src/tweet/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | from tweet.models import Tweet 3 | from rest_framework.generics import RetrieveAPIView, ListCreateAPIView 4 | from rest_framework import serializers 5 | 6 | 7 | # Create your views here. 8 | class TweetSerializer(serializers.HyperlinkedModelSerializer): 9 | href = serializers.HyperlinkedIdentityField(view_name='get_tweet') 10 | 11 | class Meta: 12 | model = Tweet 13 | fields = ('text', 'timestamp', 'href') 14 | 15 | 16 | class TweetView(RetrieveAPIView): 17 | queryset = Tweet.objects.all() 18 | serializer_class = TweetSerializer 19 | 20 | 21 | class TweetListView(ListCreateAPIView): 22 | queryset = Tweet.objects.all() 23 | serializer_class = TweetSerializer 24 | -------------------------------------------------------------------------------- /docker/deps/build_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo 'Building dependencies' 4 | mkdir -p /opt/wheels/ 5 | cd /opt/vendor 6 | 7 | echo 'Building wheels from requirements' 8 | pip3 wheel -r /opt/deps/requirements.txt --process-dependency-links 9 | 10 | echo 'Done' 11 | 12 | # Clean the direct dependencies to avoid issues with caches 13 | for D in /opt/deps/*; do 14 | if [ -d "${D}" ]; then 15 | echo "Removing dependency in ${D}" # your processing here 16 | PKG_NAME=`python3 ${D}/setup.py --name` 17 | echo "Package name to remove ${PKG_NAME}" 18 | WHEEL=`python3 /opt/search_wheels.py $PKG_NAME -d /opt/vendor` 19 | echo "Deleting file $WHEEL" 20 | rm $WHEEL 21 | fi 22 | done 23 | 24 | echo "Wheels available: `ls /opt/vendor/*.whl`" 25 | -------------------------------------------------------------------------------- /src/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "templatesite.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | except ImportError: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) 21 | raise 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /system-test/healthcheck/test_healthcheck.py: -------------------------------------------------------------------------------- 1 | import os 2 | import requests 3 | 4 | HOSTPORT = os.environ.get('SYSTEM_TEST_HOSTPORT') 5 | HEALTHCHECK_URL = HOSTPORT + 'healthcheck' 6 | 7 | 8 | def test_healthcheck(): 9 | response = requests.get(HEALTHCHECK_URL) 10 | assert response.status_code == 200 11 | 12 | 13 | def test_request_id(): 14 | ''' Ensure the request id is returned ''' 15 | response = requests.get(HEALTHCHECK_URL) 16 | assert response.status_code == 200 17 | assert 'X-REQUEST-ID' in response.headers 18 | 19 | 20 | def test_external_request_id(): 21 | ''' Ensure the request id returned s the same as set up''' 22 | headers = { 23 | 'X-REQUEST-ID': 'test_id', 24 | } 25 | response = requests.get(HEALTHCHECK_URL, headers=headers) 26 | assert response.status_code == 200 27 | assert 'X-REQUEST-ID' in response.headers 28 | assert response.headers['X-REQUEST-ID'] == 'test_id' 29 | -------------------------------------------------------------------------------- /docker/server/start_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | _term() { 4 | echo "Caught SIGTERM signal! Sending graceful stop to uWSGI through the master-fifo" 5 | # See details in the uwsgi.ini file and 6 | # in http://uwsgi-docs.readthedocs.io/en/latest/MasterFIFO.html 7 | # q means "graceful stop" 8 | echo q > /tmp/uwsgi-fifo 9 | } 10 | 11 | trap _term SIGTERM 12 | 13 | # Allow to define dollars in the templates 14 | export DOLLAR='$' 15 | envsubst < /opt/server/nginx.conf.template > /etc/nginx/conf.d/default.conf 16 | envsubst < /opt/server/uwsgi.ini.template > /opt/server/uwsgi.ini 17 | nginx 18 | uwsgi --ini /opt/server/uwsgi.ini & 19 | 20 | # We need to wait to properly catch the signal, that's why uWSGI is started 21 | # in the background. $! is the PID of uWSGI 22 | wait $! 23 | # The container exits with code 143, which means "exited because SIGTERM" 24 | # 128 + 15 (SIGTERM) 25 | # http://www.tldp.org/LDP/abs/html/exitcodes.html 26 | # http://tldp.org/LDP/Bash-Beginners-Guide/html/sect_12_02.html 27 | -------------------------------------------------------------------------------- /src/templatesite/urls.py: -------------------------------------------------------------------------------- 1 | """templatesite URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.11/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.conf.urls import include, url 17 | from django.contrib import admin 18 | 19 | urlpatterns = [ 20 | url(r'^admin/', admin.site.urls), 21 | url(r'^healthcheck/', include('healthcheck.urls')), 22 | url(r'^tweet/', include('tweet.urls')), 23 | url('', include('django_prometheus.urls')), 24 | ] 25 | -------------------------------------------------------------------------------- /docker/server/stack-fix.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | // THIS IS TO AVOID A SIGFAULT WHEN RUNNING python3.6 manage.py runserver 6 | // This should be fixed at some point by Alpine and/or Python 7 | // Check this issue for more info 8 | // https://github.com/docker-library/python/issues/211 9 | typedef int (*func_t)(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void*), void *arg); 10 | 11 | int pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void*), void *arg) { 12 | printf("XXX pthread_create override called.\n"); 13 | 14 | pthread_attr_t local; 15 | int used = 0, ret; 16 | 17 | if (!attr) { 18 | used = 1; 19 | pthread_attr_init(&local); 20 | attr = &local; 21 | } 22 | pthread_attr_setstacksize((void*)attr, 2 * 1024 * 1024); // 2 MB 23 | 24 | func_t orig = (func_t)dlsym(RTLD_NEXT, "pthread_create"); 25 | 26 | ret = orig(thread, attr, start_routine, arg); 27 | 28 | if (used) { 29 | pthread_attr_destroy(&local); 30 | } 31 | 32 | return ret; 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Jaime Buelta 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/healthcheck/tests.py: -------------------------------------------------------------------------------- 1 | from django.core.urlresolvers import reverse 2 | from django.test import TestCase 3 | from unittest import mock 4 | 5 | 6 | class HealthCheck(TestCase): 7 | 8 | def test_good_healthcheck(self): 9 | url = reverse('healthcheck') 10 | response = self.client.get(url) 11 | assert response.status_code == 200 12 | 13 | # Check structure 14 | result = response.json() 15 | assert result == {'status': 'ok', 'db': {'status': 'ok'}} 16 | 17 | @mock.patch('tweet.models.Tweet.objects.first') 18 | def test_bad_healthcheck(self, db_mock): 19 | db_mock.side_effect = Exception('This is an error') 20 | 21 | url = reverse('healthcheck') 22 | response = self.client.get(url) 23 | assert response.status_code == 500 24 | 25 | # Check structure 26 | result = response.json() 27 | expected_result = { 28 | 'status': 'nok', 29 | 'db': { 30 | 'status': 'nok', 31 | 'err_msg': 'Error accessing DB: This is an error', 32 | }, 33 | } 34 | assert result == expected_result 35 | -------------------------------------------------------------------------------- /src/healthcheck/views.py: -------------------------------------------------------------------------------- 1 | from django.http import JsonResponse 2 | from tweet.models import Tweet 3 | import logging 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | 8 | def healthcheck(request): 9 | ''' 10 | Check status of each external service. 11 | Remember to keep everything lightweight and add short timeouts 12 | ''' 13 | result = {'status': 'ok'} 14 | logger.info('Performing healthcheck') 15 | 16 | # Check DB making a lightweight DB query 17 | try: 18 | Tweet.objects.first() 19 | result['db'] = {'status': 'ok'} 20 | except Exception as err: 21 | result['status'] = 'nok' 22 | err_msg = 'Error accessing DB: {}'.format(err) 23 | result['db'] = { 24 | 'status': 'nok', 25 | 'err_msg': err_msg, 26 | } 27 | logger.error(err_msg) 28 | 29 | logger.debug('Healtchcheck result {}'.format(result)) 30 | 31 | status_code = 200 32 | if result['status'] != 'ok': 33 | logger.error('Healthcheck result is bad') 34 | status_code = 500 35 | else: 36 | logger.info('Healtchcheck result is ok') 37 | 38 | response = JsonResponse(result) 39 | response.status_code = status_code 40 | return response 41 | -------------------------------------------------------------------------------- /src/tweet/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django.core.urlresolvers import reverse 3 | from tweet.models import Tweet 4 | 5 | 6 | class TweetTest(TestCase): 7 | 8 | def setUp(self): 9 | self.tweet = Tweet.objects.create(text='Test tweet') 10 | Tweet.objects.create(text='Another tweet') 11 | 12 | def test_list_tweets(self): 13 | url = reverse('list_tweets') 14 | response = self.client.get(url) 15 | assert response.status_code == 200 16 | result = response.json() 17 | assert len(result) == 2 18 | 19 | # Walk throught the results 20 | for tweet in result: 21 | response = self.client.get(tweet['href']) 22 | assert response.status_code == 200 23 | 24 | def test_single_tweet(self): 25 | url = reverse('get_tweet', kwargs={'pk': self.tweet.id}) 26 | response = self.client.get(url) 27 | assert response.status_code == 200 28 | 29 | # Check structure 30 | result = response.json() 31 | assert set(result.keys()) == {'text', 'href', 'timestamp'} 32 | assert result['text'] == 'Test tweet' 33 | assert url in result['href'] 34 | 35 | def test_bad_tweet(self): 36 | url = reverse('get_tweet', kwargs={'pk': 12345}) 37 | response = self.client.get(url) 38 | assert response.status_code == 404 39 | -------------------------------------------------------------------------------- /vendor/README.md: -------------------------------------------------------------------------------- 1 | Pregenerated wheels 2 | =================== 3 | The dependencies can be precompiled in wheels to avoid time compiling while building the service. 4 | 5 | The service `build-deps` is doing that. Collect all dependencies from requirements.txt file, downloading them, 6 | and compiling them as python wheel files. The wheel files are there shared with the host in the ./vendor 7 | directory 8 | 9 | **This step is optional. The container should build with an empty ./vendor directory** 10 | 11 | Some extra dependencies (like compilers, dev packages, etc) may be required in `docker/deps/Dockerfile` 12 | to allow the creation of the wheel. 13 | 14 | To generate the dependencies, run 15 | 16 | docker-compose up --build build-deps 17 | 18 | Remember to run it again if you change the recipe (like adding a new dependency), which will rebuild all dependencies. 19 | The generation of wheels will be performed on deployment using the cache. 20 | 21 | If you want to force the rebuild, run 22 | 23 | docker-compose build --no-cache build-deps 24 | docker-compose up build-deps 25 | 26 | Note that direct dependencies in ./deps won't generate a wheel*, but their dependencies will. This is 27 | done to ensure they are always installed fresh, as they are likely to be changed often. See more info 28 | in ./deps/README.md 29 | 30 | * (The wheel will be generated internally to ensure compilation of dependencies, but it will be deleted) * 31 | -------------------------------------------------------------------------------- /docker/db/postgres-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | chown -R postgres "$PGDATA" 3 | 4 | if [ -z "$(ls -A "$PGDATA")" ]; then 5 | su-exec postgres initdb 6 | sed -ri "s/^#(listen_addresses\s*=\s*)\S+/\1'*'/" "$PGDATA"/postgresql.conf 7 | 8 | : ${POSTGRES_USER:="postgres"} 9 | : ${POSTGRES_DB:=$POSTGRES_USER} 10 | 11 | if [ "$POSTGRES_PASSWORD" ]; then 12 | pass="PASSWORD '$POSTGRES_PASSWORD'" 13 | authMethod=md5 14 | else 15 | echo "===============================" 16 | echo "!!! Use \$POSTGRES_PASSWORD env var to secure your database !!!" 17 | echo "===============================" 18 | pass= 19 | authMethod=trust 20 | fi 21 | echo 22 | 23 | if [ "$POSTGRES_DB" != 'postgres' ]; then 24 | echo "Creating database $POSTGRES_DB" 25 | createSql="CREATE DATABASE $POSTGRES_DB;" 26 | echo $createSql | su-exec postgres postgres --single -jE 27 | echo 28 | fi 29 | 30 | if [ "$POSTGRES_USER" != 'postgres' ]; then 31 | op=CREATE 32 | else 33 | op=ALTER 34 | fi 35 | 36 | userSql="$op USER $POSTGRES_USER WITH SUPERUSER $pass;" 37 | echo $userSql | su-exec postgres postgres --single -jE 38 | echo 39 | 40 | { echo; echo "host all all 0.0.0.0/0 $authMethod"; } >> "$PGDATA"/pg_hba.conf 41 | fi 42 | 43 | # Create the run socket directory and grant proper permissions 44 | mkdir -p /run/postgresql/ 45 | chown -R postgres:postgres /run/postgresql/ 46 | # exec su-exec postgres "$@" 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *.cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # Jupyter Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # SageMath parsed files 79 | *.sage.py 80 | 81 | # Environments 82 | .env 83 | .venv 84 | env/ 85 | venv/ 86 | ENV/ 87 | 88 | # Spyder project settings 89 | .spyderproject 90 | .spyproject 91 | 92 | # Rope project settings 93 | .ropeproject 94 | 95 | # mkdocs documentation 96 | /site 97 | 98 | # mypy 99 | .mypy_cache/ 100 | 101 | .DS_Store 102 | 103 | *.whl 104 | -------------------------------------------------------------------------------- /docker/deps/search_wheels.py: -------------------------------------------------------------------------------- 1 | import zipfile 2 | import re 3 | import os 4 | import argparse 5 | 6 | 7 | def main(dir, name_to_search): 8 | # Check all files in the directory and print the name of the package 9 | for root, dirs, files in os.walk(dir): 10 | wheels = (fname for fname in files if fname.endswith('whl')) 11 | for fname in wheels: 12 | filename = os.path.join(dir, fname) 13 | zfile = zipfile.ZipFile(filename) 14 | metadata = [file for file in zfile.infolist() 15 | if file.filename.endswith('METADATA')][0] 16 | data = zfile.open(metadata.filename) 17 | name = [line.rstrip().decode('ascii') 18 | for line in data.readlines() if b'Name' in line][0] 19 | # Extract the name 20 | name = re.match('Name: (?P\S+)$', name).groupdict()['name'] 21 | if name == name_to_search: 22 | print(filename) 23 | exit(0) 24 | 25 | # Check if the name replaces underscores with dashes 26 | # The wheel documentation is VERY confusing and inconsistent 27 | # about this 28 | if name.replace('_', '-') == name_to_search: 29 | print(filename) 30 | exit(0) 31 | 32 | if name.replace('-', '_') == name_to_search: 33 | print(filename) 34 | exit(0) 35 | 36 | print('Package {} not found'.format(name_to_search)) 37 | exit(1) 38 | 39 | 40 | if __name__ == '__main__': 41 | desc = 'Return the wheel that that contains the package name' 42 | parser = argparse.ArgumentParser(description=desc) 43 | parser.add_argument('-d', dest='dir', 44 | help='directory to search') 45 | parser.add_argument('name', help='Name of the package to search') 46 | args = parser.parse_args() 47 | 48 | main(args.dir, args.name) 49 | -------------------------------------------------------------------------------- /deps/README.md: -------------------------------------------------------------------------------- 1 | Embedded dependencies 2 | ===================== 3 | This directory is created to set direct dependencies that are not under pypi control or have more 4 | manual installation. This is mainly aimed to modules that live in private git repos and are not 5 | downloadable from PyPI, avoiding problems like requiring to pull from the repo with a ssh key 6 | from inside the container. 7 | 8 | The recommended way of dealing with them is to add them into this subdir git submodules and use 9 | the Python [setuptool module](https://docs.python.org/3.6/distributing/index.html) (setup.py). 10 | 11 | Note that dependencies here won't be installed from wheel, though their dependencies will be (if 12 | done in the proper format, through a setup.py). That's to avoid problems with setup and cache the 13 | wrong version, with often happens while developing. 14 | Remember to include the dependency in the requirements.txt file 15 | 16 | An example of a module has been included (django-prometheus) 17 | 18 | How to add a new submodule 19 | ========================== 20 | 21 | cd deps 22 | git submodule add https://github.com/foo 23 | 24 | this creates the subdir foo with the submodule. The file .gitmodules will be updated and needs 25 | to be tracked and commited. 26 | 27 | How to update a submodule 28 | ========================== 29 | 30 | Log into the submodule and set git to the desired commit/tag/branch 31 | 32 | cd deps/foo 33 | git checkout v7.5.0 34 | # or, for latest commit in current branch 35 | git pull 36 | 37 | Then add the commit to the main repo, like a regular file 38 | 39 | cd .. 40 | git add foo 41 | git commit 42 | 43 | 44 | If the submodule is updated 45 | ============================ 46 | 47 | and yours is not the proper version, it will appear as 48 | 49 | git status 50 | modified: foo (new commits) 51 | 52 | Get the new commits with the command 53 | 54 | git submodule update --remote 55 | 56 | 57 | NOTE: Working with git submodules is a little tricky. Feel free to add and modify this document. 58 | -------------------------------------------------------------------------------- /docker/server/nginx.conf.template: -------------------------------------------------------------------------------- 1 | ## 2 | # You should look at the following URL's in order to grasp a solid understanding 3 | # of Nginx configuration files in order to fully unleash the power of Nginx. 4 | # http://wiki.nginx.org/Pitfalls 5 | # http://wiki.nginx.org/QuickStart 6 | # http://wiki.nginx.org/Configuration 7 | # 8 | # Generally, you will want to move this file somewhere, and start with a clean 9 | # file but keep this around for reference. Or just disable in sites-enabled. 10 | # 11 | # Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples. 12 | ## 13 | 14 | # Default server configuration 15 | # 16 | # Specifies the log format. 17 | log_format custom '${DOLLAR}remote_addr - ${DOLLAR}remote_user [${DOLLAR}time_local] "${DOLLAR}request" ' 18 | '${DOLLAR}status ${DOLLAR}body_bytes_sent "${DOLLAR}http_referer" ' 19 | '"${DOLLAR}http_user_agent" "${DOLLAR}http_x_forwarded_for"'; 20 | 21 | server { 22 | listen 80 default_server; 23 | listen [::]:80 default_server; 24 | 25 | 26 | # Direct the error log towards the centralised logging 27 | error_log syslog:server=${SYSLOG_HOST}:${SYSLOG_PORT} debug; 28 | access_log syslog:server=${SYSLOG_HOST}:${SYSLOG_PORT},tag=nginx,severity=info custom; 29 | rewrite_log on; 30 | 31 | # SSL configuration 32 | # 33 | # listen 443 ssl default_server; 34 | # listen [::]:443 ssl default_server; 35 | # 36 | # Note: You should disable gzip for SSL traffic. 37 | # See: https://bugs.debian.org/773332 38 | # 39 | # Read up on ssl_ciphers to ensure a secure configuration. 40 | # See: https://bugs.debian.org/765782 41 | # 42 | # Self signed certs generated by the ssl-cert package 43 | # Don't use them in a production server! 44 | # 45 | # include snippets/snakeoil.conf; 46 | 47 | root /opt/; 48 | 49 | # Add index.php to the list if you are using PHP 50 | # index index.html index.htm index.nginx-debian.html; 51 | 52 | # server_name _; 53 | 54 | location /static/ { 55 | autoindex on; 56 | try_files ${DOLLAR}uri ${DOLLAR}uri/ =404; 57 | } 58 | 59 | location / { 60 | proxy_set_header Host ${DOLLAR}host; 61 | proxy_set_header X-Real-IP ${DOLLAR}remote_addr; 62 | uwsgi_pass unix:///tmp/uwsgi.sock; 63 | include uwsgi_params; 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | ARG django_secret_key 4 | ENV DJANGO_SECRET_KEY $django_secret_key 5 | 6 | # Add requirements for python and pip 7 | RUN apk add --update python3 pytest 8 | RUN apk add --update postgresql-libs 9 | RUN apk add --update curl 10 | # Add envsubts 11 | RUN apk add --update gettext 12 | # Add nginx 13 | RUN apk add --update nginx 14 | RUN mkdir -p /run/nginx 15 | 16 | RUN mkdir -p /opt/code 17 | WORKDIR /opt/code 18 | 19 | ADD requirements.txt /opt/code 20 | 21 | # Try to use local wheels. Even if not present, it will proceed 22 | ADD ./vendor /opt/vendor 23 | ADD ./deps /opt/deps 24 | # Only install them if there's any 25 | RUN if ls /opt/vendor/*.whl 1> /dev/null 2>&1; then pip3 install /opt/vendor/*.whl; fi 26 | 27 | # Add uwsgi and nginx configuration 28 | RUN mkdir -p /opt/server 29 | RUN mkdir -p /opt/static 30 | 31 | 32 | # Add fix for stack for Python3.6 33 | ADD ./docker/server/stack-fix.c /opt/server 34 | 35 | # Some Docker-fu. In one step install the compile packages, install the 36 | # dependencies and then remove them. That skims the image size quite 37 | # sensibly. 38 | RUN apk add --no-cache --virtual .build-deps \ 39 | python3-dev build-base linux-headers gcc postgresql-dev \ 40 | # Hack to fix the problem with runserver 41 | && gcc -shared -fPIC /opt/server/stack-fix.c -o /opt/server/stack-fix.so \ 42 | # Installing python requirements 43 | && pip3 install -r requirements.txt \ 44 | && find /usr/local \ 45 | \( -type d -a -name test -o -name tests \) \ 46 | -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ 47 | -exec rm -rf '{}' + \ 48 | && runDeps="$( \ 49 | scanelf --needed --nobanner --recursive /usr/local \ 50 | | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \ 51 | | sort -u \ 52 | | xargs -r apk info --installed \ 53 | | sort -u \ 54 | )" \ 55 | # Install uwsgi, from python 56 | && pip3 install uwsgi \ 57 | && apk add --virtual .rundeps $runDeps \ 58 | && apk del .build-deps 59 | 60 | 61 | ADD ./docker/server/uwsgi.ini.template /opt/server 62 | ADD ./docker/server/nginx.conf.template /opt/server 63 | ADD ./docker/server/start_server.sh /opt/server 64 | 65 | # Add code 66 | ADD ./src/ /opt/code/ 67 | 68 | # Generate static files 69 | RUN python3 manage.py collectstatic 70 | 71 | EXPOSE 80 72 | CMD ["/bin/sh", "/opt/server/start_server.sh"] 73 | HEALTHCHECK CMD curl --fail http://localhost/healthcheck/ 74 | -------------------------------------------------------------------------------- /docker/db/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | # Add the proper env variables for init the db 4 | ARG POSTGRES_DB 5 | ENV POSTGRES_DB $POSTGRES_DB 6 | ARG POSTGRES_USER 7 | ENV POSTGRES_USER $POSTGRES_USER 8 | ARG POSTGRES_PASSWORD 9 | ENV POSTGRES_PASSWORD $POSTGRES_PASSWORD 10 | ARG POSTGRES_PORT 11 | 12 | # secret key for Django 13 | ARG django_secret_key 14 | ENV DJANGO_SECRET_KEY $django_secret_key 15 | 16 | # For usage in migrations, etc 17 | ENV POSTGRES_HOST localhost 18 | 19 | RUN apk --update add \ 20 | bash nano curl su-exec\ 21 | python3 \ 22 | postgresql postgresql-contrib postgresql-dev && \ 23 | rm -rf /var/cache/apk/* 24 | 25 | ENV LANG en_US.utf8 26 | ENV PGDATA /var/lib/postgresql/data 27 | 28 | 29 | # ENTRYPOINT ["/postgres-entrypoint.sh"] 30 | 31 | EXPOSE $POSTGRES_PORT 32 | VOLUME /var/lib/postgresql/data 33 | 34 | 35 | # Adding our code 36 | RUN mkdir -p /opt/code 37 | RUN mkdir -p /opt/data 38 | # Store the data inside the container, as we don't care for 39 | # persistence 40 | ENV PGDATA /opt/data 41 | WORKDIR /opt/code 42 | 43 | RUN mkdir -p /opt/code/db 44 | WORKDIR /opt/code/db 45 | # Add postgres setup 46 | ADD ./docker/db/postgres-setup.sh /opt/code/db/ 47 | RUN ./postgres-setup.sh 48 | 49 | # Install our code to run migrations and prepare DB 50 | WORKDIR /opt/code 51 | ADD requirements.txt /opt/code 52 | 53 | # Try to use local wheels. Even if not present, it will proceed 54 | ADD ./vendor /opt/vendor 55 | ADD ./deps /opt/deps 56 | # Only install them if there's any 57 | RUN if ls /opt/vendor/*.whl 1> /dev/null 2>&1; then pip3 install /opt/vendor/*.whl; fi 58 | 59 | # Some Docker-fu. In one step install the compile packages, install the 60 | # dependencies and then remove them. That skims the image size quite 61 | # sensibly. 62 | RUN apk add --no-cache --virtual .build-deps \ 63 | python3-dev build-base linux-headers gcc \ 64 | && pip3 install -r requirements.txt \ 65 | && find /usr/local \ 66 | \( -type d -a -name test -o -name tests \) \ 67 | -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ 68 | -exec rm -rf '{}' + \ 69 | && runDeps="$( \ 70 | scanelf --needed --nobanner --recursive /usr/local \ 71 | | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \ 72 | | sort -u \ 73 | | xargs -r apk info --installed \ 74 | | sort -u \ 75 | )" \ 76 | && apk add --virtual .rundeps $runDeps \ 77 | && apk del .build-deps 78 | 79 | # Need to import all the code, due tangled dependencies 80 | ADD ./src/ /opt/code/ 81 | # Add all DB commanda 82 | ADD ./docker/db/* /opt/code/db/ 83 | 84 | # get migrations, etc, ready 85 | RUN /opt/code/db/prepare_django_db.sh 86 | 87 | CMD ["su-exec", "postgres", "postgres"] 88 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | # Development related 5 | db: 6 | env_file: environment.env 7 | build: 8 | context: . 9 | dockerfile: ./docker/db/Dockerfile 10 | args: 11 | # These values should be in sync with environment.env 12 | # for development. If you change them, you'll 13 | # need to rebuild the container 14 | - POSTGRES_DB=templatesitedb 15 | - POSTGRES_USER=postgres 16 | - POSTGRES_PASSWORD=somepassword 17 | - POSTGRES_PORT=5432 18 | # Remember to keep this consistent 19 | - django_secret_key=secret_key! 20 | log: 21 | env_file: environment.env 22 | build: 23 | context: . 24 | dockerfile: ./docker/log/Dockerfile 25 | metrics: 26 | build: 27 | context: . 28 | dockerfile: ./docker/metrics/Dockerfile 29 | ports: 30 | - "9090:9090" 31 | metrics-graph: 32 | image: grafana/grafana 33 | ports: 34 | - "3000:3000" 35 | depends_on: 36 | - metrics 37 | build-deps: 38 | env_file: environment.env 39 | build: 40 | context: . 41 | dockerfile: ./docker/deps/Dockerfile 42 | volumes: 43 | - ./vendor:/opt/ext_vendor 44 | command: /opt/copy_deps.sh 45 | dev-server: 46 | env_file: environment.env 47 | environment: 48 | - CONSOLE_LOGS=1 49 | build: 50 | context: . 51 | args: 52 | # Remember to keep this consistent 53 | - django_secret_key=secret_key! 54 | command: ./start_dev_server.sh 55 | ports: 56 | - "8000:80" 57 | volumes: 58 | - ./src:/opt/code 59 | depends_on: 60 | - db 61 | - log 62 | test: 63 | env_file: environment.env 64 | environment: 65 | - CONSOLE_LOGS=1 66 | build: 67 | context: . 68 | args: 69 | # Remember to keep this consistent 70 | - django_secret_key=secret_key! 71 | entrypoint: pytest 72 | volumes: 73 | - ./src:/opt/code 74 | depends_on: 75 | - db 76 | - build-deps 77 | system-test: 78 | env_file: environment.env 79 | build: 80 | context: . 81 | dockerfile: ./docker/system-test/Dockerfile 82 | entrypoint: pytest 83 | volumes: 84 | - ./system-test:/opt/system-test 85 | depends_on: 86 | - server 87 | 88 | # Producion related 89 | server: 90 | image: templatesite 91 | env_file: environment.env 92 | build: 93 | context: . 94 | args: 95 | # Remember to keep this consistent 96 | - django_secret_key=secret_key! 97 | command: /opt/server/start_server.sh 98 | ports: 99 | - "8080:80" 100 | depends_on: 101 | - db 102 | - log 103 | - metrics 104 | -------------------------------------------------------------------------------- /docker/metrics/django.rules: -------------------------------------------------------------------------------- 1 | # Aggregate request counters 2 | job:django_http_requests_before_middlewares_total:sum_rate30s = sum(rate(django_http_requests_before_middlewares_total[30s])) by (job) 3 | job:django_http_requests_unknown_latency_total:sum_rate30s = sum(rate(django_http_requests_unknown_latency_total[30s])) by (job) 4 | job:django_http_ajax_requests_total:sum_rate30s = sum(rate(django_http_ajax_requests_total[30s])) by (job) 5 | job:django_http_responses_before_middlewares_total:sum_rate30s = sum(rate(django_http_responses_before_middlewares_total[30s])) by (job) 6 | job:django_http_requests_unknown_latency_including_middlewares_total:sum_rate30s = sum(rate(django_http_requests_unknown_latency_including_middlewares_total[30s])) by (job) 7 | job:django_http_requests_body_total_bytes:sum_rate30s = sum(rate(django_http_requests_body_total_bytes[30s])) by (job) 8 | job:django_http_responses_streaming_total:sum_rate30s = sum(rate(django_http_responses_streaming_total[30s])) by (job) 9 | job:django_http_responses_body_total_bytes:sum_rate30s = sum(rate(django_http_responses_body_total_bytes[30s])) by (job) 10 | job:django_http_requests_total:sum_rate30s = sum(rate(django_http_requests_total_by_method[30s])) by (job) 11 | job:django_http_requests_total_by_method:sum_rate30s = sum(rate(django_http_requests_total_by_method[30s])) by (job,method) 12 | job:django_http_requests_total_by_transport:sum_rate30s = sum(rate(django_http_requests_total_by_transport[30s])) by (job,transport) 13 | job:django_http_requests_total_by_view:sum_rate30s = sum(rate(django_http_requests_total_by_view_transport_method[30s])) by (job,view) 14 | job:django_http_requests_total_by_view_transport_method:sum_rate30s = sum(rate(django_http_requests_total_by_view_transport_method[30s])) by (job,view,transport,method) 15 | job:django_http_responses_total_by_templatename:sum_rate30s = sum(rate(django_http_responses_total_by_templatename[30s])) by (job,templatename) 16 | job:django_http_responses_total_by_status:sum_rate30s = sum(rate(django_http_responses_total_by_status[30s])) by (job,status) 17 | job:django_http_responses_total_by_charset:sum_rate30s = sum(rate(django_http_responses_total_by_charset[30s])) by (job,charset) 18 | job:django_http_exceptions_total_by_type:sum_rate30s = sum(rate(django_http_exceptions_total_by_type[30s])) by (job,type) 19 | job:django_http_exceptions_total_by_view:sum_rate30s = sum(rate(django_http_exceptions_total_by_view[30s])) by (job,view) 20 | 21 | # Aggregate latency histograms 22 | job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s{quantile="50"} = histogram_quantile(0.50, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) by (job, le)) 23 | job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s{quantile="95"} = histogram_quantile(0.95, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) by (job, le)) 24 | job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s{quantile="99"} = histogram_quantile(0.99, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) by (job, le)) 25 | job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s{quantile="99.9"} = histogram_quantile(0.999, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) by (job, le)) 26 | job:django_http_requests_latency_seconds:quantile_rate30s{quantile="50"} = histogram_quantile(0.50, sum(rate(django_http_requests_latency_seconds_bucket[30s])) by (job, le)) 27 | job:django_http_requests_latency_seconds:quantile_rate30s{quantile="95"} = histogram_quantile(0.95, sum(rate(django_http_requests_latency_seconds_bucket[30s])) by (job, le)) 28 | job:django_http_requests_latency_seconds:quantile_rate30s{quantile="99"} = histogram_quantile(0.99, sum(rate(django_http_requests_latency_seconds_bucket[30s])) by (job, le)) 29 | job:django_http_requests_latency_seconds:quantile_rate30s{quantile="99.9"} = histogram_quantile(0.999, sum(rate(django_http_requests_latency_seconds_bucket[30s])) by (job, le)) 30 | 31 | # Aggregate model operations 32 | job:django_model_inserts_total:sum_rate1m = sum(rate(django_model_inserts_total[1m])) by (job, model) 33 | job:django_model_updates_total:sum_rate1m = sum(rate(django_model_updates_total[1m])) by (job, model) 34 | job:django_model_deletes_total:sum_rate1m = sum(rate(django_model_deletes_total[1m])) by (job, model) 35 | 36 | # Aggregate database operations 37 | job:django_db_new_connections_total:sum_rate30s = sum(rate(django_db_new_connections_total[30s])) by (alias, vendor) 38 | job:django_db_new_connection_errors_total:sum_rate30s = sum(rate(django_db_new_connection_errors_total[30s])) by (alias, vendor) 39 | job:django_db_execute_total:sum_rate30s = sum(rate(django_db_execute_total[30s])) by (alias, vendor) 40 | job:django_db_execute_many_total:sum_rate30s = sum(rate(django_db_execute_many_total[30s])) by (alias, vendor) 41 | job:django_db_errors_total:sum_rate30s = sum(rate(django_db_errors_total[30s])) by (alias, vendor, type) 42 | 43 | # Aggregate migrations 44 | job:django_migrations_applied_total:max = max(django_migrations_applied_total) by (job, connection) 45 | job:django_migrations_unapplied_total:max = max(django_migrations_unapplied_total) by (job, connection) 46 | -------------------------------------------------------------------------------- /docker/metrics/django.rules.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: /etc/prometheus/django.rules 3 | rules: 4 | - record: job:django_http_requests_before_middlewares_total:sum_rate30s 5 | expr: sum by(job) (rate(django_http_requests_before_middlewares_total[30s])) 6 | - record: job:django_http_requests_unknown_latency_total:sum_rate30s 7 | expr: sum by(job) (rate(django_http_requests_unknown_latency_total[30s])) 8 | - record: job:django_http_ajax_requests_total:sum_rate30s 9 | expr: sum by(job) (rate(django_http_ajax_requests_total[30s])) 10 | - record: job:django_http_responses_before_middlewares_total:sum_rate30s 11 | expr: sum by(job) (rate(django_http_responses_before_middlewares_total[30s])) 12 | - record: job:django_http_requests_unknown_latency_including_middlewares_total:sum_rate30s 13 | expr: sum by(job) (rate(django_http_requests_unknown_latency_including_middlewares_total[30s])) 14 | - record: job:django_http_requests_body_total_bytes:sum_rate30s 15 | expr: sum by(job) (rate(django_http_requests_body_total_bytes[30s])) 16 | - record: job:django_http_responses_streaming_total:sum_rate30s 17 | expr: sum by(job) (rate(django_http_responses_streaming_total[30s])) 18 | - record: job:django_http_responses_body_total_bytes:sum_rate30s 19 | expr: sum by(job) (rate(django_http_responses_body_total_bytes[30s])) 20 | - record: job:django_http_requests_total:sum_rate30s 21 | expr: sum by(job) (rate(django_http_requests_total_by_method[30s])) 22 | - record: job:django_http_requests_total_by_method:sum_rate30s 23 | expr: sum by(job, method) (rate(django_http_requests_total_by_method[30s])) 24 | - record: job:django_http_requests_total_by_transport:sum_rate30s 25 | expr: sum by(job, transport) (rate(django_http_requests_total_by_transport[30s])) 26 | - record: job:django_http_requests_total_by_view:sum_rate30s 27 | expr: sum by(job, view) (rate(django_http_requests_total_by_view_transport_method[30s])) 28 | - record: job:django_http_requests_total_by_view_transport_method:sum_rate30s 29 | expr: sum by(job, view, transport, method) (rate(django_http_requests_total_by_view_transport_method[30s])) 30 | - record: job:django_http_responses_total_by_templatename:sum_rate30s 31 | expr: sum by(job, templatename) (rate(django_http_responses_total_by_templatename[30s])) 32 | - record: job:django_http_responses_total_by_status:sum_rate30s 33 | expr: sum by(job, status) (rate(django_http_responses_total_by_status[30s])) 34 | - record: job:django_http_responses_total_by_charset:sum_rate30s 35 | expr: sum by(job, charset) (rate(django_http_responses_total_by_charset[30s])) 36 | - record: job:django_http_exceptions_total_by_type:sum_rate30s 37 | expr: sum by(job, type) (rate(django_http_exceptions_total_by_type[30s])) 38 | - record: job:django_http_exceptions_total_by_view:sum_rate30s 39 | expr: sum by(job, view) (rate(django_http_exceptions_total_by_view[30s])) 40 | - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s 41 | expr: histogram_quantile(0.5, sum by(job, le) (rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))) 42 | labels: 43 | quantile: "50" 44 | - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s 45 | expr: histogram_quantile(0.95, sum by(job, le) (rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))) 46 | labels: 47 | quantile: "95" 48 | - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s 49 | expr: histogram_quantile(0.99, sum by(job, le) (rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))) 50 | labels: 51 | quantile: "99" 52 | - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s 53 | expr: histogram_quantile(0.999, sum by(job, le) (rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))) 54 | labels: 55 | quantile: "99.9" 56 | - record: job:django_http_requests_latency_seconds:quantile_rate30s 57 | expr: histogram_quantile(0.5, sum by(job, le) (rate(django_http_requests_latency_seconds_bucket[30s]))) 58 | labels: 59 | quantile: "50" 60 | - record: job:django_http_requests_latency_seconds:quantile_rate30s 61 | expr: histogram_quantile(0.95, sum by(job, le) (rate(django_http_requests_latency_seconds_bucket[30s]))) 62 | labels: 63 | quantile: "95" 64 | - record: job:django_http_requests_latency_seconds:quantile_rate30s 65 | expr: histogram_quantile(0.99, sum by(job, le) (rate(django_http_requests_latency_seconds_bucket[30s]))) 66 | labels: 67 | quantile: "99" 68 | - record: job:django_http_requests_latency_seconds:quantile_rate30s 69 | expr: histogram_quantile(0.999, sum by(job, le) (rate(django_http_requests_latency_seconds_bucket[30s]))) 70 | labels: 71 | quantile: "99.9" 72 | - record: job:django_model_inserts_total:sum_rate1m 73 | expr: sum by(job, model) (rate(django_model_inserts_total[1m])) 74 | - record: job:django_model_updates_total:sum_rate1m 75 | expr: sum by(job, model) (rate(django_model_updates_total[1m])) 76 | - record: job:django_model_deletes_total:sum_rate1m 77 | expr: sum by(job, model) (rate(django_model_deletes_total[1m])) 78 | - record: job:django_db_new_connections_total:sum_rate30s 79 | expr: sum by(alias, vendor) (rate(django_db_new_connections_total[30s])) 80 | - record: job:django_db_new_connection_errors_total:sum_rate30s 81 | expr: sum by(alias, vendor) (rate(django_db_new_connection_errors_total[30s])) 82 | - record: job:django_db_execute_total:sum_rate30s 83 | expr: sum by(alias, vendor) (rate(django_db_execute_total[30s])) 84 | - record: job:django_db_execute_many_total:sum_rate30s 85 | expr: sum by(alias, vendor) (rate(django_db_execute_many_total[30s])) 86 | - record: job:django_db_errors_total:sum_rate30s 87 | expr: sum by(alias, vendor, type) (rate(django_db_errors_total[30s])) 88 | - record: job:django_migrations_applied_total:max 89 | expr: max by(job, connection) (django_migrations_applied_total) 90 | - record: job:django_migrations_unapplied_total:max 91 | expr: max by(job, connection) (django_migrations_unapplied_total) 92 | -------------------------------------------------------------------------------- /docker/metrics/consoles/django.html: -------------------------------------------------------------------------------- 1 | {{template "head" .}} 2 | 3 | {{template "prom_right_table_head"}} 4 | 5 | Django 6 | {{ template "prom_query_drilldown" (args "sum(up{job='django'})") }} 7 | / {{ template "prom_query_drilldown" (args "count(up{job='django'})") }} 8 | 9 | 10 | 11 | avg CPU 12 | {{ template "prom_query_drilldown" (args "avg by(job)(rate(process_cpu_seconds_total{job='django'}[5m]))" "s/s" "humanizeNoSmallPrefix") }} 13 | 14 | 15 | 16 | avg Memory 17 | {{ template "prom_query_drilldown" (args "avg by(job)(process_resident_memory_bytes{job='django'})" "B" "humanize1024") }} 18 | 19 | 20 | {{template "prom_right_table_tail"}} 21 | 22 | 23 | {{template "prom_content_head" .}} 24 |

Django

25 | 26 |

Requests

27 |

Total

28 |
29 | 41 | 42 |

By view

43 |
44 | 57 | 58 |

Latency (median)

59 |
60 | 72 | 73 |

Latency (99.9th percentile)

74 |
75 | 87 | 88 |

Models

89 |

Insertions/s

90 |
91 | 102 | 103 |

Updates/s

104 |
105 | 116 | 117 |

Deletions/s

118 |
119 | 130 | 131 |

Database

132 |

Connections/s

133 |
134 | 146 | 147 |

Connections errors/s

148 |
149 | 161 | 162 |

Queries/s

163 |
164 | 176 | 177 |

Errors/s

178 |
179 | 191 | 192 | {{template "prom_content_tail" .}} 193 | 194 | {{template "tail"}} 195 | -------------------------------------------------------------------------------- /src/templatesite/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for templatesite project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.11.3. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.11/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.11/ref/settings/ 11 | """ 12 | 13 | import os 14 | import logging 15 | from logging.handlers import SysLogHandler 16 | 17 | SYSLOG_ADDRESS = ( 18 | os.environ.get('SYSLOG_HOST', 'localhost'), 19 | int(os.environ.get('SYSLOG_PORT', 514)), 20 | ) 21 | 22 | 23 | # Add a special logger to log related occurrences in settings 24 | formatter = logging.Formatter('SETTINGS %(levelname)-8s %(message)s') 25 | settings_logger = logging.getLogger('settings') 26 | 27 | if not os.environ.get('CONSOLE_LOGS'): 28 | handler = SysLogHandler(address=SYSLOG_ADDRESS) 29 | handler.setFormatter(formatter) 30 | settings_logger.addHandler(handler) 31 | 32 | # Log settings also in stdout 33 | handler = logging.StreamHandler() 34 | handler.setFormatter(formatter) 35 | settings_logger.addHandler(handler) 36 | 37 | settings_logger.setLevel(logging.INFO) 38 | 39 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 40 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 41 | 42 | # Quick-start development settings - unsuitable for production 43 | # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ 44 | 45 | # SECURITY WARNING: keep the secret key used in production secret! 46 | SECRET_KEY = os.environ.get('DJANGO_SECRET_KEY') 47 | 48 | # SECURITY WARNING: don't run with debug turned on in production! 49 | DEBUG = bool(os.environ.get('DEBUG_MODE', False)) 50 | if DEBUG: 51 | settings_logger.critical('STARTING SERVER IN DEBUG MODE') 52 | 53 | ALLOWED_HOSTS = [] 54 | allowed_hosts = os.environ.get('ALLOWED_HOSTS', []) 55 | if allowed_hosts: 56 | ALLOWED_HOSTS = allowed_hosts.split(',') 57 | settings_logger.info('ALLOWED_HOSTS: {}'.format(ALLOWED_HOSTS)) 58 | 59 | 60 | # Application definition 61 | 62 | INSTALLED_APPS = [ 63 | 'django.contrib.admin', 64 | 'django.contrib.auth', 65 | 'django.contrib.contenttypes', 66 | 'django.contrib.sessions', 67 | 'django.contrib.messages', 68 | 'django.contrib.staticfiles', 69 | 'django_prometheus', 70 | 'rest_framework', 71 | 'tweet', 72 | ] 73 | 74 | MIDDLEWARE = [ 75 | 'django_prometheus.middleware.PrometheusBeforeMiddleware', 76 | 'log_request_id.middleware.RequestIDMiddleware', 77 | 'django.middleware.security.SecurityMiddleware', 78 | 'django.contrib.sessions.middleware.SessionMiddleware', 79 | 'django.middleware.common.CommonMiddleware', 80 | 'django.middleware.csrf.CsrfViewMiddleware', 81 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 82 | 'django.contrib.messages.middleware.MessageMiddleware', 83 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 84 | 'django_prometheus.middleware.PrometheusAfterMiddleware', 85 | ] 86 | 87 | ROOT_URLCONF = 'templatesite.urls' 88 | 89 | TEMPLATES = [ 90 | { 91 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 92 | 'DIRS': [], 93 | 'APP_DIRS': True, 94 | 'OPTIONS': { 95 | 'context_processors': [ 96 | 'django.template.context_processors.debug', 97 | 'django.template.context_processors.request', 98 | 'django.contrib.auth.context_processors.auth', 99 | 'django.contrib.messages.context_processors.messages', 100 | ], 101 | }, 102 | }, 103 | ] 104 | 105 | WSGI_APPLICATION = 'templatesite.wsgi.application' 106 | 107 | 108 | # Database 109 | # https://docs.djangoproject.com/en/1.11/ref/settings/#databases 110 | 111 | if 'POSTGRES_HOST' not in os.environ: 112 | settings_logger.warning('No DB configured. this may be initialisation') 113 | else: 114 | DATABASES = { 115 | 'default': { 116 | 'ENGINE': 'django.db.backends.postgresql', 117 | 'NAME': os.environ.get('POSTGRES_DB', 'WRONG_DB'), 118 | 'USER': os.environ.get('POSTGRES_USER', 'WRONG_USER'), 119 | 'HOST': os.environ.get('POSTGRES_HOST'), 120 | 'PORT': int(os.environ.get('DATABASE_PORT', 5432)), 121 | 'PASSWORD': os.environ.get('POSTGRES_PASSWORD', 'WRONG_PASSWORD'), 122 | }, 123 | } 124 | 125 | 126 | # Password validation 127 | # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators 128 | 129 | AUTH_PASSWORD_VALIDATORS = [ 130 | { 131 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 132 | }, 133 | { 134 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 135 | }, 136 | { 137 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 138 | }, 139 | { 140 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 141 | }, 142 | ] 143 | 144 | 145 | # Internationalization 146 | # https://docs.djangoproject.com/en/1.11/topics/i18n/ 147 | 148 | LANGUAGE_CODE = 'en-us' 149 | 150 | TIME_ZONE = 'UTC' 151 | 152 | USE_I18N = True 153 | 154 | USE_L10N = True 155 | 156 | USE_TZ = True 157 | 158 | 159 | # Static files (CSS, JavaScript, Images) 160 | # https://docs.djangoproject.com/en/1.11/howto/static-files/ 161 | 162 | STATIC_URL = '/static/' 163 | STATIC_ROOT = '/opt/static/' 164 | 165 | 166 | LOGGING = { 167 | 'version': 1, 168 | 'disable_existing_loggers': False, 169 | 'filters': { 170 | 'request_id': { 171 | '()': 'log_request_id.filters.RequestIDFilter' 172 | } 173 | }, 174 | 'formatters': { 175 | 'standard': { 176 | 'format': 'templatesite: %(levelname)-8s [%(asctime)s] [%(request_id)s] %(name)s: %(message)s' 177 | }, 178 | }, 179 | 'handlers': { 180 | # Only send to syslog info or higher 181 | 'syslog': { 182 | 'level': 'INFO', 183 | 'class': 'logging.handlers.SysLogHandler', 184 | 'address': SYSLOG_ADDRESS, 185 | 'filters': ['request_id'], 186 | 'formatter': 'standard', 187 | }, 188 | 'console': { 189 | 'level': 'DEBUG', 190 | 'class': 'logging.StreamHandler', 191 | 'filters': ['request_id'], 192 | 'formatter': 'standard', 193 | }, 194 | }, 195 | 'loggers': { 196 | # Top level for the application. Remember to set on 197 | # all loggers 198 | '': { 199 | 'handlers': ['syslog'], 200 | 'level': 'DEBUG', 201 | 'propagate': False, 202 | }, 203 | # For usage on runserver (dev-server) 204 | 'django.server': { 205 | 'handlers': ['console'], 206 | 'level': 'DEBUG', 207 | 'propagate': False, 208 | }, 209 | }, 210 | } 211 | 212 | if os.environ.get('CONSOLE_LOGS'): 213 | # Log all to the console as well. This is used while running unit tests 214 | del LOGGING['handlers']['syslog'] 215 | LOGGING['loggers']['']['handlers'] = ['console'] 216 | 217 | 218 | LOG_REQUESTS = True 219 | # Origin request will be X-REQUEST-ID 220 | LOG_REQUEST_ID_HEADER = "HTTP_X_REQUEST_ID" 221 | GENERATE_REQUEST_ID_IF_NOT_IN_HEADER = True 222 | REQUEST_ID_RESPONSE_HEADER = "X-REQUEST-ID" 223 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | A Django project template for a RESTful Application using Docker 2 | === 3 | 4 | This is a template for using Docker when building and operating a Django RESTful application 5 | 6 | Some opinionated ideas has been used. Please refer to this [blog post](https://wrongsideofmemphis.wordpress.com/2017/07/30/a-django-project-template-for-a-restful-application-using-docker/) 7 | for more info about the ideas and whys of this template, but in brief it aims to 8 | prepare a project to be easy to develop and deploy to production using Docker containers. 9 | 10 | 11 | Description 12 | ======= 13 | 14 | The application is a simple RESTful app that stores `tweets` and allow to create and retrieve them. 15 | It is a simple [Django](https://www.djangoproject.com/) project that uses [Django REST framework](http://www.django-rest-framework.org/). The template makes use of Docker to create several containers, with 16 | specific tasks aimed to development or to productionize the project. 17 | 18 | It uses python3, Django 1.11 and requires Docker 17 19 | 20 | 21 | Tree structure 22 | ======== 23 | 24 | ``` 25 | ├── README.md (this file) 26 | ├── docker-compose.yaml (general dockerfile description, aimed at development) 27 | ├── Dockerfile (General Dockerfile of the main server) 28 | ├── requirements.txt (python requirements) 29 | ├── environment.env (environment variables) 30 | ├── vendor (cache with generated wheels from dependencies) 31 | ├── deps (git submodules with embedded dependencies) 32 | ├── system-test 33 | │   └── pytest.ini 34 | │   └── requirements.txt (requirements for system tests) 35 | │   └── healtcheck/ 36 | │   └── ... 37 | ├── docker (Files related to build and operation of containers) 38 | │   └── (docker subdirs, like db or server) 39 | │      └── Scripts related to docker creation and operation of that service 40 | └── src (the Django project files) 41 | ├── manage.py 42 | ├── pytest.ini 43 | ├── healthcheck/ 44 |    └── ... 45 | ``` 46 | 47 | The code of the application is stored in `src`, and most of the docker-related files are in `docker` 48 | subdir. Two main ones are in the root directory, docker-compose.yaml and Dockerfile. These 49 | are the main docker files to operate at development. 50 | 51 | The application includes some healtchecks that should be used to check if the service is healthy. At the 52 | moment it just includes a check for the db, as well as a general check that the application is 53 | responding, but more checks can be added under the smoketests view. The view is included as heathcheck 54 | in the docker server, but it can be pulled externally as well in loadbalancers, etc. 55 | 56 | environment.env stores environment variables that can set up the configuration. This include details 57 | for the DB connection (host, user, password, etc), but also for other services. For example, 58 | the environment variable SYSLOG_HOST points to the syslog facility to store logs. In this 59 | file, it points to the `log` service container, but on deployment it should point to 60 | the proper host. 61 | 62 | 63 | Docker services for development 64 | ========= 65 | 66 | The main docker-compose file has all the services to run at development 67 | 68 | - *test*: Run the using tests, using [pytest](https://docs.pytest.org) and 69 | [pytest-django](https://pytest-django.readthedocs.io/) 70 | 71 | ``` 72 | docker-compose run test [pytest args] 73 | ``` 74 | pytest is very powerful and allows a big variety of parameters, I recomend that everyone checks the docs and 75 | learn a little bit about it. Some examples 76 | 77 | ``` 78 | # Run all tests 79 | docker-compose run test 80 | # Recreate the DB 81 | docker-compose run test --create-db 82 | # Run tests that fits with stringexp 83 | docker-compose run test -k stringexp 84 | # Run failed tests from last run 85 | docker-compose run test --lf 86 | ``` 87 | 88 | Some basic tests are added to the template. Note that the logs are directed to the console while running 89 | the tests, and will be captured by pytest. 90 | 91 | 92 | - *dev-server*: Run a dev server, aimed to check interactivly the app through a browser. It 93 | can be accessed at port 8000, and it will restart if the code of the application 94 | changes. It is using the django `runserver` command under the hood. 95 | ``` 96 | docker-compose up [-d] dev-server 97 | ``` 98 | - *db*: Database backend. It is started containing the data in the fixtures described in the code. 99 | 100 | 101 | Most of the changes in the code doesn't require restarting the services or rebuilding, but changes 102 | in the DB do, like new fixtures or a new migration. Build all services with 103 | 104 | ``` 105 | docker-compose build 106 | ``` 107 | 108 | - *build-deps*: Precompile all dependencies into wheels and copy them in ./vendor. This is not 109 | required, but can save time when rebuilding the containers often, to avoid compile the same code 110 | over and over. Check the more detailed documentation in the ./vendor/README.md file. 111 | Note that dependencies embedded in ./deps won't be compiled (though their dependencies will be). 112 | Check more details in the ./deps/README.md file. 113 | 114 | - *system-test*: Run system tests over the whole system. It send requests to the started system 115 | and checks the results. It used pytest, so all the details from *test* works here. 116 | Remember to be sure to rebuild the *server* service before running them. Be also sure to check the 117 | logs from the *log* service for insight while running them. 118 | 119 | - *log*: A syslog facility that centralises all the logs. All the system will direct their logs 120 | to here, making convenient to check. You can check on the logs as they are generated running 121 | 122 | ``` 123 | docker-compose exec log tail -f /var/log/templatesite.log 124 | ``` 125 | Remember that restarting the container will clean the file. This is convenient to not keep old logs 126 | around, but it needs to keep in mind. Generallt, you don not need to bring down this container. 127 | 128 | Most important logs are the one generated by Django, that are prepend with "templatesite". A 129 | request id is added on each request helping group logs. This request id can be supplied externally 130 | using the header X-REQUEST-ID, and it will be returned with the response. 131 | 132 | - *metrics*: Report metrics in a [Prometheus](https://prometheus.io/) format. The Prometheus console 133 | can be checked in the port 9090. The metrics are exported in the server in the url /metrics 134 | 135 | A Django dashboard can be found at `http://localhost:9090/consoles/django.html`. This can be 136 | tweaked in the file ./docker/metrics/consoles/django.html 137 | 138 | - *metrics-graph*: A Grafana container, as reference. This is presented directly from the Grafana 139 | standard container, and it should be pointed towards the metrics container 140 | in http://metrics:9090/. Follow the instructions in 141 | the [Grafana docs](http://docs.grafana.org/installation/docker/) 142 | Graphs and dashboards may be added, for example, querying: 143 | ``` 144 | rate(django_http_requests_total_by_view_transport_method[1m]) 145 | ``` 146 | To display all Django views. Be careful as the inherent non persistency of containers may destroy 147 | your dashboards. This should be used only as example. Getting good dashboards is important for 148 | production, but not so much for development. 149 | 150 | 151 | Docker services oriented to production 152 | ========= 153 | 154 | At the moment, the main docker-composer runs the main container with a developer configuration 155 | 156 | - *server*: Starts an http server serving the application. The application is served through 157 | uwsgi and nginx, and static files are cached through nginx. 158 | The service is available in http://localhost:8080/ when called through docker-compose. 159 | Please note that the container opens port 80. 160 | 161 | Once build, it can be used directly from the built container, though it need to connect to a valid db. 162 | And fill the environment variables adequately. A simple test can be done in 163 | 164 | ``` 165 | # Start the container djangodocker_server routing its port 80 to locahost:8080 166 | docker run -it --name test -p 8080:80 --env-file=your_environment.env templatesite 167 | ``` 168 | The command option `-it` allows to stop the container hitting CTRL+C, as it connects to it, instead of having to use `docker stop test`. See [Docker documentation](https://docs.docker.com/engine/reference/commandline/run/#examples) for more details. 169 | 170 | The container will be configurable using environment variables. Check the available values in 171 | the environment.env file. These variables will need to point to the proper deployment values. In the 172 | file they are defined for development purposes, but they won't work for a test outside that, as they 173 | refer to docker-compose specific values. 174 | Also note that any changes to the contaniner won't be in effect until is rebuild. 175 | --------------------------------------------------------------------------------