├── .dockerignore ├── .gitignore ├── .gitlab-ci.yml ├── Dockerfile ├── README.md ├── apps ├── __init__.py └── polling │ ├── __init__.py │ ├── apps.py │ ├── migrations │ ├── 0001_initial.py │ └── __init__.py │ ├── models.py │ └── views.py ├── conf ├── __init__.py ├── celery.py ├── settings.py ├── urls.py └── wsgi.py ├── devops ├── deploy.sh └── nginx.conf ├── docker-compose.yml ├── manage.py └── requirements.txt /.dockerignore: -------------------------------------------------------------------------------- 1 | .git -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | # InetlliJ platform 92 | .idea 93 | 94 | static/ 95 | media/ 96 | .DS_Store 97 | 98 | json-server-mok.json 99 | 100 | notes 101 | test.json 102 | 103 | #KDE 104 | .directory 105 | 106 | #db-files 107 | db.sqlite3 -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: docker:stable 2 | services: 3 | - docker:dind 4 | 5 | stages: 6 | - build 7 | - test 8 | - deploy 9 | 10 | variables: 11 | DOCKER_HOST: tcp://docker:2375 12 | DOCKER_DRIVER: overlay2 13 | 14 | before_script: 15 | - apk add --update py-pip && 16 | pip install docker-compose 17 | - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com 18 | 19 | build: 20 | stage: build 21 | script: 22 | - docker build -t registry.gitlab.com/your_username/your_project_name . 23 | - docker push registry.gitlab.com/your_username/your_project_name 24 | only: 25 | - master 26 | 27 | deploy: 28 | stage: deploy 29 | before_script: 30 | - apk add --no-cache openssh-client bash 31 | - mkdir -p ~/.ssh 32 | - echo "$DEPLOY_KEY" | tr -d '\r' > ~/.ssh/id_rsa 33 | - cat ~/.ssh/id_rsa 34 | - chmod 700 ~/.ssh/id_rsa 35 | - eval "$(ssh-agent -s)" 36 | - ssh-add ~/.ssh/id_rsa 37 | - ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts 38 | script: 39 | - bash devops/deploy.sh 40 | only: 41 | - master 42 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | 3 | ENV PYTHONUNBUFFERED 1 4 | 5 | ENV APP_ROOT /project 6 | 7 | WORKDIR ${APP_ROOT} 8 | 9 | RUN apt-get update 10 | 11 | RUN pip3 install -U pip 12 | 13 | COPY requirements.txt ${APP_ROOT}/requirements.txt 14 | 15 | RUN pip3 install -r ${APP_ROOT}/requirements.txt 16 | 17 | # Set the working directory to /app 18 | WORKDIR ${APP_ROOT} 19 | 20 | # Copy the current directory contents into the container at /app 21 | ADD . ${APP_ROOT} 22 | 23 | RUN chmod 775 -R ${APP_ROOT} 24 | 25 | CMD ['python3 manage.py collectstatic --noinput', '&&', '/bin/sh','-c','python manage.py runserver'] 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # How to deploy your Django application in 2 hours using Docker & GitLab CI/CD 2 | 3 | _Estimated reading time: 7 minutes_ 4 | 5 | ## Before start 6 | 7 | The guide is aimed to give a reader common knowledge about using Docker and Gitlab CI/CD for Django project deployment. 8 | 9 | Default demo-project contains: **Django + PostgreSQL + Celery + RabbitMQ + nginx** 10 | 11 | **Docker, Docker-Compose and GitLab CI/CD** are also used for the project continuous delivery. 12 | 13 | ## Step 1: Create a GitLab repository 14 | 15 | Create a [blank GitLab project](https://docs.gitlab.com/ee/gitlab-basics/create-project.html), 16 | push your project from local machine to GitLab. After that you will probably get something like that 17 | ![Project structure](https://pp.userapi.com/c850224/v850224063/1058e3/tEToCCj-dBk.jpg) 18 | 19 | For the guide I will use following project URL: 20 | 21 | `https://gitlab.com/your_username/your_project_name/` 22 | 23 | You have to replace `your_username` and `your_project_name` in the source code to make it work. 24 | 25 | ## Step 2: Create a Dockerfile 26 | 27 | **Source file:** [Dockerfile](Dockerfile) 28 | 29 | The second step is creating a Django & Celery image. A Docker image is a file, comprised of multiple layers, 30 | used to execute code in a Docker container. A Docker image is described in `Dockerfile` by default. The most interesting commands is: 31 | * `FROM python:3.6` creates a layer from the python:3.6 Docker image. You can use other versions of Python (3.5, 3.4, 2.7, ...) 32 | * `RUN pip3 install -r ${APP_ROOT}/requirements.txt` installs Python dependencies 33 | * `CMD ['python3 manage.py collectstatic --noinput', '&&', '/bin/sh','-c','python manage.py runserver']` 34 | is used to set a default command for the image. However it will be over-written by a Docker-compose file. 35 | 36 | ## Step 3: Define services in a Docker-Compose file 37 | 38 | **Source file:** [docker-compose.yml](docker-compose.yml) 39 | 40 | Using Docker-Compose is the one of the most easiest way to orchestrate all containers. 41 | For each part of the project you must create an image, there are images which described in `docker-compose.yml`: 42 | 43 | * `nginx` pulls an image from Docker Hub. The volume section overrides a default nginx config with the `devops/nginx.conf` 44 | * `web` is the main Django application's service that uses a Docker image created in the [Docker chapter](#docker) 45 | * `postgres` service. Using all default settings 46 | * `rabbit` uses `rabbitmq:3.7-management` which automatically run RabbitMQ admin web interface 47 | * `celery` service also using an previously created `Dockerfile` 48 | 49 | ## Step 4: Connect containers using environment variables 50 | **Source file:** [.env](.env) 51 | 52 | A module called `envparse` will be used to export environment variables. 53 | 54 | First of all, add it to `requirements.txt`. Then create `.env` at the root of the project: 55 | 56 | ```.dotenv 57 | POSTGRES_DB=postgres 58 | POSTGRES_USER=postgres 59 | POSTGRES_PASSWORD=postgres 60 | POSTGRES_HOST=postgres 61 | POSTGRES_PORT=5432 62 | CELERY_BROKER_URL=amqp://rabbitmq:rabbitmq@rabbit:5672/ 63 | DJANGO_SETTINGS_MODULE=conf.settings 64 | ``` 65 | 66 | ## Step 5: Create GitLab CI/CD pipeline 67 | **Source file:** [.gitlab-ci.yml](.gitlab-ci.yml) 68 | 69 | For the education purposes `.gitlab-ci.yml` contains only 2 basic steps: 70 | * `build` step builds Django application image and push it to a private registry. Substitute a registry URL by your own link. 71 | * `deploy` step logins to a server via ssh and pulls changes from registry 72 | 73 | The deploy step uses a `devops/deploy.sh` file. You have to change it: replace **47.47.47.47** to your remote address IP 74 | and replace registry URL: 75 | > docker pull registry.gitlab.com/your_username/your_project_name:latest 76 | 77 | ## Step 6: Push new files 78 | 79 | Push all new files and directories (`Dockerfile`, `docker-compose.yml`, `.gitlab-ci.yml`, `devops/`) to the branch `master`. 80 | 81 | ## Step 7: Generate SSH keys 82 | 83 | Locally run a command `ssh-keygen -t rsa`, it will prompt you to enter passphrase - **leave it blank**. 84 | The command generates two files with two keys: public and private. 85 | 86 | Copy the public key to your server `~/.ssh/authorized_keys` file. 87 | 88 | Also you must to set a CI/CD environment variable called `DEPLOY_KEY` to your private key. 89 | Go to **GitLab project page - Settings - CI / CD - Environment variables** and create a variable 90 | ![Set variable](https://pp.userapi.com/c854120/v854120736/8f3a/C-NCoEPFCBg.jpg) 91 | 92 | ## Step 8: Clone a project 93 | 94 | Log in the remote server via ssh and clone a project at the root: 95 | 96 | `git clone https://gitlab.com/your_username/your_project_name.git` 97 | 98 | ## Step 9. Make a commit to prove that everything works fine 99 | 100 | Or create an issue to make me now that the guide has a mistake 101 | -------------------------------------------------------------------------------- /apps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toert/django-gitlab-ci-guide/c4e5917a172a8746444673bec219f16ea4a881c6/apps/__init__.py -------------------------------------------------------------------------------- /apps/polling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toert/django-gitlab-ci-guide/c4e5917a172a8746444673bec219f16ea4a881c6/apps/polling/__init__.py -------------------------------------------------------------------------------- /apps/polling/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class PollingApp(AppConfig): 5 | name = 'apps.polling' 6 | -------------------------------------------------------------------------------- /apps/polling/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.7 on 2019-03-16 19:20 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [ 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='ExampleModel', 16 | fields=[ 17 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 18 | ('foobar', models.IntegerField()), 19 | ], 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /apps/polling/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toert/django-gitlab-ci-guide/c4e5917a172a8746444673bec219f16ea4a881c6/apps/polling/migrations/__init__.py -------------------------------------------------------------------------------- /apps/polling/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class ExampleModel(models.Model): 5 | foobar = models.IntegerField() -------------------------------------------------------------------------------- /apps/polling/views.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toert/django-gitlab-ci-guide/c4e5917a172a8746444673bec219f16ea4a881c6/apps/polling/views.py -------------------------------------------------------------------------------- /conf/__init__.py: -------------------------------------------------------------------------------- 1 | from .celery import app as celery_app 2 | 3 | __all__ = ["celery_app"] 4 | -------------------------------------------------------------------------------- /conf/celery.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import celery 4 | 5 | 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conf.settings") 7 | 8 | 9 | app = celery.Celery(__name__) 10 | 11 | # Using a string here means the worker don't have to serialize 12 | # the configuration object to child processes. 13 | # - namespace='CELERY' means all celery-related configuration keys 14 | # should have a `CELERY_` prefix. 15 | app.config_from_object("django.conf:settings", namespace="CELERY") 16 | 17 | # Load task modules from all registered Django app configs. 18 | app.autodiscover_tasks() 19 | -------------------------------------------------------------------------------- /conf/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for backend project. 3 | 4 | Generated by 'django-admin startproject' using Django 2.0. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.0/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/2.0/ref/settings/ 11 | """ 12 | import os 13 | 14 | from envparse import env 15 | 16 | env.read_envfile() 17 | 18 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 19 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 20 | 21 | # Quick-start development settings - unsuitable for production 22 | # See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ 23 | 24 | # SECURITY WARNING: keep the secret key used in production secret! 25 | SECRET_KEY = "g!zs#n$-95wdj7_63mieww$hq#o2vz==oqx^!@%-)n2nm1fqui" 26 | 27 | # SECURITY WARNING: don't run with debug turned on in production! 28 | DEBUG = env("DEBUG", cast=bool, default=True) 29 | 30 | ALLOWED_HOSTS = ["*"] 31 | 32 | INTERNAL_IPS = ["127.0.0.1"] 33 | 34 | 35 | LOG_LEVEL = 'ERROR' 36 | 37 | # Application definition 38 | 39 | PREREQ_APPS = [ 40 | "django.contrib.admin", 41 | "django.contrib.auth", 42 | "django.contrib.contenttypes", 43 | "django.contrib.sessions", 44 | "django.contrib.messages", 45 | "django.contrib.staticfiles", 46 | "django_celery_beat", 47 | 48 | ] 49 | 50 | PROJECT_APPS = [ 51 | "apps.polling", 52 | ] 53 | 54 | INSTALLED_APPS = PREREQ_APPS + PROJECT_APPS 55 | 56 | MIDDLEWARE = [ 57 | "django.middleware.security.SecurityMiddleware", 58 | "django.contrib.sessions.middleware.SessionMiddleware", 59 | "django.middleware.common.CommonMiddleware", 60 | "django.middleware.csrf.CsrfViewMiddleware", 61 | "django.contrib.auth.middleware.AuthenticationMiddleware", 62 | "django.contrib.messages.middleware.MessageMiddleware", 63 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 64 | ] 65 | 66 | ROOT_URLCONF = "conf.urls" 67 | 68 | TEMPLATES = [ 69 | { 70 | "BACKEND": "django.template.backends.django.DjangoTemplates", 71 | "DIRS": [os.path.join(BASE_DIR, "core/org_structure/templates")], 72 | "APP_DIRS": True, 73 | "OPTIONS": { 74 | "context_processors": [ 75 | "django.template.context_processors.debug", 76 | "django.template.context_processors.request", 77 | "django.contrib.auth.context_processors.auth", 78 | "django.contrib.messages.context_processors.messages", 79 | ] 80 | }, 81 | } 82 | ] 83 | 84 | WSGI_APPLICATION = "conf.wsgi.application" 85 | 86 | DATABASES = { 87 | "default": { 88 | "ENGINE": "django.db.backends.postgresql", 89 | "NAME": env("POSTGRES_DB", default=""), 90 | "USER": env("POSTGRES_USER", default=""), 91 | "PASSWORD": env("POSTGRES_PASSWORD", default=""), 92 | "HOST": env("POSTGRES_HOST", default=""), 93 | "PORT": env("POSTGRES_PORT", default=""), 94 | } 95 | } 96 | 97 | # http://docs.celeryq.org/en/latest/userguide/configuration.html#new-lowercase-settings 98 | 99 | 100 | CELERY_BROKER_URL = env("CELERY_BROKER_URL", "") 101 | 102 | CELERY_TASK_ROUTES = {} 103 | 104 | # Password validation 105 | # https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators 106 | 107 | AUTH_PASSWORD_VALIDATORS = [ 108 | {"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"}, 109 | {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, 110 | {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, 111 | {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, 112 | ] 113 | 114 | # Internationalization 115 | # https://docs.djangoproject.com/en/2.0/topics/i18n/ 116 | DEFAULT_LANGUAGE_CODE = "en" 117 | 118 | LANGUAGE_CODE = "en" 119 | 120 | TIME_ZONE = "UTC" 121 | 122 | USE_I18N = True 123 | 124 | USE_L10N = True 125 | 126 | USE_TZ = True 127 | 128 | # Static files (CSS, JavaScript, Images) 129 | # https://docs.djangoproject.com/en/2.0/howto/static-files/ 130 | 131 | ADMIN_PATH = env('DJANGO_ADMIN_PATH', 'admin/') 132 | STATIC_URL = os.path.join('/', ADMIN_PATH, 'static/') 133 | MEDIA_URL = "/media/" 134 | MEDIA_ROOT = env("MEDIA_ROOT", default=os.path.join(BASE_DIR, "media")) 135 | STATIC_ROOT = env("STATIC_ROOT", default=os.path.join(BASE_DIR, "static")) 136 | 137 | AUTHENTICATION_BACKENDS = ["django.contrib.auth.backends.ModelBackend"] 138 | -------------------------------------------------------------------------------- /conf/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.conf.urls.static import static 3 | from django.contrib import admin 4 | from django.urls import path 5 | 6 | urlpatterns = ( 7 | [ 8 | 9 | path(settings.ADMIN_PATH, admin.site.urls), 10 | ] 11 | + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) 12 | + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) 13 | ) 14 | -------------------------------------------------------------------------------- /conf/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for backend project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conf.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /devops/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ssh -o StrictHostKeyChecking=no root@47.47.47.47 << 'ENDSSH' 3 | cd /your_project_name 4 | docker login -u $REGISTRY_USER -p $CI_BUILD_TOKEN $CI_REGISTRY 5 | docker pull registry.gitlab.com/your_username/your_project_name:latest 6 | docker-compose up -d 7 | ENDSSH 8 | -------------------------------------------------------------------------------- /devops/nginx.conf: -------------------------------------------------------------------------------- 1 | upstream web { 2 | ip_hash; 3 | server web:8000; 4 | } 5 | 6 | server { 7 | location / { 8 | proxy_pass http://web/; 9 | } 10 | location /static { 11 | autoindex on; 12 | alias /static; 13 | } 14 | listen 80; 15 | server_name localhost; 16 | } -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | 5 | nginx: 6 | image: nginx:1.13 7 | ports: 8 | - 80:80 9 | restart: always 10 | volumes: 11 | - ./devops/nginx.conf:/etc/nginx/conf.d/default.conf 12 | - ./static:/static 13 | depends_on: 14 | - web 15 | 16 | web: 17 | image: registry.gitlab.com/your_username/your_project_name:latest 18 | build: 19 | context: . 20 | container_name: web 21 | ports: 22 | - '8000:8000' 23 | command: bash -c 'python manage.py migrate --noinput && python manage.py collectstatic --noinput && gunicorn conf.wsgi:application -w 2 -b :8000 --capture-output --log-level=info' 24 | depends_on: 25 | - postgres 26 | volumes: 27 | - ./static:/static 28 | - ./media:/media/ 29 | env_file: .env 30 | links: 31 | - rabbit 32 | - postgres 33 | 34 | postgres: 35 | restart: always 36 | image: postgres:10 37 | container_name: postgres 38 | 39 | rabbit: 40 | image: rabbitmq:3.7-management 41 | hostname: "rabbit" 42 | environment: 43 | RABBITMQ_DEFAULT_USER: "rabbitmq" 44 | RABBITMQ_DEFAULT_PASS: "rabbitmq" 45 | ports: 46 | - "15672:15672" 47 | - "5672:5672" 48 | 49 | celery: 50 | image: registry.gitlab.com/your_username/your_project_name:latest 51 | build: 52 | context: . 53 | container_name: cl01 54 | env_file: .env 55 | command: celery -A conf.celery:app worker -B --loglevel=debug --purge 56 | volumes: 57 | - ./static:/static 58 | - ./media:/media/ 59 | depends_on: 60 | - rabbit 61 | - web 62 | - postgres 63 | links: 64 | - rabbit 65 | - postgres 66 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conf.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError as exc: 10 | raise ImportError( 11 | "Couldn't import Django. Are you sure it's installed and " 12 | "available on your PYTHONPATH environment variable? Did you " 13 | "forget to activate a virtual environment?" 14 | ) from exc 15 | execute_from_command_line(sys.argv) 16 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | amqp==2.4.2 2 | billiard==3.5.0.5 3 | celery==4.2.1 4 | Django==2.1.7 5 | django-celery-beat==1.1.1 6 | envparse==0.2.0 7 | gunicorn==19.9.0 8 | kombu==4.4.0 9 | psycopg2==2.7.7 10 | pytz==2018.9 11 | vine==1.2.0 12 | --------------------------------------------------------------------------------