├── ci_testing_python ├── __init__.py └── app │ ├── __init__.py │ └── identidock.py ├── tests ├── e2e │ ├── test_e2e_identidock.py │ └── docker-compose.yml ├── integration │ ├── test_integration_identidock.py │ └── docker-compose.yml ├── component │ ├── mountebank │ │ └── dnmonster │ │ │ ├── imposters.ejs │ │ │ ├── mountebank_dnmonster_response.ejs │ │ │ ├── Dockerfile │ │ │ ├── mountebank_dnmonster_stub.ejs │ │ │ └── mountebank_dnmonster_response_2.ejs │ ├── Dockerfile │ ├── docker-compose.yml │ └── test_component_identidock.py ├── conftest.py ├── contract │ ├── test_contract_identidock.py │ └── docker-compose.yml ├── unit │ ├── docker-compose.yml │ └── test_unit_identidock.py └── test.sh ├── pytest.ini ├── requirements ├── requirements-main.txt └── requirements-dev.txt ├── setup.cfg ├── travis_deploy_rsa.enc ├── docker.env ├── scripts ├── add_known_hosts.sh ├── env ├── copy-ssh-keys-to-server.sh ├── deploy.sh ├── jenkins-setup.sh ├── travis-setup.sh ├── wait-for-it.sh └── build-tag-push-image.sh ├── .coveragerc ├── docker-compose.yml ├── docs ├── overview.rst ├── features.rst ├── ci_setup.rst ├── index.rst ├── installation.rst ├── usage.rst ├── testing.rst ├── Makefile └── conf.py ├── tox.ini ├── Dockerfile ├── LICENSE ├── .dockerignore ├── docker-entrypoint.sh ├── docker-compose.prod.yml ├── .gitignore ├── Jenkinsfile ├── setup.py ├── Makefile ├── .travis.yml └── README.rst /ci_testing_python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ci_testing_python/app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_identidock.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | print(sys.path) 4 | -------------------------------------------------------------------------------- /tests/integration/test_integration_identidock.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | print(sys.path) 4 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | norecursedirs = tests/component tests/contract tests/e2e 3 | testpaths = tests docs -------------------------------------------------------------------------------- /requirements/requirements-main.txt: -------------------------------------------------------------------------------- 1 | Flask == 0.12 2 | uwsgi == 2.0.14 3 | requests == 2.13.0 4 | redis == 2.10.5 -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | norecursedirs = tests/component tests/contract tests/e2e 3 | testpaths = tests docs -------------------------------------------------------------------------------- /travis_deploy_rsa.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anirbanroydas/ci-testing-python/HEAD/travis_deploy_rsa.enc -------------------------------------------------------------------------------- /tests/component/mountebank/dnmonster/imposters.ejs: -------------------------------------------------------------------------------- 1 | { 2 | "imposters": [ 3 | <% include mountebank_dnmonster_stub.ejs %> 4 | ] 5 | } -------------------------------------------------------------------------------- /requirements/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest == 3.0.6 2 | pytest-flask == 0.10.0 3 | doubles == 1.2.1 4 | tox == 2.6.0 5 | coverage == 4.3.4 6 | pytest-cov == 2.4.0 7 | coveralls == 1.1 -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from ci_testing_python.app.identidock import app as flask_app 3 | 4 | 5 | @pytest.fixture(scope='session') 6 | def app(): 7 | return flask_app 8 | -------------------------------------------------------------------------------- /tests/contract/test_contract_identidock.py: -------------------------------------------------------------------------------- 1 | # import unittest 2 | import pytest 3 | # from ci_testing_python.app.identidock import app 4 | 5 | 6 | 7 | if __name__ == '__main__': 8 | # unittest.main() 9 | pytest.main() 10 | -------------------------------------------------------------------------------- /docker.env: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export IDENTIDOCK_IMAGE_DEV_TAG=latest-dev 4 | export IDENTIDOCK_IMAGE_PROD_TAG=latest 5 | export REDIS_DATA_DIR=/data 6 | #export REDIS_DATA_VOLUME_NAME=redis_data_citestingpython_3 7 | export DOCKER_REPO=aroyd/ci-testing-python -------------------------------------------------------------------------------- /tests/contract/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | identidock_contract_tester: 6 | build: ../.. 7 | image: aroyd/identidock_contract_test:${IDENTIDOCK_CONTRACT_TEST_IMAGE_TAG:-latest} 8 | environment: 9 | ENV: CONTRACT_TEST 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/unit/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | identidock_unit_tester: 6 | build: ../.. 7 | image: aroyd/identidock_unit_test:${IDENTIDOCK_UNIT_TEST_IMAGE_TAG:-latest} 8 | environment: 9 | ENV: UNIT_TEST 10 | # volumes: 11 | # - .:/project 12 | 13 | 14 | -------------------------------------------------------------------------------- /scripts/add_known_hosts.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | 4 | set -e 5 | 6 | 7 | echo "Waiting for all the ssh keyscan to complete" 8 | for manager in $(seq 0 $((MANAGER_COUNT-1))); 9 | do 10 | MGR=MANAGER_$((manager+1)) 11 | ssh-keyscan -H ${!MGR} >> ~/.ssh/known_hosts 12 | 13 | done 14 | 15 | echo "ssh key scan complete" -------------------------------------------------------------------------------- /tests/component/mountebank/dnmonster/mountebank_dnmonster_response.ejs: -------------------------------------------------------------------------------- 1 | function (request, state) { 2 | var mocked_png_img = Buffer.from('I am a mocked image', 'binary') 3 | 4 | return { 5 | headers: { 6 | 'Content-Type': 'image/png' 7 | }, 8 | body: mocked_png_img 9 | }; 10 | } -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = true 3 | data_file = .coverage 4 | source = ./ci_testing_python 5 | 6 | [paths] 7 | source = 8 | ./ci_testing_python 9 | /project/ci_testing_python 10 | 11 | [report] 12 | show_missing = true 13 | skip_covered = false 14 | precision = 2 15 | 16 | [html] 17 | title = Coverage HTML Report 18 | directory = htmlcov -------------------------------------------------------------------------------- /tests/e2e/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | identidock_e2e_tester: 6 | build: ../.. 7 | image: aroyd/identidock_e2e_test:${IDENTIDOCK_E2E_TEST_IMAGE_TAG:-latest} 8 | environment: 9 | ENV: END_TO_END_TEST 10 | depends_on: 11 | - dnmonster 12 | - redis 13 | 14 | dnmonster: 15 | image: amouat/dnmonster:1.0 16 | 17 | redis: 18 | image: redis:3.0-alpine 19 | -------------------------------------------------------------------------------- /tests/integration/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | identidock_integration_tester: 6 | build: ../.. 7 | image: aroyd/identidock_integration_test:${IDENTIDOCK_INTEGRATION_TEST_IMAGE_TAG:-latest} 8 | environment: 9 | ENV: INTEGRATION_TEST 10 | depends_on: 11 | - dnmonster 12 | - redis 13 | 14 | dnmonster: 15 | image: amouat/dnmonster:1.0 16 | 17 | redis: 18 | image: redis:3.0-alpine 19 | -------------------------------------------------------------------------------- /tests/component/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6-alpine 2 | 3 | WORKDIR /component_test 4 | 5 | COPY requirements/requirements-dev.txt /component_test/ 6 | RUN pip install -r requirements-dev.txt 7 | RUN pip install requests==2.13.0 8 | 9 | COPY tests/component/test_component_identidock.py /component_test/ 10 | COPY scripts/wait-for-it.sh /usr/local/bin/ 11 | 12 | CMD ["wait-for-it.sh", "--host=identidock", "--port=5000", "--timeout=10", "--", "pytest", "-v", "-s", "test_component_identidock.py"] 13 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | identidock: 6 | build: . 7 | image: aroyd/identidock:${IDENTIDOCK_IMAGE_DEV_TAG:-latest-dev} 8 | ports: 9 | - "5000:5000" 10 | environment: 11 | ENV: DEV 12 | volumes: 13 | - ./ci_testing_python:/project/ci_testing_python 14 | depends_on: 15 | - dnmonster 16 | - redis 17 | 18 | dnmonster: 19 | image: amouat/dnmonster:1.0 20 | 21 | redis: 22 | image: redis:3.0-alpine 23 | -------------------------------------------------------------------------------- /tests/component/mountebank/dnmonster/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:7.7-alpine 2 | 3 | WORKDIR /mountebank 4 | 5 | RUN npm install -g mountebank 6 | 7 | COPY imposters.ejs /mountebank/ 8 | COPY mountebank_dnmonster_stub.ejs /mountebank/ 9 | COPY mountebank_dnmonster_response.ejs /mountebank/ 10 | 11 | # COPY mountebank_dnmonster_response_2.ejs /mountebank/ 12 | 13 | EXPOSE 2525 14 | EXPOSE 8080 15 | 16 | CMD ["mb", "--configfile", "/mountebank/imposters.ejs", "--allowInjection"] 17 | 18 | # CMD ["mb", "--allowInjection"] -------------------------------------------------------------------------------- /scripts/env: -------------------------------------------------------------------------------- 1 | export DOCKER_EMAIL= 2 | export DOCKER_REPO= 3 | export DOCKER_USER= 4 | export DOCKER_PASS= 5 | export MANAGER_COUNT= 6 | export MANAGER_1= 7 | export MANAGER_2= 8 | export MANAGER_3= 9 | export SERVER_USER= 10 | export JENKINS_SERVER= -------------------------------------------------------------------------------- /docs/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ========= 3 | 4 | 5 | It is a Sample **Microservice** Application written in **Python** Language for **CI-CD** and **Testing Purpose** using `flask `_, `pytest `_, `pytest-flask `_, `uber\'s test doubles package `_, `tox `_ with Dockerized environment and can used to learn, experiment with `docker `_, testing, pytest and have beginner\'s introduction/hands-on with CI servers like `Jenkins `_ and `Travis-CI `_. -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (https://tox.readthedocs.io/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py27, py34, py35, py36, pypy 8 | skipsdist=True 9 | 10 | [testenv] 11 | passenv = LANG 12 | setenv = 13 | TESTDIR = {toxinidir}/tests 14 | whitelist_externals = 15 | make 16 | /bin/bash 17 | deps = 18 | -rrequirements/requirements-main.txt 19 | -rrequirements/requirements-dev.txt 20 | pytest 21 | pytest-flask 22 | doubles 23 | commands = 24 | pip install --no-deps -e . 25 | # pytest -v -s tests 26 | pytest -v -s tests/unit 27 | pytest -v -s tests/integration 28 | -------------------------------------------------------------------------------- /scripts/copy-ssh-keys-to-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | echo "copy ssh keys to remote server" 6 | 7 | export FILE_NAME="$2_deploy_rsa.pub" 8 | echo "File name : $FILE_NAME" 9 | 10 | if [ -r "$1"/scripts/.env ]; then 11 | source "$1"/scripts/.env 12 | else 13 | echo ".env file not present, either export the values in the command line via export command or edit the values in this file itself" 14 | fi 15 | 16 | 17 | for manager in $(seq 0 $((MANAGER_COUNT-1))); 18 | do 19 | ( 20 | MGR=MANAGER_$((manager+1)) 21 | echo "[${!MGR}] Copying ssh public key..." 22 | ssh-copy-id -i $FILE_NAME $SERVER_USER@${!MGR} 23 | 24 | ) & 25 | 26 | done 27 | 28 | echo "Waiting for all the ssh public key copy processes to return" 29 | wait 30 | echo "Copying of ssh keys successful" -------------------------------------------------------------------------------- /tests/component/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | identidock_component_tester: 5 | build: 6 | context: ../.. 7 | dockerfile: tests/component/Dockerfile 8 | image: aroyd/identidock_component_tester:${IDENTIDOCK_COMPONENT_TESTER_IMAGE_TAG:-latest} 9 | depends_on: 10 | - identidock 11 | identidock: 12 | build: ../.. 13 | image: aroyd/identidock_component_test:${IDENTIDOCK_COMPONENT_TEST_IMAGE_TAG:-latest} 14 | environment: 15 | ENV: COMPONENT_TEST 16 | depends_on: 17 | - dnmonster 18 | - redis 19 | 20 | dnmonster: 21 | build: mountebank/dnmonster 22 | image: aroyd/dnmoster_mountebank_stub:${DNMONSTER_MOUNTEBANK_STUB_IMAGE_TAG:-latest} 23 | # image: amouat/dnmonster:1.0 24 | 25 | redis: 26 | image: redis:3.0-alpine 27 | -------------------------------------------------------------------------------- /docs/features.rst: -------------------------------------------------------------------------------- 1 | Features 2 | ========= 3 | 4 | 5 | Technical Specs 6 | ---------------- 7 | 8 | :python 3.6: Python Language (Cpython) 9 | :Flask: Micro Python Web Framework, good for microservice development and python WSGI apps. 10 | :pytest: Python testing library and test runner with awesome test discobery 11 | :pytest-flask: Pytest plugin for flask apps, to test fask apps using pytest library. 12 | :Uber\'s Test-Double: Test Double library for python, a good alternative to the `mock `_ library 13 | :Jenkins (Optional): A Self-hosted CI server 14 | :Travis-CI (Optional): A hosted CI server free for open-source projecs 15 | :Docker: A containerization tool for better devops 16 | 17 | 18 | 19 | Feature Specs 20 | -------------- 21 | 22 | 23 | * Web App 24 | * Microservice 25 | * Testing using Docker and Docker Compose 26 | * CI servers like Jenkins, Travis-CI 27 | 28 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6-alpine 2 | 3 | # RUN apk add --no-cache \ 4 | # pcre-dev 5 | # build-base 6 | # python3-dev 7 | 8 | RUN addgroup -S uwsgi && adduser -S -g uwsgi uwsgi 9 | 10 | WORKDIR /project 11 | 12 | COPY requirements/requirements-main.txt /project/ 13 | COPY requirements/requirements-dev.txt /project/ 14 | 15 | RUN set -e && \ 16 | apk add --no-cache --virtual .build-deps \ 17 | gcc \ 18 | libc-dev \ 19 | linux-headers \ 20 | && \ 21 | pip install -r requirements-main.txt && \ 22 | pip install -r requirements-dev.txt && \ 23 | apk del .build-deps 24 | 25 | COPY setup.py /project/ 26 | COPY setup.cfg /project/ 27 | COPY .coveragerc /project/ 28 | COPY tox.ini /project/ 29 | COPY pytest.ini /project/ 30 | COPY README.rst /project/ 31 | COPY docker-entrypoint.sh /usr/local/bin/ 32 | 33 | COPY ci_testing_python /project/ci_testing_python 34 | COPY tests /project/tests/ 35 | 36 | 37 | EXPOSE 5000 9090 9191 38 | 39 | CMD ["docker-entrypoint.sh"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Anirban Roy Das 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /scripts/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DOCKER="docker" 4 | DOCKER_COMPOSE="docker-compose" 5 | 6 | echo "CI SERVER : $1" 7 | echo "Deploy Environment: $2" 8 | 9 | if [ "$1" = "jenkins" ]; then 10 | DOCKER="sudo docker" 11 | DOCKER_COMPOSE="sudo docker-compose" 12 | COMMIT=${GIT_COMMIT::7} 13 | PULL_REQUEST=${GIT_PULL_REQUEST} 14 | BRANCH=${GIT_BRANCH} 15 | PROJECT_TAG=${GIT_TAG} 16 | BUILD_NUMBER=${BUILD_NUMBER} 17 | 18 | elif [ "$1" = "travis" ]; then 19 | DOCKER="docker" 20 | DOCKER_COMPOSE="docker-compose" 21 | COMMIT=${TRAVIS_COMMIT::7} 22 | PULL_REQUEST=${TRAVIS_PULL_REQUEST} 23 | BRANCH=${TRAVIS_BRANCH} 24 | PROJECT_TAG=${TRAVIS_TAG} 25 | BUILD_NUMBER=${TRAVIS_BUILD_NUMBER} 26 | fi 27 | 28 | DEPLOY_IDENTIDOCK_IMAGE_TAG=master-${COMMIT} 29 | 30 | 31 | SERVER_COUNT=0 32 | DEPLOY_SUCCESS=1 33 | SERVICE_NAME=cipython_identidock 34 | 35 | while [ "$DEPLOY_SUCCESS" -gt 0 ]; 36 | do 37 | SERVER_COUNT=$((SERVER_COUNT+1)) 38 | 39 | if [ $SERVER_COUNT -eq $((MANAGER_COUNT+1)) ]; then 40 | echo "Unable to update service in any of the managers" 41 | break 42 | fi 43 | 44 | SERVER="MANAGER_$SERVER_COUNT" 45 | ssh $SERVER_USER@${!SERVER} $DOCKER service update --image $DOCKER_REPO:$DEPLOY_IDENTIDOCK_IMAGE_TAG $SERVICE_NAME 46 | 47 | DEPLOY_SUCCESS=$(echo $?); 48 | echo "DEPLOY SUCCESS : $DEPLOY_SUCCESS" 49 | done 50 | -------------------------------------------------------------------------------- /scripts/jenkins-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | if [ -r "$1"/scripts/.env ]; then 6 | source "$1"/scripts/.env 7 | else 8 | echo ".env file not present, either export the values in the command line via export command or edit the values in this file itself" 9 | fi 10 | 11 | 12 | echo "Generating New Public/Private RSA Key Pair for use with Travis, skip the passpharase" 13 | ssh-keygen -t rsa -b 4096 -C 'jenkins@"$JENKINS_SERVER"' -f ./jenkins_deploy_rsa 14 | 15 | echo "Copying rsa public key to server" 16 | "$1"/scripts/copy-ssh-keys-to-server.sh "$1" jenkins 17 | 18 | echo "Note down the files generated, one public, one private. Public ssh keys are already \ 19 | copied to the servers so to have ssh access to them by your Jenkins server" 20 | 21 | echo "Copy the content of the private key file and paste it in jenkins ssh keys to give it access to the deploy servers" 22 | 23 | echo "Also, create credential secret text of all the environment variables mentioned in the env/.env file \ 24 | with appropriate values so as to refer them in the Jenkinsfile via withCredentials function" 25 | 26 | 27 | echo "Delete the secret file now. Have you noted the ids already? \ 28 | If not please do it. After you hit any key, the ssh key files will be deleted automatically for security reasons" 29 | read 30 | 31 | echo "Deleting the generated ssh keys" 32 | rm -f jenkins_deploy_rsa jenkins_deploy_rsa.pub -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # process related 2 | **/**/nohup.out 3 | **/*/nohup.out 4 | **/nohup.out 5 | */**/nohup.out 6 | */*/nohup.out 7 | */nohup.out 8 | nohup.out 9 | 10 | ############ Project Related ############### 11 | 12 | # SSH KEYS 13 | travis_deploy_rsa 14 | travis_deploy_rsa.pub 15 | jenkins_deploy_rsa 16 | jenkins_deploy_rsa.pub 17 | 18 | ############# Python ################### 19 | 20 | # Byte-compiled / optimized / DLL files 21 | __pycache__/ 22 | *.py[cod] 23 | *$py.class 24 | 25 | # C extensions 26 | *.so 27 | 28 | #Mac-OS-X_files 29 | .DS_Store 30 | .DS_S* 31 | 32 | 33 | # Distribution / packaging 34 | .Python 35 | env/ 36 | venv/ 37 | build/ 38 | develop-eggs/ 39 | dist/ 40 | downloads/ 41 | eggs/ 42 | .eggs/ 43 | lib/ 44 | lib64/ 45 | parts/ 46 | sdist/ 47 | var/ 48 | *.egg-info/ 49 | .installed.cfg 50 | *.egg 51 | 52 | # PyInstaller 53 | # Usually these files are written by a python script from a template 54 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 55 | *.manifest 56 | *.spec 57 | 58 | # Installer logs 59 | pip-log.txt 60 | pip-delete-this-directory.txt 61 | 62 | # Unit test / coverage reports 63 | htmlcov/ 64 | .tox/ 65 | .coverage 66 | .coverage.* 67 | .cache 68 | nosetests.xml 69 | coverage.xml 70 | *,cover 71 | 72 | # Translations 73 | *.mo 74 | *.pot 75 | 76 | # Django stuff: 77 | *.log 78 | 79 | 80 | # PyBuilder 81 | target/ 82 | 83 | #IPython Notebook 84 | .ipynb_checkpoints 85 | 86 | #pyenv 87 | .python-version 88 | 89 | # windows related 90 | *.bat 91 | 92 | # backup files 93 | *.bak 94 | 95 | -------------------------------------------------------------------------------- /ci_testing_python/app/identidock.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, Response, request 2 | import requests 3 | import hashlib 4 | import redis 5 | import html 6 | 7 | app = Flask(__name__) 8 | cache = redis.StrictRedis(host='redis', port=6379, db=0) 9 | salt = "UNIQUE_SALT" 10 | default_name = 'Joe Bloggs' 11 | 12 | 13 | 14 | @app.route('/', methods=['GET', 'POST']) 15 | def mainpage(): 16 | 17 | name = default_name 18 | if request.method == 'POST': 19 | name = html.escape(request.form['name'], quote=True) 20 | 21 | salted_name = salt + name 22 | name_hash = hashlib.sha256(salted_name.encode()).hexdigest() 23 | header = 'Identidock' 24 | body = '''
25 | Hello 26 | 27 |
28 |

You look like a: 29 | 30 | '''.format(name, name_hash) 31 | footer = '' 32 | 33 | return header + body + footer 34 | 35 | 36 | @app.route('/monster/') 37 | def get_identicon(name): 38 | 39 | name = html.escape(name, quote=True) 40 | image = cache.get(name) 41 | if image is None: 42 | print("Cache miss") 43 | r = requests.get('http://dnmonster:8080/monster/' + name + '?size=80') 44 | image = r.content 45 | cache.set(name, image) 46 | 47 | return Response(image, mimetype='image/png') 48 | 49 | 50 | 51 | if __name__ == '__main__': 52 | app.run(debug=True, host='0.0.0.0') 53 | -------------------------------------------------------------------------------- /docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | if [ "$ENV" = "DEV" ]; then 5 | echo "Running Development Application" 6 | pip install --no-deps -e . 7 | exec python ci_testing_python/app/identidock.py 8 | 9 | elif [ "$ENV" = "UNIT_TEST" ]; then 10 | echo "Running Unit Tests" 11 | pip install --no-deps -e . 12 | exec pytest -v -s --cov=./ci_testing_python tests/unit 13 | # exec tox -e unit 14 | 15 | elif [ "$ENV" = "CONTRACT_TEST" ]; then 16 | echo "Running Contract Tests" 17 | pip install --no-deps -e . 18 | exec pytest -v -s --cov=./ci_testing_python tests/contract 19 | # exec tox -e contract 20 | 21 | elif [ "$ENV" = "INTEGRATION_TEST" ]; then 22 | echo "Running Integration Tests" 23 | pip install --no-deps -e . 24 | exec pytest -v -s --cov=./ci_testing_python tests/integration 25 | # exec tox -e integration 26 | 27 | elif [ "$ENV" = "COMPONENT_TEST" ]; then 28 | echo "Running Component Tests" 29 | pip install --no-deps . 30 | exec python ci_testing_python/app/identidock.py 31 | 32 | elif [ "$ENV" = "END_TO_END_TEST" ]; then 33 | echo "Running End_To_End Tests" 34 | pip install --no-deps . 35 | exec uwsgi --http 0.0.0.0:9090 --wsgi-file ci_testing_python/app/identidock.py --callable app --stats 0.0.0.0:9191 --py-autoreload 1 36 | 37 | elif [ "$ENV" = "PROD" ]; then 38 | echo "Running Production Application" 39 | pip install --no-deps . 40 | exec uwsgi --http 0.0.0.0:9090 --wsgi-file ci_testing_python/app/identidock.py --callable app --stats 0.0.0.0:9191 --py-autoreload 1 41 | 42 | else 43 | echo "Please provide an environment" 44 | echo "Stopping" 45 | fi 46 | -------------------------------------------------------------------------------- /docs/ci_setup.rst: -------------------------------------------------------------------------------- 1 | CI Setup 2 | ========= 3 | 4 | 5 | If you are using the project in a CI setup (like travis, jenkins), then, on every push to github, you can set up your travis build or jenkins pipeline. Travis will use the ``.travis.yml`` file and Jenknis will use the ``Jenkinsfile`` to do their jobs. Now, in case you are using Travis, then run the Travis specific setup commands and for Jenkins run the Jenkins specific setup commands first. You can also use both to compare between there performance. 6 | 7 | The setup keys read the values from a ``.env`` file which has all the environment variables exported. But you will notice an example ``env`` file and not a ``.env`` file. Make sure to copy the ``env`` file to ``.env`` and **change/modify** the actual variables with your real values. 8 | 9 | The ``.env`` files are not commited to git since they are mentioned in the ``.gitignore`` file to prevent any leakage of confidential data. 10 | 11 | After you run the setup commands, you will be presented with a number of secure keys. Copy those to your config files before proceeding. 12 | 13 | **NOTE:** This is a one time setup. 14 | **NOTE:** Check the setup scripts inside the ``scripts/`` directory to understand what are the environment variables whose encrypted keys are provided. 15 | **NOTE:** Don't forget to **Copy** the secure keys to your ``.travis.yml`` or ``Jenkinsfile`` 16 | 17 | **NOTE:** If you don't want to do the copy of ``env`` to ``.env`` file and change the variable values in ``.env`` with your real values then you can just edit the ``travis-setup.sh`` or ``jenknis-setup.sh`` script and update the values their directly. The scripts are in the ``scripts/`` project level directory. 18 | 19 | 20 | **IMPORTANT:** You have to run the ``travis-setup.sh`` script or the ``jenkins-setup.sh`` script in your local machine before deploying to remote server. -------------------------------------------------------------------------------- /docker-compose.prod.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | identidock: 6 | image: ${DOCKER_REPO}:${IDENTIDOCK_IMAGE_PROD_TAG:-latest} 7 | ports: 8 | - "9090:9090" 9 | - "9091:9091" 10 | environment: 11 | ENV: PROD 12 | # env_file: docker-production.env 13 | labels: 14 | io.aroyd.service.container.name: "identidock" 15 | io.aroyd.servcie.container.env: "production" 16 | 17 | deploy: 18 | mode: replicated 19 | replicas: 4 20 | placement: 21 | constraints: 22 | - node.role == worker 23 | update_config: 24 | parallelism: 2 25 | delay: 30s 26 | failure_action: pause 27 | monitor: 20s 28 | restart_policy: 29 | condition: on-failure 30 | delay: 10s 31 | max_attempts: 5 32 | window: 120s 33 | labels: 34 | io.aroyd.service.task.name: "identidock" 35 | io.aroyd.service.task.env: "production" 36 | 37 | dnmonster: 38 | image: amouat/dnmonster:1.0 39 | 40 | redis: 41 | image: redis:3.2.8-alpine 42 | labels: 43 | io.aroyd.service.container.name: "redis" 44 | io.aroyd.servcie.container.env: "production" 45 | deploy: 46 | placement: 47 | constraints: 48 | - engine.labels.io.aroyd.machine.aws.az == b 49 | update_config: 50 | parallelism: 1 51 | delay: 10s 52 | failure_action: pause 53 | monitor: 10s 54 | restart_policy: 55 | condition: on-failure 56 | delay: 10s 57 | max_attempts: 5 58 | window: 120s 59 | volumes: 60 | - redis_data_volume:${REDIS_DATA_DIR:-/data} 61 | 62 | 63 | volumes: 64 | redis_data_volume: 65 | driver: rexray 66 | driver_opts: 67 | volumeType: "gp2" 68 | size: "4" 69 | availabilityZone: "ap-south-1b" 70 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. ci-testing-python documentation master file, created by 2 | sphinx-quickstart on Wed Apr 5 22:59:25 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to ci-testing-python's documentation! 7 | ============================================= 8 | 9 | .. image:: https://travis-ci.org/anirbanroydas/ci-testing-python.svg?branch=master 10 | :target: https://travis-ci.org/anirbanroydas/ci-testing-python 11 | 12 | .. image:: https://coveralls.io/repos/github/anirbanroydas/ci-testing-python/badge.svg?branch=master 13 | :target: https://coveralls.io/github/anirbanroydas/ci-testing-python 14 | 15 | .. image:: https://readthedocs.org/projects/ci-testing-python/badge/?version=latest 16 | :target: http://ci-testing-python.readthedocs.io/en/latest/?badge=latest 17 | 18 | 19 | 20 | 21 | Sample Microservice App in Python for CI-CD and Testing Purpose using flask, pytest, pytest-flask, uber\'s test doubles package, tox with Dockerized environment on CI servers like Jenkins and Travis CI both application in docker and jenkins in docker. 22 | 23 | 24 | 25 | Project Home Page 26 | -------------------- 27 | 28 | **Link :** https://github.com/anirbanroydas/ci-testing-python 29 | 30 | 31 | 32 | Details 33 | -------- 34 | 35 | 36 | :Author: Anirban Roy Das 37 | :Email: anirban.nick@gmail.com 38 | :Copyright(C): 2017, Anirban Roy Das 39 | 40 | Check ``ci-testing-python/LICENSE`` file for full Copyright notice. 41 | 42 | 43 | 44 | Contents: 45 | 46 | .. toctree:: 47 | :maxdepth: 2 48 | 49 | overview 50 | features 51 | installation 52 | ci_setup 53 | usage 54 | testing 55 | 56 | 57 | 58 | Indices and tables 59 | ================== 60 | 61 | * :ref:`genindex` 62 | * :ref:`modindex` 63 | * :ref:`search` 64 | 65 | -------------------------------------------------------------------------------- /tests/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RED='\033[0;31m' 4 | GREEN='\033[0;32m' 5 | NC='\033[0m' 6 | 7 | ARGS=("$@") 8 | 9 | 10 | DOCKER="docker" 11 | DOCKER_COMPOSE="docker-compose" 12 | 13 | echo "CI SERVER : $6" 14 | 15 | if [ "$6" = "jenkins" ]; then 16 | DOCKER="sudo docker" 17 | DOCKER_COMPOSE="sudo docker-compose" 18 | elif [ "$6" = "travis" ]; then 19 | DOCKER="docker" 20 | DOCKER_COMPOSE="docker-compose" 21 | fi 22 | 23 | 24 | 25 | cleanup () { 26 | # stop test containers 27 | $DOCKER_COMPOSE -p ${ARGS[2]} -f ${ARGS[3]}/docker-compose.yml stop 28 | 29 | # remove test containers 30 | $DOCKER_COMPOSE -p ${ARGS[2]} -f ${ARGS[3]}/docker-compose.yml rm --force -v 31 | 32 | # clean system with dangling images, containers, volumes 33 | echo "y" | $DOCKER system prune 34 | } 35 | 36 | 37 | trap 'cleanup ; printf "${RED}Tests Failed For Unexpected Reasons${NC}\n"' HUP INT QUIT PIPE TERM 38 | 39 | # build and run 40 | echo "Building and then Running Test Containers" 41 | $DOCKER_COMPOSE -p "$3" -f "$4"/docker-compose.yml build 42 | $DOCKER_COMPOSE -p "$3" -f "$4"/docker-compose.yml up -d 43 | 44 | if [ $? -ne 0 ] ; then 45 | printf "${RED}Docker Compose Failed${NC}\n" 46 | exit -1 47 | fi 48 | 49 | DOCKER_TEST_CONTAINER="$3_$1_$2_tester_1" 50 | TEST_EXIT_CODE=$($DOCKER wait "$DOCKER_TEST_CONTAINER") 51 | 52 | 53 | echo "Current dir : $5" 54 | echo "Copyting coverage report data file to project root directory only if Unit Test" 55 | if [ "$2" = "unit" ]; then 56 | $DOCKER cp "$DOCKER_TEST_CONTAINER":/project/.coverage "$5"/.coverage.unit_docker 57 | fi 58 | 59 | echo "Test Containers Logs" 60 | $DOCKER logs "$DOCKER_TEST_CONTAINER" 61 | 62 | if [ -z ${TEST_EXIT_CODE+x} ] || [ "$TEST_EXIT_CODE" -ne 0 ] ; then 63 | printf "${RED}Tests Failed${NC} - Exit Code: $TEST_EXIT_CODE\n" 64 | else 65 | printf "${GREEN}Tests Passed${NC}\n" 66 | fi 67 | 68 | # cleanup test setup and containers 69 | echo "Cleaning up test containers" 70 | cleanup 71 | 72 | exit $TEST_EXIT_CODE 73 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============= 3 | 4 | Prerequisites (Optional) 5 | ------------------------- 6 | 7 | To safegurad secret and confidential data leakage via your git commits to public github repo, check ``git-secrets``. 8 | 9 | This `git secrets `_ project helps in preventing secrete leakage by mistake. 10 | 11 | 12 | 13 | Dependencies 14 | ------------- 15 | 16 | 1. Docker 17 | 2. Make (Makefile) 18 | 19 | See, there are so many technologies used mentioned in the tech specs and yet the dependencies are just two. This is the power of Docker. 20 | 21 | 22 | 23 | Install 24 | ------- 25 | 26 | 27 | * **Step 1 - Install Docker** 28 | 29 | Follow my another github project, where everything related to DevOps and scripts are mentioned along with setting up a development environemt to use Docker is mentioned. 30 | 31 | * Project: https://github.com/anirbanroydas/DevOps 32 | 33 | * Go to setup directory and follow the setup instructions for your own platform, linux/macos 34 | 35 | * **Step 2 - Install Make** 36 | :: 37 | 38 | # (Mac Os) 39 | $ brew install automake 40 | 41 | # (Ubuntu) 42 | $ sudo apt-get update 43 | $ sudo apt-get install make 44 | 45 | * **Step 3 - Install Dependencies** 46 | 47 | Install the following dependencies on your local development machine which will be used in various scripts. 48 | 49 | 1. openssl 50 | 2. ssh-keygen 51 | 3. openssh 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | Travis Setup 60 | ~~~~~~~~~~~~~~~~~ 61 | 62 | These steps will encrypt your environment variables to secure your confidential data like api keys, docker based keys, deploy specific keys. 63 | :: 64 | 65 | $ make travis-setup 66 | 67 | 68 | 69 | Jenkins Setup 70 | ~~~~~~~~~~~~~~~~~~~ 71 | 72 | These steps will encrypt your environment variables to secure your confidential data like api keys, docker based keys, deploy specific keys. 73 | :: 74 | 75 | $ make jenkins-setup 76 | 77 | -------------------------------------------------------------------------------- /scripts/travis-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | if [ -r "$1"/scripts/.env ]; then 6 | source "$1"/scripts/.env 7 | else 8 | echo ".env file not present, either export the values in the command line via export command or edit the values in this file itself" 9 | fi 10 | 11 | 12 | echo "Logging into Travis via travis cli" 13 | travis login 14 | 15 | echo "DOCKER_EMAIL secure key: copy and add to .travis.yml env section" 16 | travis encrypt DOCKER_EMAIL=${DOCKER_EMAIL} 17 | 18 | echo "DOCKER_REPO secure key: copy and add to .travis.yml env section" 19 | travis encrypt DOCKER_REPO=${DOCKER_REPO} 20 | 21 | echo "DOCKER_USER secure key:: copy and add to .travis.yml env section" 22 | travis encrypt DOCKER_USER=${DOCKER_USER} 23 | 24 | echo "DOCKER_PASS secure key:: copy and add to .travis.yml env section" 25 | travis encrypt DOCKER_PASS=${DOCKER_PASS} 26 | 27 | echo "MANAGER_COUNT secure key:: copy and add to .travis.yml env section" 28 | travis encrypt MANAGER_COUNT=${MANAGER_COUNT} 29 | 30 | echo "MANAGER_1 secure key:: copy and add to .travis.yml env section" 31 | travis encrypt MANAGER_1=${MANAGER_1} 32 | 33 | echo "MANAGER_2 secure key:: copy and add to .travis.yml env section" 34 | travis encrypt MANAGER_2=${MANAGER_2} 35 | 36 | echo "MANAGER_3 secure key:: copy and add to .travis.yml env section" 37 | travis encrypt MANAGER_3=${MANAGER_3} 38 | 39 | echo "SERVER_USER secure key:: copy and add to .travis.yml env section" 40 | travis encrypt SERVER_USER=${SERVER_USER} 41 | 42 | 43 | echo "Generating New Public/Private RSA Key Pair for use with Travis, skip the passpharase" 44 | ssh-keygen -t rsa -b 4096 -C 'anirbanroyds/ci-testing-python@travis-ci.org' -f ./travis_deploy_rsa 45 | 46 | echo "Encrypting rsa private key file, copy the decryption command and add to .travis.yml" 47 | travis encrypt-file travis_deploy_rsa 48 | 49 | echo "Copying rsa public key to server" 50 | "$1"/scripts/copy-ssh-keys-to-server.sh "$1" travis 51 | 52 | echo "Deleting the generated ssh keys" 53 | rm -f travis_deploy_rsa travis_deploy_rsa.pub 54 | 55 | -------------------------------------------------------------------------------- /tests/component/mountebank/dnmonster/mountebank_dnmonster_stub.ejs: -------------------------------------------------------------------------------- 1 | { 2 | "port": 8080, 3 | "protocol": "http", 4 | "name": "dnmonster", 5 | "stubs": [ 6 | { 7 | "responses": [ 8 | { 9 | "inject": "<%- stringify(filename, 'mountebank_dnmonster_response.ejs') %>" 10 | } 11 | ], 12 | "predicates": [ 13 | { 14 | "equals": { 15 | "method": "GET", 16 | "path": "/monster/ABCDEF123456789", 17 | "query": { 18 | "size": "80" 19 | } 20 | } 21 | 22 | } 23 | ] 24 | }, 25 | { 26 | "responses": [ 27 | { 28 | "inject": "<%- stringify(filename, 'mountebank_dnmonster_response.ejs') %>" 29 | } 30 | ], 31 | "predicates": [ 32 | { 33 | "matches": { 34 | "method": "GET", 35 | "path": "^/monster/[A-Za-z0-9]+$", 36 | "query": { 37 | "size": "80" 38 | } 39 | } 40 | } 41 | ] 42 | }, 43 | { 44 | "responses": [ 45 | { 46 | "is": { "statusCode": 404 } 47 | } 48 | ], 49 | "predicates": [ 50 | { 51 | "not" : { 52 | "matches": { 53 | "method": "GET", 54 | "path": "^/monster/[A-Za-z0-9]+$", 55 | "query": { 56 | "size": "80" 57 | } 58 | } 59 | } 60 | } 61 | ] 62 | } 63 | ] 64 | } -------------------------------------------------------------------------------- /tests/component/mountebank/dnmonster/mountebank_dnmonster_response_2.ejs: -------------------------------------------------------------------------------- 1 | // Sample Imosters, Refer : http://www.mbtest.org/docs/api/injection 2 | 3 | 4 | function (request, state, logger) { 5 | logger.info('origin called'); 6 | state.requests = state.requests || 0; 7 | state.requests += 1; 8 | return { 9 | headers: { 10 | 'Content-Type': 'application/json' 11 | }, 12 | body: JSON.stringify({ count: state.requests }) 13 | }; 14 | } 15 | 16 | 17 | function (request, state) { 18 | var count = state.requests ? Object.keys(state.requests).length : 0, 19 | util = require('util'); 20 | 21 | return { 22 | body: util.format('There have been %s proxied calls', count) 23 | }; 24 | } 25 | 26 | 27 | function (request, state, logger, callback) { 28 | var cacheKey = request.method + ' ' + request.path; 29 | 30 | if (typeof state.requests === 'undefined') { 31 | state.requests = {}; 32 | } 33 | 34 | if (state.requests[cacheKey]) { 35 | logger.info('Using previous response'); 36 | callback(state.requests[cacheKey]); 37 | } 38 | 39 | var http = require('http'), 40 | options = { 41 | method: request.method, 42 | hostname: 'localhost', 43 | port: 5555, 44 | path: request.path, 45 | headers: request.headers 46 | }, 47 | httpRequest = http.request(options, function (response) { 48 | var body = ''; 49 | response.setEncoding('utf8'); 50 | response.on('data', function (chunk) { 51 | body += chunk; 52 | }); 53 | response.on('end', function () { 54 | var stubResponse = { 55 | statusCode: response.statusCode, 56 | headers: response.headers, 57 | body: body 58 | }; 59 | logger.info('Successfully proxied: ' + JSON.stringify(stubResponse)); 60 | state.requests[cacheKey] = stubResponse; 61 | callback(stubResponse); 62 | }); 63 | }); 64 | httpRequest.end(); 65 | } -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | After having installed the above dependencies, and ran the **Optional** (If not using any CI Server) or **Required** (If using any CI Server) **CI Setup** Step, then just run the following commands to use it: 5 | 6 | 7 | You can run and test the app in your local development machine or you can run and test directly in a remote machine. You can also run and test in a production environment. 8 | 9 | 10 | Run 11 | ---- 12 | 13 | 14 | The below commands will start everythin in development environment. To start in a production environment, suffix ``-prod`` to every **make** command. 15 | 16 | For example, if the normal command is ``make start``, then for production environment, use ``make start-prod``. Do this modification to each command you want to run in production environment. 17 | 18 | **Exceptions:** You cannot use the above method for test commands, test commands are same for every environment. Also the ``make system-prune`` command is standalone with no production specific variation (Remains same in all environments). 19 | 20 | * **Start Applcation** 21 | :: 22 | 23 | $ make clean 24 | $ make build 25 | $ make start 26 | 27 | # OR 28 | 29 | $ docker-compose up -d 30 | 31 | 32 | 33 | 34 | * **Stop Application** 35 | :: 36 | 37 | $ make stop 38 | 39 | # OR 40 | 41 | $ docker-compose stop 42 | 43 | 44 | * **Remove and Clean Application** 45 | :: 46 | 47 | $ make clean 48 | 49 | # OR 50 | 51 | $ docker-compose rm --force -v 52 | $ echo "y" | docker system prune 53 | 54 | 55 | * **Clean System** 56 | :: 57 | 58 | $ make system-prune 59 | 60 | # OR 61 | 62 | $ echo "y" | docker system prune 63 | 64 | 65 | 66 | 67 | Logging 68 | -------- 69 | 70 | 71 | * To check the whole application Logs 72 | :: 73 | 74 | $ make check-logs 75 | 76 | # OR 77 | 78 | $ docker-compose logs --follow --tail=10 79 | 80 | 81 | 82 | * To check just the python app\'s logs 83 | :: 84 | 85 | $ make check-logs-app 86 | 87 | # OR 88 | 89 | $ docker-compose logs --follow --tail=10 identidock 90 | 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /tests/component/test_component_identidock.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import requests 3 | # from time import sleep 4 | 5 | COMPONENT_INDEX_URL = "http://identidock:5000" 6 | 7 | COMPONENT_MONSTER_BASE_URL = COMPONENT_INDEX_URL + '/monster' 8 | 9 | 10 | 11 | def test_get_mainpage(): 12 | # print('component tester sleeping for 1 sec to let the identidock app to be ready adn also start its server') 13 | # sleep(1) 14 | page = requests.get(COMPONENT_INDEX_URL) 15 | assert page.status_code == 200 16 | assert 'Joe Bloggs' in str(page.text) 17 | 18 | 19 | 20 | 21 | def test_post_mainpage(): 22 | page = requests.post(COMPONENT_INDEX_URL, data=dict(name="Moby Dock")) 23 | assert page.status_code == 200 24 | assert 'Moby Dock' in str(page.text) 25 | 26 | 27 | 28 | 29 | def test_mainpage_html_escaping(): 30 | page = requests.post(COMPONENT_INDEX_URL, data=dict(name='">TEST