├── {{ cookiecutter.service }} ├── data │ ├── .gitkeep │ ├── docs │ │ └── .gitkeep │ ├── logs │ │ └── .gitkeep │ ├── test │ │ ├── .gitkeep │ │ └── temp │ │ │ └── .gitkeep │ └── README.md ├── server │ ├── tests │ │ ├── .gitkeep │ │ └── test_unit_example.py │ ├── client │ │ └── .gitkeep │ ├── router │ │ ├── auth.py │ │ └── service.py │ ├── config.py │ ├── server.py │ └── asserts.py ├── deployments │ ├── .envs │ │ ├── .gitkeep │ │ ├── test.env │ │ └── local.env │ ├── .secrets │ │ ├── .gitkeep │ │ ├── daemon.json │ │ ├── daemon.json_mixed │ │ ├── pip_private.conf │ │ ├── .pypirc │ │ └── .pypirc_mixed │ ├── docker-compose.env.yml │ ├── docker-compose.full.yml │ ├── Dockerfile │ └── README.md ├── requirements.private ├── {{ cookiecutter.python_package }} │ ├── __init__.py │ ├── README.md │ ├── generators │ │ └── README.md │ └── methods.py ├── docs │ ├── service.md │ ├── swagger.png │ ├── IDE_workdir.png │ ├── tests.md │ ├── commands.md │ ├── structure.md │ └── errors.md ├── info.py ├── requirements ├── tests │ ├── README.md │ ├── test_health.py │ └── __init__.py ├── .editconfig ├── scripts │ └── variable.py ├── setup.py ├── http_.py ├── .gitignore ├── makefile └── README.md ├── docs ├── structure.png ├── structure.drawio └── tutorial.md ├── cookiecutter.json ├── .gitignore └── README.md /{{ cookiecutter.service }}/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/data/docs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/data/logs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/server/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/data/test/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/.envs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/requirements.private: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/data/test/temp/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/.secrets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/server/client/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/{{ cookiecutter.python_package }}/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/docker-compose.env.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/server/tests/test_unit_example.py: -------------------------------------------------------------------------------- 1 | def test_example(): 2 | assert True 3 | -------------------------------------------------------------------------------- /docs/structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/U-Company/python-private-service-layout/HEAD/docs/structure.png -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/docs/service.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.service }} 2 | 3 | {{ cookiecutter.description }} 4 | 5 | This is full service description -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/docs/swagger.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/U-Company/python-private-service-layout/HEAD/{{ cookiecutter.service }}/docs/swagger.png -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/.secrets/daemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "insecure-registries" : [ 3 | "{{ cookiecutter.docker_registry }}" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/docs/IDE_workdir.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/U-Company/python-private-service-layout/HEAD/{{ cookiecutter.service }}/docs/IDE_workdir.png -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/.secrets/daemon.json_mixed: -------------------------------------------------------------------------------- 1 | { 2 | "insecure-registries" : [ 3 | "{{ cookiecutter.docker_registry }}", 4 | "://:" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/{{ cookiecutter.python_package }}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.service }} 2 | 3 | This module consists: 4 | 5 | - **generators** -- for data generation of handlers 6 | - **methods.py** -- public methods of client 7 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/info.py: -------------------------------------------------------------------------------- 1 | name = '{{ cookiecutter.python_package }}' 2 | version = '0.1.0' 3 | description = '{{ cookiecutter.description.replace("'", "\'") }}' 4 | author = '{{ cookiecutter.author }}' 5 | email = '{{ cookiecutter.email }}' 6 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/requirements: -------------------------------------------------------------------------------- 1 | loguru>=0.5.1 2 | fastapi>=0.61.0 3 | uvicorn>=0.11.8 4 | vault-client>=0.3.4 5 | prometheus_client>=0.8.0 6 | python-clients>=1.0.3 7 | faker>=4.1.2 8 | pytest-asyncio>=0.14.0 9 | pytest>=6.0.1 10 | msgpack>=1.0.0 11 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/{{ cookiecutter.python_package }}/generators/README.md: -------------------------------------------------------------------------------- 1 | # Generators 2 | 3 | Each service works with data. Therefore a good practice is build generators for data creation of each method. This generators is part of package for using in tests of other packages. 4 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/.secrets/pip_private.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | timeout = 3 3 | retries = 0 4 | extra-index-url = {{ cookiecutter.pypi_schema }}://{{ cookiecutter.pypi_login}}:{{cookiecutter.pypi_password}}@{{ cookiecutter.pypi_host }}:{{ cookiecutter.pypi_port }} 5 | trusted-host = {{ cookiecutter.pypi_host }} 6 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/docker-compose.full.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | {{ cookiecutter.service }}: 4 | env_file: 5 | - ../${VAULT_ENV_FILE} 6 | build: 7 | context: .. 8 | dockerfile: deployments/Dockerfile 9 | depends_on: [] 10 | network_mode: host 11 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/tests/README.md: -------------------------------------------------------------------------------- 1 | # Tests 2 | 3 | This directory consists only integration tests. In this directory cannot import files from another directories except 4 | package directory (python_service_layout). We tests our service as block box method. Therefore we cannot use common 5 | functions or scripts with tested sources. 6 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/.secrets/.pypirc: -------------------------------------------------------------------------------- 1 | [distutils] 2 | index-servers= 3 | {{ cookiecutter.pypi_alias }} 4 | 5 | [{{ cookiecutter.pypi_alias }}] 6 | repository: {{ cookiecutter.pypi_schema }}://{{cookiecutter.pypi_host}}:{{cookiecutter.pypi_port}} 7 | username: {{ cookiecutter.pypi_login }} 8 | password: {{ cookiecutter.pypi_password }} 9 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/tests/test_health.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from {{ cookiecutter.python_package }} import methods 4 | import tests 5 | 6 | 7 | @pytest.mark.asyncio 8 | async def test_health(): 9 | method = methods.Health(api_key=tests.fpredictor_api_key) 10 | client = tests.client() 11 | resp, status = await client.request(method) 12 | assert status == 204 13 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/.envs/test.env: -------------------------------------------------------------------------------- 1 | {{ cookiecutter.python_package.upper() }}_SCHEMA=http 2 | {{ cookiecutter.python_package.upper() }}_HOST=0.0.0.0 3 | {{ cookiecutter.python_package.upper() }}_PORT=8080 4 | {{ cookiecutter.python_package.upper() }}_PROMETHEUS_PORT=9090 5 | {{ cookiecutter.python_package.upper() }}_API_KEY=1234567890abcdefg 6 | {{ cookiecutter.python_package.upper() }}_WORKERS=2 7 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/.envs/local.env: -------------------------------------------------------------------------------- 1 | {{ cookiecutter.python_package.upper() }}_SCHEMA=http 2 | {{ cookiecutter.python_package.upper() }}_HOST=0.0.0.0 3 | {{ cookiecutter.python_package.upper() }}_PORT=8080 4 | {{ cookiecutter.python_package.upper() }}_PROMETHEUS_PORT=9090 5 | {{ cookiecutter.python_package.upper() }}_API_KEY=1234567890ABCDEFG 6 | {{ cookiecutter.python_package.upper() }}_WORKERS=3 7 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/data/README.md: -------------------------------------------------------------------------------- 1 | # data 2 | 3 | There are some files and folders: 4 | 5 | - `docs` folder for docs (README import, images and other files) 6 | - `logs` folder for logs (this folder put to `.gitignore`. All files here are ignoring) 7 | - `test` files for test. This files can be push into repo 8 | - `test/temp` this is temporary files. (this folder put to `.gitignore`. All files here are ignoring) 9 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/.editconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = tab 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [Makefile] 18 | indent_style = tab 19 | 20 | [*.{diff,patch}] 21 | trim_trailing_whitespace = false 22 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/.secrets/.pypirc_mixed: -------------------------------------------------------------------------------- 1 | [distutils] 2 | index-servers= 3 | pypi 4 | {{ cookiecutter.pypi_alias }} 5 | 6 | [{{ cookiecutter.pypi_alias }}] 7 | repository: {{ cookiecutter.pypi_schema }}://{{ cookiecutter.pypi_host }}:{{ cookiecutter.pypi_port }} 8 | username: {{ cookiecutter.pypi_login }} 9 | password: {{ cookiecutter.pypi_password }} 10 | 11 | [pypi] 12 | repository: https://upload.pypi.org/legacy/ 13 | username: 14 | password: 15 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 AS BUILD 2 | 3 | RUN apt-get update && apt-get -y install make python3.7 python3-pip git 4 | RUN python3.7 -m pip install --upgrade pip 5 | 6 | COPY . /app 7 | WORKDIR /app 8 | 9 | ENV PIP_CONFIG_FILE /app/deployments/.secrets/pip_private.conf 10 | ENV VAULT_ENV ${VAULT_ENV} 11 | 12 | RUN PIP=pip PYTHON=python3.7 make clean 13 | RUN PIP=pip PYTHON=python3.7 make deps 14 | CMD PIP=pip PYTHON=python3.7 VAULT_ENV=${VAULT_ENV} make run 15 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/scripts/variable.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from vault_client.client import VaultClient 5 | 6 | 7 | variable = sys.argv[1] 8 | 9 | VAULT_ENV = os.environ.get('VAULT_ENV', 'LOCAL') 10 | VAULT_ENV_FILE = os.environ.get('VAULT_ENV_FILE', 'deployments/.envs/local.env') 11 | vault_client = VaultClient(environ=VAULT_ENV, env_file=VAULT_ENV_FILE) 12 | 13 | assert vault_client.is_authenticated, 'Vault client not authenticated' 14 | assert vault_client.is_initialized, 'Vault client not initialized' 15 | assert not vault_client.is_sealed, 'Vault client sealed' 16 | 17 | namespace = '{{ cookiecutter.python_package.upper() }}' 18 | print(vault_client.get(namespace, variable.upper())) 19 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/docs/tests.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | If we integration-test, we have some stages: 4 | 5 | * Clean storage 6 | * Data generation 7 | * Create objects 8 | * Testing 9 | * Validation 10 | * Clean storage 11 | 12 | If we unit-test, we have some stages: 13 | 14 | * Create objects 15 | * Testing 16 | * Validation 17 | 18 | In this way unit-test is partial case of integration test. We use unit-test for testing 19 | logic in context one object, function (method). For example, we have function: 20 | 21 | def plus(a, b): 22 | return a + b 23 | 24 | This is unit-test. We want to test simple object. This situation is not always possible. 25 | If we want communication some systems and modules, we deal with integration tests. 26 | For example, we test connect to data base and collect data from there. This is 27 | integration-tests. 28 | -------------------------------------------------------------------------------- /cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": null, 3 | "email": null, 4 | "description": null, 5 | "service": null, 6 | "python_package": "{{cookiecutter.service.lower().replace(' ', '_').replace('-', '_')}}", 7 | 8 | "pypi_schema": "http", 9 | "pypi_host": null, 10 | "pypi_port": null, 11 | "pypi_registry": "{{ cookiecutter.pypi_schema + '://' + cookiecutter.pypi_host + ':' + cookiecutter.pypi_port }}", 12 | "pypi_alias": "{{ cookiecutter.python_package }}", 13 | "pypi_login": null, 14 | "pypi_password": null, 15 | 16 | "docker_schema": "http", 17 | "docker_host": null, 18 | "docker_port": null, 19 | "docker_registry": "{{ cookiecutter.docker_schema + '://' + cookiecutter.docker_host + ':' + cookiecutter.docker_port}}", 20 | "docker_image": "{{ cookiecutter.docker_host + ':' + cookiecutter.docker_port + '/' + cookiecutter.service }}", 21 | "docker_login": null, 22 | "docker_password": null 23 | } 24 | 25 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/deployments/README.md: -------------------------------------------------------------------------------- 1 | # /deployments 2 | 3 | You can deploy applications with different docker-compose files and different environment files. Our docker files are built by a make file, which uses as API interface for commutication building of any program. 4 | 5 | Type of docker files: 6 | 7 | * **docker-compose.env.yml** -- environment for your application. For example, you have some databases and some services, but you want to run your application from source (for debugging, for instance). Then, you can runnig only this file. 8 | * **docker-compose.full.yml** -- full application with environment 9 | 10 | For each docker-compose we have environment file into `./.envs` directory. 11 | 12 | **Notice**: we configure compose with `network_mode: host` (in) therefore, our image links with localhost (no mapping ports, for example) 13 | 14 | More about deployments you can read in this [notice](https://github.com/U-Company/notes/tree/master/deployments) 15 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | import info 4 | 5 | 6 | def parse_requirements(filename): 7 | """ load requirements from a pip requirements file """ 8 | lineiter = (line.strip() for line in open(filename)) 9 | return [line for line in lineiter if line and not line.startswith("#")] 10 | 11 | 12 | with open("README.md", "r") as fh: 13 | long_description = fh.read() 14 | 15 | 16 | install_reqs = parse_requirements('./requirements') 17 | print(install_reqs) 18 | 19 | setuptools.setup( 20 | name=info.name, 21 | version=info.version, 22 | author=info.author, 23 | author_email=info.email, 24 | description=info.description, 25 | long_description=long_description, 26 | long_description_content_type="text/markdown", 27 | url=f'https://github.com/Hedgehogues/{info.name}', 28 | packages=setuptools.find_packages(exclude=['tests', 'service']), 29 | classifiers=[ 30 | "Programming Language :: Python :: 3.7", 31 | "Operating System :: OS Independent", 32 | ], 33 | install_requires=install_reqs, 34 | ) 35 | -------------------------------------------------------------------------------- /docs/structure.drawio: -------------------------------------------------------------------------------- 1 | 7Vlbk5owGP01Pu4MIcvtUdG1fdDp1Gn72IkQgW4kbIhV++sbJNw26OpUVtzpk3i+XPhOTs6X6AC6692UoSScUR+Tga75uwEcD3QdQMMQHxmyzxFLl0DAIl82qoBF9AdLUJPoJvJx2mjIKSU8SpqgR+MYe7yBIcbottlsRUlz1gQFWAEWHiIq+iPyeZijtm5V+CccBWExMzCdPLJGRWOZSRoin25rEJwMoMso5fnTeudikpFX8JL3ezoSLV+M4Zif02Fqz77P5iPLwAT8pPOX5WLOH4o8fiOykRnLt+X7goKA0U0im2HG8a6NeLQsmmvqi4EyXaETTNeYs71oIiUBLNllWxEMQDFOWGMXQgkiuapBOViVuHiQuV/AA2ihwSRi2tGKiix0M8iem1iNI/NlQ4vAQ3oQ8VA00O1kVwWLUb4w+iuTqhxMvG/bHEv2H/lXRKH2lbKFsGMfZ7rQRHgbRhwvEuRl0a2wM4GFfC10NAZl75YdUFf6iU2m6v+Weofn7ns4Sjmjz9ilhLJDAJqejZerTO4RITV8PJzYT+4pqs4xi4solMMX40hHcVRDKams8wucrvgtqly3vnpiaU+q7Z3JAPob7nqGk+qtToq856x+d7HdL9b22xo2ercwmrIwM8xD6qdX5W7kTLShIRdVHvEyJjrgsvSDpiGU7lq33DbHNbui2lGYnuIYM8Qpu3eyQd+4Bo8fqrhBx+hbdTNvV90ee2eiaqlfYCZy7qb+XHFbm72jUtXVtzjKDggcp7z/PnmE0N4WJaDe/r/SDcd3UJJOU92/kmQrTA/TVCT+0Zi+NdG6WvtdEokU751oxT1uzTRQj7RHDgENloTn3OAMZVlN9npwhlLvXte5FH+OOQ7ExSKicVU47+KCnCvqfQ4k4mv10/shVvsDA07+Ag== -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/server/router/auth.py: -------------------------------------------------------------------------------- 1 | import fastapi 2 | from starlette.status import HTTP_403_FORBIDDEN 3 | 4 | from server import config 5 | 6 | 7 | async def get_api_key( 8 | api_key_query: str = fastapi.Security(config.api_key_query), 9 | api_key_header: str = fastapi.Security(config.api_key_header), 10 | api_key_cookie: str = fastapi.Security(config.api_key_cookie), 11 | ): 12 | """ 13 | get_api_key is middleware for check authorization of methods 14 | :param api_key_query: query param authorization 15 | :param api_key_header: header authorization 16 | :param api_key_cookie: cookie authorization 17 | :return: 18 | """ 19 | if api_key_query == config.{{ cookiecutter.python_package }}_api_key: 20 | return api_key_query 21 | elif api_key_header == config.{{ cookiecutter.python_package }}_api_key: 22 | return api_key_header 23 | elif api_key_cookie == config.{{ cookiecutter.python_package }}_api_key: 24 | return api_key_cookie 25 | else: 26 | # TODO: здесь нужно добавить стандартный assertor с контекстами 27 | raise fastapi.HTTPException(status_code=HTTP_403_FORBIDDEN, detail='Could not validate credentials') 28 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from clients import http 4 | from vault_client.client import VaultClient 5 | 6 | import info 7 | 8 | 9 | VAULT_ENV = os.environ.get('VAULT_ENV', 'LOCAL') 10 | VAULT_ENV_FILE = os.environ.get('VAULT_ENV_FILE', 'deployments/.envs/test.env') 11 | client = VaultClient(environ=VAULT_ENV, env_file=VAULT_ENV_FILE) 12 | 13 | assert client.is_authenticated, 'Vault client not authenticated' 14 | assert client.is_initialized, 'Vault client not initialized' 15 | assert not client.is_sealed, 'Vault client sealed' 16 | 17 | namespace = info.name.upper() 18 | {{ cookiecutter.python_package }}_host = client.get(namespace, 'HOST') 19 | {{ cookiecutter.python_package }}_port = client.get(namespace, 'PORT') 20 | {{ cookiecutter.python_package }}_schema = client.get(namespace, 'SCHEMA') 21 | {{ cookiecutter.python_package }}_token = client.get(namespace, 'API_KEY') 22 | 23 | {{ cookiecutter.python_package }}_backend_url = {{ cookiecutter.python_package }}_schema + '://' + {{ cookiecutter.python_package }}_host + ':' + {{ cookiecutter.python_package }}_port 24 | 25 | 26 | def client(): 27 | """ 28 | client_storage creates individual client for each test. This is very famous for aiohttp event loop in tests 29 | :return: 30 | """ 31 | return http.AsyncClient({{ cookiecutter.python_package }}_backend_url) 32 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/docs/commands.md: -------------------------------------------------------------------------------- 1 | ## Commands 2 | 3 | Create conda environment 4 | 5 | make config 6 | 7 | Build python docker container. Package is built while publishing 8 | 9 | make build-image 10 | 11 | Publish docker image. You can get docker container tag from console, after running `make build-image` 12 | 13 | VERSION=a.b.c TAG= make publish-image 14 | 15 | Publish python package 16 | 17 | make publish-package 18 | 19 | Clean source of python package after building and all temporary files 20 | 21 | make clean 22 | 23 | Install all packages dependencies. We suppose that you have not more two registry: [public PyPi-registry](https://pypi.org/project/registry/) and maybe your private pypi-registry (optional). This command install from both or only public 24 | 25 | make deps 26 | 27 | Run service in operation system 28 | 29 | make run 30 | 31 | Run service in docker with environment services 32 | 33 | make run-full 34 | 35 | Run service in docker with environment services 36 | 37 | make run-env 38 | 39 | Rebuild docker container 40 | 41 | make run-rebuild 42 | 43 | Run integration tests (you must run service and environments before running tests: `TEST=yes make run-full`): 44 | 45 | make test-integration 46 | 47 | Run unit tests 48 | 49 | make test-unit 50 | 51 | Run all tests 52 | 53 | make test 54 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/server/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from fastapi import security 4 | from vault_client.client import VaultClient 5 | 6 | import info 7 | 8 | 9 | VAULT_ENV = os.environ.get('VAULT_ENV', 'LOCAL') 10 | VAULT_ENV_FILE = os.environ.get('VAULT_ENV_FILE', 'deployments/.envs/local.env') 11 | vault_client = VaultClient(environ=VAULT_ENV, env_file=VAULT_ENV_FILE) 12 | 13 | assert vault_client.is_authenticated, 'Vault client not authenticated' 14 | assert vault_client.is_initialized, 'Vault client not initialized' 15 | assert not vault_client.is_sealed, 'Vault client sealed' 16 | 17 | api_key_name = 'access_token' 18 | 19 | namespace = info.name.upper() 20 | {{ cookiecutter.python_package }}_schema = vault_client.get(namespace, 'SCHEMA') 21 | {{ cookiecutter.python_package }}_host = vault_client.get(namespace, 'HOST') 22 | {{ cookiecutter.python_package }}_port = int(vault_client.get(namespace, 'PORT')) 23 | {{ cookiecutter.python_package }}_api_key = vault_client.get(namespace, 'API_KEY') 24 | prometheus_port = int(vault_client.get(namespace, 'PROMETHEUS_PORT')) 25 | allow_origins = vault_client.get(namespace, 'ALLOW_ORIGINS') 26 | if VAULT_ENV == 'LOCAL' and allow_origins is None: 27 | allow_origins = ['*'] 28 | 29 | 30 | api_key_query = security.APIKeyQuery(name=api_key_name, auto_error=False) 31 | api_key_header = security.APIKeyHeader(name=api_key_name, auto_error=False) 32 | api_key_cookie = security.APIKeyCookie(name=api_key_name, auto_error=False) 33 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/server/server.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | import fastapi 4 | from starlette.middleware import cors 5 | 6 | import info 7 | 8 | 9 | class ThreadMutex: 10 | def __init__(self, message): 11 | """ 12 | This is server mutex for handlers. This object can block event loop from another requests. Careful about 13 | synchronization, if you use nginx 14 | 15 | :param message: message of exception 16 | """ 17 | self.__flag = False 18 | self.__mutex = threading.Lock() 19 | self.__message = message 20 | 21 | def check(self): 22 | if self.__flag: 23 | raise fastapi.HTTPException(409, detail=self.__message) 24 | 25 | def acquire(self): 26 | self.__mutex.acquire() 27 | if self.__flag: 28 | self.__flag = False 29 | self.__mutex.release() 30 | raise fastapi.HTTPException(409, detail=self.__message) 31 | self.__flag = True 32 | 33 | def release(self): 34 | self.__flag = False 35 | self.__mutex.release() 36 | 37 | 38 | def check_mutex(l): 39 | """ 40 | :param l: list of mutex for check 41 | :return: 42 | """ 43 | for m in l: 44 | m.check() 45 | 46 | 47 | def service_name(): 48 | return ' '.join(info.name.split('_')) 49 | 50 | 51 | def build_app(allow_origins): 52 | app = fastapi.FastAPI( 53 | version=info.version, title=service_name(), docs_url=None, redoc_url=None, openapi_url=None, 54 | ) 55 | app.add_middleware( 56 | cors.CORSMiddleware, 57 | allow_origins=allow_origins, 58 | allow_credentials=True, 59 | allow_methods=["*"], 60 | allow_headers=["*"] 61 | ) 62 | return app 63 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/{{ cookiecutter.python_package }}/methods.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | from clients import http 4 | 5 | 6 | api_key_name = 'access_token' 7 | 8 | 9 | class Field(enum.Enum): 10 | Cookie = 'cookie' 11 | Header = 'header' 12 | QueryParam = 'query_param' 13 | 14 | 15 | class CookieNotImplemented(Exception): 16 | pass 17 | 18 | 19 | class APIKey(http.Method): 20 | headers = { 21 | 'Content-Type': 'application/json', 22 | 'Accept': 'application/json', 23 | } 24 | 25 | def __init__(self, *, api_key: str, field: Field = Field.Header): 26 | http.Method.__init__(self) 27 | if field == Field.Cookie: 28 | raise CookieNotImplemented('cookie not implemented') 29 | elif field == Field.Header: 30 | self.headers[api_key_name] = api_key 31 | elif field == Field.QueryParam: 32 | self.params[api_key_name] = api_key 33 | 34 | 35 | class OpenAPI(APIKey): 36 | url_ = "/openapi.json" 37 | m_type = "GET" 38 | headers = { 39 | 'Content-Type': 'application/json', 40 | 'Accept': 'application/json', 41 | } 42 | 43 | 44 | class Docs(APIKey): 45 | url_ = "/docs" 46 | m_type = "GET" 47 | headers = { 48 | 'Content-Type': 'application/json', 49 | 'Accept': 'application/json', 50 | } 51 | 52 | 53 | class Health(APIKey): 54 | url_ = "/health" 55 | m_type = "GET" 56 | headers = { 57 | 'Content-Type': 'application/json', 58 | 'Accept': 'application/json', 59 | } 60 | 61 | 62 | class Info(APIKey): 63 | url_ = "/info" 64 | m_type = "GET" 65 | headers = { 66 | 'Content-Type': 'application/json', 67 | 'Accept': 'application/json', 68 | } 69 | -------------------------------------------------------------------------------- /docs/tutorial.md: -------------------------------------------------------------------------------- 1 | ## Tutorial 2 | 3 | After initialization, cookiecutter ask you about clonning the latest version into temporary repo (if you have our already layout). Anothercase, cookiecutter clonning repo into `/home/your-name/.cookiecutters`: 4 | 5 | You've downloaded /home/your-name/.cookiecutters/python-service-layout before. Is it okay to delete and re-download it? [yes]: 6 | 7 | If you said `yes`, cookicutter clonning repo with replace. Othercase (`no`), you get another else question: 8 | 9 | Do you want to re-use the existing version? [yes]: 10 | 11 | If you said `no`, cookiecutter is finished. 12 | 13 | Now, cookiecutter ask you some questions about your environment. 14 | 15 | The first block: 16 | 17 | 1. `author`. Enter your name 18 | 2. `email`. Enter your email 19 | 3. `description`. Enter description your service's name 20 | 4. `service`. Enter your service's name 21 | 5. `python_package`. Compiled python package's name 22 | 23 | The second block: 24 | 25 | 6. `pypi_schema`. Enter your pypi registry's schema 26 | 7. `pypi_host`. Enter your pypi registry's host and path (for example `pypi.org/simple` or `192.168.0.1`) 27 | 8. `pypi_port`. Enter your pypi registry's port 28 | 9. `pypi_registry`. Compiled python registry's name 29 | 10. `pypi_alias`: alias to pypi registry 30 | 11. `pypi_login`: login to pypi registry (for publishing and installing) 31 | 12. `pypi_password`: password to pypi registry (for publishing and installing) 32 | 33 | The third block: 34 | 35 | 13. `docker_schema`. Enter your docker registry's schema 36 | 14. `docker_host`. Enter your docker registry's host. For example `192.168.0.1` 37 | 16. `docker_port`. Enter your docker registry's port. For example `8000` 38 | 18. `docker_registry`. Compiled docker registry's name 39 | 19. `docker_image`. Compiled docker image's name 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *.pyc 5 | *$py.class 6 | 7 | # OSX useful to ignore 8 | *.DS_Store 9 | .AppleDouble 10 | .LSOverride 11 | 12 | # Thumbnails 13 | ._* 14 | 15 | # Files that might appear in the root of a volume 16 | .DocumentRevisions-V100 17 | .fseventsd 18 | .Spotlight-V100 19 | .TemporaryItems 20 | .Trashes 21 | .VolumeIcon.icns 22 | .com.apple.timemachine.donotpresent 23 | 24 | # Directories potentially created on remote AFP share 25 | .AppleDB 26 | .AppleDesktop 27 | Network Trash Folder 28 | Temporary Items 29 | .apdisk 30 | 31 | # C extensions 32 | *.so 33 | 34 | # Distribution / packaging 35 | .Python 36 | env/ 37 | build/ 38 | develop-eggs/ 39 | dist/ 40 | downloads/ 41 | eggs/ 42 | .eggs/ 43 | lib/ 44 | lib64/ 45 | parts/ 46 | sdist/ 47 | var/ 48 | *.egg-info/ 49 | .installed.cfg 50 | *.egg 51 | 52 | # PyInstaller 53 | # Usually these files are written by a python script from a template 54 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 55 | *.manifest 56 | *.spec 57 | 58 | # Installer logs 59 | pip-log.txt 60 | pip-delete-this-directory.txt 61 | 62 | # Unit test / coverage reports 63 | htmlcov/ 64 | .tox/ 65 | .coverage 66 | .coverage.* 67 | .cache 68 | nosetests.xml 69 | coverage.xml 70 | *,cover 71 | .hypothesis/ 72 | .pytest_cache/ 73 | 74 | # Translations 75 | *.mo 76 | *.pot 77 | 78 | # Django stuff: 79 | *.log 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # IntelliJ Idea family of suites 85 | .idea 86 | *.iml 87 | ## File-based project format: 88 | *.ipr 89 | *.iws 90 | ## mpeltonen/sbt-idea plugin 91 | .idea_modules/ 92 | 93 | # PyBuilder 94 | target/ 95 | 96 | # Cookiecutter 97 | output/ 98 | python_boilerplate/ 99 | cookiecutter-pypackage-env/ 100 | 101 | # IDE settings 102 | .vscode/ 103 | 104 | # Secrets deploying 105 | deployments/.secrets/* 106 | !deployments/.secrets/.gitkeep 107 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/http_.py: -------------------------------------------------------------------------------- 1 | import fastapi 2 | import uvicorn 3 | from loguru import logger 4 | from fastapi.openapi import docs 5 | from fastapi.openapi.models import APIKey 6 | from fastapi.openapi.utils import get_openapi 7 | from starlette.responses import JSONResponse 8 | from prometheus_client import start_http_server 9 | 10 | from server.router import service, auth 11 | from server import server, config 12 | import info 13 | 14 | 15 | app = server.build_app(config.allow_origins) 16 | app.include_router(service.router) 17 | 18 | 19 | tag = 'documentation' 20 | 21 | desc = 'This method returns openAPI json with service configuration' 22 | handler = '/openapi.json' 23 | summary = 'Open API' 24 | @app.get(handler, summary=summary, description=desc, tags=[tag]) 25 | async def open_api_endpoint(api_key: APIKey = fastapi.Depends(auth.get_api_key)): 26 | open_api = get_openapi(title=server.service_name(), version=info.version, routes=app.routes) 27 | return JSONResponse(open_api) 28 | 29 | desc = 'This method returns info about service. Version, service name and environment' 30 | handler = '/docs' 31 | summary = 'Service documentation' 32 | @app.get(handler, summary=summary, description=desc, tags=[tag]) 33 | async def documentation(api_key: APIKey = fastapi.Depends(auth.get_api_key)): 34 | response = docs.get_swagger_ui_html(openapi_url='/openapi.json', title='docs') 35 | return response 36 | 37 | 38 | logger.info(f'app: {info.name}; version: {info.version}') 39 | logger.info(f'environment: {config.VAULT_ENV}') 40 | swagger_endpoint = f'{config.{{ cookiecutter.python_package }}_schema}://{config.{{ cookiecutter.python_package }}_host}:{config.{{ cookiecutter.python_package }}_port}/api-key?{config.api_key_name}={config.{{ cookiecutter.python_package }}_api_key}' 41 | logger.info(f'swagger: {swagger_endpoint}') 42 | if __name__ == "__main__": 43 | start_http_server(config.prometheus_port) 44 | logger.info(f'prometheus port: {config.prometheus_port}') 45 | uvicorn.run(app, host=config.{{ cookiecutter.python_package }}_host, port=config.{{ cookiecutter.python_package }}_port) 46 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/server/router/service.py: -------------------------------------------------------------------------------- 1 | import fastapi 2 | from fastapi import APIRouter 3 | from fastapi.security import api_key 4 | from starlette.responses import RedirectResponse 5 | 6 | from server import config 7 | from server.router import auth 8 | import info 9 | 10 | 11 | router = APIRouter() 12 | 13 | tag = 'service' 14 | 15 | desc = 'This method set API key to cookie. We use get method for simple browser supporting' 16 | handler = '/api-key' 17 | summary = 'Create API token' 18 | @router.get(handler, summary=summary, description=desc, tags=[tag]) 19 | async def create_token(api_key: api_key.APIKey = fastapi.Depends(auth.get_api_key)): 20 | response = RedirectResponse(url="/docs") 21 | response.set_cookie( 22 | config.api_key_name, 23 | value=api_key, 24 | domain=config.{{ cookiecutter.python_package }}_host, 25 | httponly=True, 26 | max_age=1800, 27 | expires=1800, 28 | ) 29 | return response 30 | 31 | 32 | desc = 'This method delete API key from cookie' 33 | handler = '/api-key' 34 | summary = 'Delete API token' 35 | @router.delete(handler, summary=summary, description=desc, tags=[tag]) 36 | async def delete_token(): 37 | response = RedirectResponse(url="/") 38 | response.delete_cookie(config.api_key_name, domain=config.{{ cookiecutter.python_package }}_host) 39 | return response 40 | 41 | 42 | desc = 'This method check health of service' 43 | handler = '/health' 44 | summary = 'Health of service' 45 | @router.get(handler, summary=summary, description=desc, status_code=204, tags=[tag]) 46 | async def health(api_key: api_key.APIKey = fastapi.Depends(auth.get_api_key)): 47 | return 48 | 49 | 50 | desc = 'This method returns info about service. Version, service name and environment' 51 | handler = '/info' 52 | summary = 'Information about service' 53 | @router.get(handler, summary=summary, description=desc, status_code=200, tags=[tag]) 54 | async def info_method(api_key: api_key.APIKey = fastapi.Depends(auth.get_api_key)): 55 | return {'version': info.version, 'name': info.name, 'environment': config.VAULT_ENV} 56 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *.pyc 5 | *$py.class 6 | 7 | # OSX useful to ignore 8 | *.DS_Store 9 | .AppleDouble 10 | .LSOverride 11 | 12 | # Thumbnails 13 | ._* 14 | 15 | # Files that might appear in the root of a volume 16 | .DocumentRevisions-V100 17 | .fseventsd 18 | .Spotlight-V100 19 | .TemporaryItems 20 | .Trashes 21 | .VolumeIcon.icns 22 | .com.apple.timemachine.donotpresent 23 | 24 | # Directories potentially created on remote AFP share 25 | .AppleDB 26 | .AppleDesktop 27 | Network Trash Folder 28 | Temporary Items 29 | .apdisk 30 | 31 | # C extensions 32 | *.so 33 | 34 | # Distribution / packaging 35 | .Python 36 | env/ 37 | build/ 38 | develop-eggs/ 39 | dist/ 40 | downloads/ 41 | eggs/ 42 | .eggs/ 43 | lib/ 44 | lib64/ 45 | parts/ 46 | sdist/ 47 | var/ 48 | *.egg-info/ 49 | .installed.cfg 50 | *.egg 51 | 52 | # PyInstaller 53 | # Usually these files are written by a python script from a template 54 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 55 | *.manifest 56 | *.spec 57 | 58 | # Installer logs 59 | pip-log.txt 60 | pip-delete-this-directory.txt 61 | 62 | # Unit test / coverage reports 63 | htmlcov/ 64 | .tox/ 65 | .coverage 66 | .coverage.* 67 | .cache 68 | nosetests.xml 69 | coverage.xml 70 | *,cover 71 | .hypothesis/ 72 | .pytest_cache/ 73 | 74 | # Translations 75 | *.mo 76 | *.pot 77 | 78 | # Django stuff: 79 | *.log 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # IntelliJ Idea family of suites 85 | .idea 86 | *.iml 87 | ## File-based project format: 88 | *.ipr 89 | *.iws 90 | ## mpeltonen/sbt-idea plugin 91 | .idea_modules/ 92 | 93 | # PyBuilder 94 | target/ 95 | 96 | # Cookiecutter 97 | output/ 98 | python_boilerplate/ 99 | cookiecutter-pypackage-env/ 100 | 101 | # IDE settings 102 | .vscode/ 103 | 104 | # Secrets deploying 105 | deployments/.secrets/* 106 | !deployments/.secrets/.gitkeep 107 | 108 | # Application logs 109 | data/logs 110 | !data/logs/.gitkeep 111 | 112 | # Application test's temporary files 113 | data/test/temp 114 | !data/test/temp/.gitkeep 115 | 116 | */.ipynb_checkpoints/ 117 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/makefile: -------------------------------------------------------------------------------- 1 | PYTHONPATH=. 2 | SCRIPTS=scripts/ 3 | TESTS_UNIT={{ cookiecutter.python_package }}/tests/ 4 | TESTS_INTEGRATION=tests/ 5 | REQ=requirements 6 | REQ_PRIVATE=requirements.private 7 | PIP_CONFIG_FILE=deployments/.secrets/pip_private.conf 8 | DOCKER_COMPOSE_FULL=deployments/docker-compose.full.yml 9 | DOCKER_COMPOSE_ENV=deployments/docker-compose.env.yml 10 | VAULT_ENV_FILE_TEST=deployments/.envs/test.env 11 | VAULT_ENV_FILE=deployments/.envs/local.env 12 | CONDA=conda 13 | UVICORN=uvicorn 14 | 15 | ifndef PYTHON 16 | PYTHON=python 17 | endif 18 | ifndef DOCKER_COMPOSE 19 | DOCKER_COMPOSE=docker-compose 20 | endif 21 | ifndef DOCKER 22 | DOCKER=docker 23 | endif 24 | ifndef PYTEST 25 | PYTEST=pytest 26 | endif 27 | ifndef PIP 28 | PIP=pip 29 | endif 30 | ifndef TEST_SUBFOLDER 31 | TEST_SUBFOLDER=./ 32 | endif 33 | ifndef VAULT_ENV 34 | VAULT_ENV=LOCAL 35 | endif 36 | ifeq ($(TEST),yes) 37 | VAULT_ENV_FILE=${VAULT_ENV_FILE_TEST} 38 | endif 39 | 40 | 41 | .PHONY: config build publish clean deps run run-env run-full run-rebuild test-integration test-unit test 42 | 43 | 44 | ENVS=PYTHONPATH=${PYTHONPATH} VAULT_ENV_FILE=${VAULT_ENV_FILE} VAULT_ENV=${VAULT_ENV} 45 | ENVS_TEST=PYTHONPATH=${PYTHONPATH} VAULT_ENV_FILE=${VAULT_ENV_FILE_TEST} VAULT_ENV=${VAULT_ENV} 46 | 47 | 48 | WORKERS=$(shell $(ENVS) $(PYTHON) scripts/variable.py WORKERS) 49 | PORT=$(shell $(ENVS) $(PYTHON) scripts/variable.py PORT) 50 | 51 | 52 | config: 53 | $(ENVS) $(CONDA) create --name {{ cookiecutter.service }} python=3.7 54 | 55 | build-image: 56 | $(ENVS) $(DOCKER_COMPOSE) -f $(DOCKER_COMPOSE_FULL) build 57 | 58 | publish-image: 59 | $(ENVS) $(DOCKER) tag $(TAG) {{ cookiecutter.docker_image }}:$(VERSION) 60 | $(ENVS) $(DOCKER) push {{ cookiecutter.docker_image }}:$(VERSION) 61 | 62 | publish-package: 63 | $(ENVS) $(PYTHON) setup.py bdist_wheel upload -r {{ cookiecutter.pypi_alias }} 64 | 65 | clean: 66 | rm -rf build/ 67 | rm -rf dist/ 68 | rm -rf {{ cookiecutter.python_package }}.egg-info 69 | rm -rf data/test/temp/ 70 | 71 | deps: 72 | $(ENVS) $(PIP) install -r $(REQ) --use-feature=2020-resolver 73 | $(ENVS) PIP_CONFIG_FILE=${PIP_CONFIG_FILE} $(PIP) install -r $(REQ_PRIVATE) --use-feature=2020-resolver 74 | 75 | run: 76 | $(ENVS) $(UVICORN) http_:app --workers=$(WORKERS) --port=$(PORT) --host=0.0.0.0 77 | 78 | run-full: 79 | $(ENVS) $(DOCKER_COMPOSE) -f $(DOCKER_COMPOSE_FULL) up 80 | 81 | run-env: 82 | $(ENVS) $(DOCKER_COMPOSE) -f $(DOCKER_COMPOSE_ENV) up 83 | 84 | run-rebuild: 85 | $(ENVS) $(DOCKER_COMPOSE) -f $(DOCKER_COMPOSE_FULL) up --build 86 | 87 | test-integration: 88 | $(ENVS_TEST) $(PYTEST) -v -l --disable-warnings ${TESTS_INTEGRATION}${TEST_SUBFOLDER} 89 | 90 | test-unit: 91 | $(ENVS_TEST) $(PYTEST) -v -l --disable-warnings ${TESTS_UNIT}${TEST_SUBFOLDER} 92 | 93 | test: test-unit test-integration 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Template for private python service 2 | 3 | ![](%7B%7B%20cookiecutter.service%20%7D%7D/docs/swagger.png) 4 | 5 | This is a [cookiecutter](https://github.com/cookiecutter/cookiecutter) template for an internal REST API service, written in Python, inspired by [layout-golang](https://github.com/golang-standards/project-layout). The stack is based on [FastAPI](https://github.com/tiangolo/fastapi) and [uvicorn](https://www.uvicorn.org/) and runs on [docker](https://www.docker.com/) and docker-compose. 6 | 7 | ## Service layout 8 | 9 | Below is the 1000ft structure of project's modules. Here, red ones are private, green ones are public. Package consists of both public and private sources. 10 | 11 | ![](docs/structure.png) 12 | 13 | We have some modules: 14 | 15 | - **Package**. Publish to pypi registry 16 | - **Server**. Server has some submodules for configuration handlers, logging and exceptions. Server has all business logic. 17 | - **Integration tests**. Integration tests communicate with server as black box. You can read differences between unit and integration tests [here](%7B%7B%20cookiecutter.service%20%7D%7D/docs/tests.md) 18 | 19 | **Notice**. Bad practice to import any function from server module. 20 | 21 | Read more on the project's main principles [here](%7B%7B%20cookiecutter.service%20%7D%7D/docs/structure.md). 22 | you can also read more on our approach [here](https://github.com/U-Company/notes). 23 | 24 | ## Service 25 | 26 | Our service has built-in: 27 | 28 | - [prometheus endpoint](https://github.com/prometheus/client_python) 29 | - [vault-client](https://github.com/U-Company/vault-client) 30 | - healthcheck 31 | - [docker](https://www.docker.com/) and docker-compose for local development and deploying 32 | - isolated docker development 33 | - loguru for logging 34 | - autogeneration of README.md for your service 35 | - swagger from FastAPI /docs 36 | - [FastAPI](https://github.com/tiangolo/fastapi) as service 37 | - [Uvicorn](https://www.uvicorn.org/) as asgi server 38 | - console server 39 | - templates for unit and integration tests 40 | - interface for control your service via makefile 41 | - completely to publishing package (private pypi-registry) 42 | - completely to publishing dockerfile (private docker-registry) 43 | - basic token (api-key) [authentication](https://medium.com/data-rebels/fastapi-authentication-revisited-enabling-api-key-authentication-122dc5975680) 44 | 45 | ## Usage 46 | 47 | To use this project, you need to install [cookiecutter](https://github.com/cookiecutter/cookiecutter): 48 | 49 | pip install cookiecutter 50 | cookiecutter https://github.com/U-Company/python-private-service-layout.git 51 | 52 | Next, you need to have `docker` and `docker-compose`: 53 | 54 | sudo apt-get install make docker.io docker-compose 55 | 56 | [Here](%7B%7B%20cookiecutter.service%20%7D%7D/docs/commands.md) you cand find all available commands for communicate with service with a command line. 57 | 58 | If you have any errors, you can read about in documentary after project generation. You can communicate with Egor Urvanov by UrvanovCompany@yandex.ru or in telegram (@egor_urvanov) 59 | 60 | For the full tutorial of service generation you can read [here](docs/tutorial.md). 61 | 62 | 63 | ## Execution and infrastructure 64 | 65 | To deploy service **WITHOUT** dependencies, just run the docker-compose from the root via make: 66 | 67 | make run 68 | 69 | To deploy service **WITH** dependencies, just run the docker-compose from the root via make: 70 | 71 | make run-full 72 | 73 | (Default deployment is based on [infrastructure](https://github.com/U-Company/infrastructure)) 74 | 75 | That's it! Enjoy 76 | 77 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/docs/structure.md: -------------------------------------------------------------------------------- 1 | ## [Project structure]({{ cookiecutter.python_package }}/docs/structure.md) 2 | 3 | We provide a very simple interface for running your server (**after installation dependencies**): 4 | 5 | make run 6 | 7 | Or, if you have some service dependencies, you can do this: 8 | 9 | make run-full 10 | 11 | ## Installation dependencies 12 | 13 | You need to use: 14 | 15 | - docker 16 | - docker-compose 17 | - make 18 | - Anaconda 19 | 20 | Before work with service, you need to install: 21 | 22 | sudo apt-get install make docker.io make docker-compose 23 | 24 | Make and docker is not required features. This tools is needed for more useful development. We recommend to use Anaconda 25 | or another environment manager for safety system interpreter. You can download Anaconda 26 | [here](https://www.anaconda.com/). After installing Anaconda please create new environment: 27 | 28 | make config 29 | conda activate {{ cookiecutter.python_package }} 30 | 31 | Now, you can install all python dependencies 32 | 33 | ## Project structure 34 | 35 | ### [data/]({{ cookiecutter.python_package }}/data) 36 | 37 | This folder must consists the data of service. We think to a large data file must dumps to any storage ( 38 | databases, file storage, git LFS, docs and other). Most part of data here is test's data. 39 | 40 | ### data/tests/ 41 | 42 | Files for tests 43 | 44 | ### data/docs/ 45 | 46 | Design and user documents 47 | 48 | ### data/logs/ 49 | 50 | Folder for logs files 51 | 52 | ### [deployments/]({{ cookiecutter.python_package }}/deployments) 53 | 54 | IaaS, PaaS, system and container orchestration deployment configurations and templates (docker-compose, kubernetes/helm, 55 | mesos, terraform, bosh). 56 | 57 | You can deploy applications with different docker-compose files and different environment files. Our docker files are built by a make file, which uses as API interface for commutication building of any programm. 58 | 59 | Type of docker files: 60 | 61 | * **docker-compose.env.yml** -- environment for your application. For example, you have some databases and some servicese, but you want to run your application from source (for debugging, for instance). Then, you can runnig only this file. 62 | * **docker-compose.full.yml** -- full application with environment 63 | 64 | We have environment file into `./.envs` directory. 65 | 66 | **Notice**: we configure compose with `network_mode: host` (in) therefore, our image links with localhost (no mapping ports, for example) 67 | 68 | You can read about publishing packages [here]({{ cookiecutter.python_package }}/deployments). 69 | 70 | ### {{ cookiecutter.python_package }}/ 71 | 72 | Library code that's ok to use by external applications (e.g., /{{ cookiecutter.python_package }}/mypubliclib). Other projects will 73 | import these libraries expecting them to work, so think twice before you put something here :-) 74 | 75 | We have only two type modules in python package of service: 76 | 77 | - generators 78 | - public API methods for service 79 | 80 | ### [{{ cookiecutter.python_package }}/generators]({{ cookiecutter.python_package }}/generators/) 81 | 82 | Each service works with data. Therefore a good practice is build generators for data creation of each method. This generators is part of package for using in tests of other packages. 83 | 84 | ### [{{ cookiecutter.python_package }}/methods.py]({{ cookiecutter.python_package }}/methods.py) 85 | 86 | Here we save methods for connect to our service. Example of usage, you can find [here]({{ cookiecutter.python_package }}/methods.py) 87 | 88 | ### [server/tests](server/tests/) 89 | 90 | This repo consists unit tests. Unit tests is used to test functions, not services 91 | 92 | ### [scripts/](scripts) 93 | 94 | Service scripts are stored here. Default scripts: 95 | 96 | - Getting variables from Vault or file (variable.py). This script it need for `make run` command 97 | 98 | ### [tests/](tests) 99 | 100 | This directory consists only integration tests. In this directory cannot import files from another directories except 101 | package directory. We tests our service as block box method. Therefore we cannot use common 102 | functions or scripts with tested sources. 103 | 104 | More details about testing, you can find [here](tests.md) 105 | 106 | ### makefile 107 | 108 | Makefile is given interface for control of application: dependencies, run, test, deploying. 109 | 110 | ### setup 111 | 112 | This is template for configuration of package of setuptools package 113 | 114 | ### info 115 | 116 | This file consists version and package's name. 117 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.service }} 2 | 3 | {{ cookiecutter.description }} 4 | 5 | This service is built with a [python-private-service-layout](https://github.com/U-Company/python-private-service-layout) 6 | 7 | You can read about service [here](docs/service.md) or abouts structure project [here](docs/structure.md) 8 | 9 | ## Installing 10 | 11 | Before work with our service, you need to install: 12 | 13 | sudo apt-get install make docker.io make docker-compose 14 | 15 | Make and docker is not required features. This tools is needed for more useful development. We recommend to use Anaconda 16 | or another environment manager for safety system interpreter. You can download Anaconda 17 | [here](https://www.anaconda.com/). After installing Anaconda please create new environment: 18 | 19 | conda create --name your-name python=3.7 20 | conda activate 21 | 22 | or, you can do this: 23 | 24 | make config 25 | conda activate 26 | 27 | After that, you get project name's environment: 28 | 29 | make deps 30 | 31 | ## Configuration for publishing 32 | 33 | If you want to publish images and packages, you need configure docker and PyPi. You can read short instruction below, or 34 | full instruction [here](deployments) 35 | 36 | ### Prepare config for setup.py (Ubuntu) 37 | 38 | **Before publishing package**, you need to create file `~/.pypirc`. Copy [this file](deployments/.secrets/.pypirc) to `~/.pypirc`. If you already have such file, you need to mix it like [this](deployments/.secrets/.pypirc_mixed). `.pypirc` is required for publishing python packages. 39 | 40 | ### Configure docker 41 | 42 | You must lay the config file [this](deployments/.secrets/daemon.json) into `/etc/docker/daemon.json`. If you already have 43 | such file, you need to mix it like [this](deployments/.secrets/daemon.json_mixed). 44 | 45 | After that, you must restart docker (first time): 46 | 47 | sudo service docker restart 48 | 49 | Now, login in docker registry with your login and password (first time): 50 | 51 | docker login {{cookiecutter.docker_registry}} -u="" -p="" 52 | 53 | If you don't login, while pulling or pushing, do this by handle before pushing. 54 | 55 | ## Dependencies 56 | 57 | Install all package dependencies. We suppose that you have not more two registry: 58 | 59 | - [public PyPi-registry](https://pypi.org/project/registry/) and 60 | - your private pypi-registry (it is optional) 61 | 62 | This command install from both or only public: 63 | 64 | make deps 65 | 66 | We automatically create file to installing packages from repository (private or public). You can see this file [here](deployments/.secrets/pip_private.conf). If your repo is [common](https://pypi.org/), then set cookiecutter's default settings while init procedure. Othercase set your custom repositroy. 67 | 68 | ## Publishing image 69 | 70 | If you want to publish docker image into registry, you need to do this: 71 | 72 | make build 73 | 74 | Copy tag from console and do this: 75 | 76 | VERSION=x.y.z TAG= make publish-image 77 | 78 | ## Publishing package 79 | 80 | If you want to publish docker image into registry, you need to do this: 81 | 82 | make publish-package 83 | 84 | ## Clean 85 | 86 | You can clean python package after building and all temporary files: 87 | 88 | make clean 89 | 90 | ## Starting 91 | 92 | Before starting please install all python package dependencies. Don't forget it: 93 | 94 | make deps 95 | 96 | We have three mode of starting: 97 | 98 | - full subsystem 99 | - development 100 | - make 101 | 102 | ### Running 103 | 104 | We use docker-compose for local development and starting you service and environment. If you want to start full 105 | subsystem, you need to do this: 106 | 107 | make run-full 108 | 109 | After that our service and environment is started. If you want to start our service the first time, docker container with service is built. Other container is pulled. 110 | 111 | If you want to start service not first time, maybe you need rebuilt service for apply last changes: 112 | 113 | make run-rebuild 114 | 115 | ### development 116 | 117 | For development, you can use only environment: 118 | 119 | make run-env 120 | 121 | After that, it starts all dependencies services. Now, you can run our service in your IDE for development. 122 | 123 | ### make 124 | 125 | For fast start our service we use command: 126 | 127 | make run 128 | 129 | ## Environment variables 130 | 131 | Our service takes all environments variables from config: `deployments/.envs/local.env`. More about it you can read into 132 | this file: `{{ cookiecutter.service }}/__service/config.py`. You can add new variables there and here: `deployments/.envs/local.env`. If you want to configure testing environment, you need change file `deployments/.envs/test.env`. 133 | 134 | We separates variables by namespaces, therefore we set prefix before variable name. You can see in files, which we 135 | denote above. 136 | 137 | ## Testing 138 | 139 | We have three mode of testing: 140 | 141 | - unit testing 142 | - integration testing 143 | - all: unit and integration testing 144 | 145 | **NOTICE**: before start tests, set envs: `VAULT_ENV=LOCAL`, `VAULT_ENV_PATH=deployments/.envs/test.env`. `VAULT_ENV` says to Vault client takes envs from file. `VAULT_ENV_PATH` sets path to this file. If you use make file, you need to use, only `TEST=yes`. Makefile configure test environment automatically 146 | 147 | We have three commands: 148 | 149 | make test-integration 150 | 151 | Before starting integration tests (above) you need to start service with environment: 152 | 153 | TEST=yes make run-full 154 | 155 | Environment `TEST=yes` change env file to `deployments/tests/test.env` 156 | 157 | Unit tests: 158 | 159 | make test-unit 160 | 161 | All tests: 162 | 163 | make test 164 | 165 | If you want to configure testing environment, you need change file `deployments/.envs/test.env`. 166 | 167 | ## Swagger 168 | 169 | If you want to use Swagger, you need go to `http:///api-key?access_token=` (by default, 170 | `=1234567890ABCDEFG`, `=0.0.0.0`). After that, browser save your token and use them. You can see 171 | token in environment variable [file]({{ cookiecutter.service }}/deployments/.envs/local.env). You can change query param 172 | name in the same file. 173 | 174 | If you want to say browser forget api-key, you need request `DELETE /api-key` method. 175 | 176 | ## Notice 177 | 178 | We use makefile as interface for communicate our application with our systems by command line while development and 179 | deployments 180 | 181 | ## Common errors 182 | 183 | If you have some errors, you can read 184 | [Common errors]({{ cookiecutter.python_package }}/docs/errors.md) doc. Or you can communicate with Egor Urvanov by UrvanovCompany@yandex.ru or in telegram (@egor_urvanov) 185 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/docs/errors.md: -------------------------------------------------------------------------------- 1 | # Not secured connection 2 | 3 | Probably you don't set `TAG` or `VERSION` before `make publish-image`. Please, add `daemon.json` to `/etc/docker/daemon.json`. You can read more [here](/{{ cookiecutter.python_package }}) or [here](/{{ cookiecutter.python_package }}/deployments) 4 | 5 | The push refers to repository [{{ cookiecutter.docker_registry }}/{{ cookiecutter.python_package }}] 6 | Get {{ cookiecutter.docker_registry }}/v2/: http: server gave HTTP response to HTTPS client 7 | 8 | # Anaconda prefix error 9 | 10 | CondaValueError: prefix already exists: /home/username/anaconda3/envs/environment 11 | 12 | Probably, you use `make config` twice. The latest version of anaconda try to replace existed environment. 13 | 14 | Probably, you use not latest conda version (conda 4.5.* and lower). You need to update anaconda: 15 | 16 | conda update conda 17 | 18 | # Anaconda not found 19 | 20 | You can [install](https://www.anaconda.com/products/individual) anaconda You can read this answer https://stackoverflow.com/questions/35246386/conda-command-not-found/44319368 21 | 22 | Conda command not found 23 | 24 | You can insert to the file `~/.bashrc` next line: 25 | 26 | export PATH="/path/to/anaconda/bin:$PATH" 27 | 28 | Example: 29 | 30 | export PATH="/home/username/anaconda3/bin:$PATH" 31 | 32 | # Some command not found 33 | 34 | Please see [this](https://github.com/U-Company/python-private-service-layout#usage) page 35 | 36 | # Tag or version not set 37 | 38 | Probably you don't set `TAG` or `VERSION` before `make publish-image`. You can see some info [here](https://github.com/U-Company/python-private-service-layout#usage) or [here](/{{ cookiecutter.python_package }}/docs/commands.md) 39 | 40 | docker tag {{ cookiecutter.docker_registry }}/{{ cookiecutter.docker_image }}: 41 | "docker tag" requires exactly 2 arguments. 42 | See 'docker tag --help'. 43 | 44 | Usage: docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG] 45 | 46 | Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE 47 | makefile:53: recipe for target 'publish-image' failed 48 | make: *** [publish-image] Error 1 49 | 50 | # .pypirc file configuration 51 | 52 | Error: 53 | 54 | Traceback (most recent call last): 55 | File "setup.py", line 38, in 56 | '{{ cookiecutter.python_package }}_http={{ cookiecutter.python_package }}.__cmd.http_:main', 57 | File "/home/username/anaconda3/envs/l/lib/python3.7/site-packages/setuptools/__init__.py", line 161, in setup 58 | return distutils.core.setup(**attrs) 59 | File "/home/username/anaconda3/envs/l/lib/python3.7/distutils/core.py", line 148, in setup 60 | dist.run_commands() 61 | File "/home/username/anaconda3/envs/l/lib/python3.7/distutils/dist.py", line 966, in run_commands 62 | self.run_command(cmd) 63 | File "/home/username/anaconda3/envs/l/lib/python3.7/distutils/dist.py", line 985, in run_command 64 | cmd_obj.run() 65 | File "/home/username/anaconda3/envs/l/lib/python3.7/distutils/command/upload.py", line 64, in run 66 | self.upload_file(command, pyversion, filename) 67 | File "/home/username/anaconda3/envs/l/lib/python3.7/distutils/command/upload.py", line 74, in upload_file 68 | raise AssertionError("unsupported schema " + schema) 69 | AssertionError: unsupported schema 70 | makefile:58: recipe for target 'publish-package' failed 71 | make: *** [publish-package] Error 1 72 | 73 | 74 | Probably you run the `make publish-package` 75 | 76 | Cases: 77 | 78 | 1. You forget to copy or change the file `./deployments/.secrets/.pypirc` to `~/`. 79 | 2. You forget to add alias to `[distutils]` section into `.pypirc` file after change him. 80 | 3. You forget to add section with alias to `.pypirc` file after change him. 81 | 82 | You can find more info [here](/{{ cookiecutter.python_package }}#prepare-config-for-pip-ubuntu) or [here](/{{ cookiecutter.python_package }}deployments). 83 | 84 | # Duplicate package 85 | 86 | Upload failed (409): Conflict 87 | error: Upload failed (409): Conflict 88 | makefile:58: recipe for target 'publish-package' failed 89 | make: *** [publish-package] Error 1 90 | 91 | Probably you run the `make publish-package` and get this error: 92 | 93 | It means that, this package already exists. Please change version or remove old version. You can remove by [this](https://github.com/U-Company/notes/tree/master/deployments#publish-image-into-docker-registry-for-local-development-and-testing) way. 94 | 95 | # Problem with uninstall 96 | 97 | Cannot uninstall 'certifi'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall. 98 | 99 | **Best rule**. You can solve this problem with reinitialization anaconda: 100 | 101 | make config 102 | 103 | Or you can change make file rule `deps`. Add `--ignore-installed` for pip. You can read [some](https://pip.pypa.io/en/stable/reference/pip_install/#cmdoption-i) [topics](https://stackoverflow.com/questions/51913361/difference-between-pip-install-options-ignore-installed-and-force-reinstall) [about](https://github.com/pypa/pip/issues/5247) [it](https://github.com/galaxyproject/galaxy/issues/7324). 104 | 105 | The second way can break down you application. 106 | 107 | # Vault client not authenticated 108 | 109 | Traceback (most recent call last): 110 | File "/home/username/python/version/version/__cmd/http_.py", line 16, in 111 | from version.__server import config, models 112 | File "/home/username/python/version/version/__server/config.py", line 13, in 113 | assert vault_client.is_authenticated, 'Vault client not authenticated' 114 | AssertionError: Vault client not authenticated 115 | 116 | If you use NOT `LOCAL` mode of development, check vault credentials. You need to set these environments for [vault-client](https://github.com/U-Company/vault-client): 117 | 118 | VAULT_TOKEN= 119 | VAULT_PORT= 120 | VAULT_HOST= 121 | VAULT_MOUNT_POINT= 122 | VAULT_ENV= 123 | 124 | One way do this: set environments into `.env` file into `deployments/.envs/` or pass another way. 125 | 126 | If you use `LOCAL` mode of develop and you get such problem, than probably you use a PyCharm or another IDE. In this case, you can forget change a workdir. Please, set them as root directory of project. PyCharm example: 127 | 128 | ![](/docs/IDE_workdir.png) 129 | 130 | # Not set environments 131 | 132 | Traceback (most recent call last): 133 | File "/home/username/python/version/version/__cmd/http_.py", line 16, in 134 | from version.__server import config, models 135 | File "/home/username/python/version/version/__server/config.py", line 11, in 136 | vault_client = VaultClient(environ=VAULT_ENV, env_file=VAULT_ENV_FILE) 137 | File "", line 5, in __init__ 138 | File "/home/username/anaconda3/envs/version/lib/python3.7/site-packages/vault_client/client.py", line 22, in __post_init__ 139 | self.environ = self.environ.upper() 140 | AttributeError: 'NoneType' object has no attribute 'upper' 141 | 142 | If you start with mode of vault `LOCAL`, probably, you forget set environments: `VAULT_ENV` or `VAULT_ENV_FILE`. Please, check default values by path: ![]({{ cookiecutter.docker_registry }}/__server/config.py): 143 | 144 | VAULT_ENV = os.environ.get('VAULT_ENV') 145 | VAULT_ENV_FILE = os.environ.get('VAULT_ENV_FILE') 146 | 147 | Change to 148 | 149 | VAULT_ENV = os.environ.get('VAULT_ENV', 'LOCAL') 150 | VAULT_ENV_FILE = os.environ.get('VAULT_ENV_FILE', 'deployments/.envs/local.env') 151 | 152 | Or set environments by handle. 153 | -------------------------------------------------------------------------------- /{{ cookiecutter.service }}/server/asserts.py: -------------------------------------------------------------------------------- 1 | from fastapi import HTTPException 2 | from loguru import logger 3 | 4 | import copy 5 | import dataclasses 6 | import typing 7 | import collections 8 | 9 | 10 | Variable = collections.namedtuple('Variable', ['deleted', 'value']) 11 | 12 | 13 | @dataclasses.dataclass 14 | class Context: 15 | def __init__( 16 | self, 17 | *, 18 | handler: str, 19 | method: str, 20 | body: typing.Dict = None, 21 | query_params: typing.Dict = None, 22 | headers: typing.Dict = None, 23 | ): 24 | """ 25 | 26 | :param handler: handler path 27 | :param method: POST, GET, HEAD, etc... 28 | :param body: pydantic.BaseModel 29 | :param query_params: typing.Dict 30 | :param headers: typing.Dict 31 | """ 32 | self.__vars = { 33 | 'handler': Variable(value=handler, deleted=False), 34 | 'request': Variable( 35 | value={ 36 | 'endpoint': body, 37 | 'method': method, 38 | 'body': body, 39 | 'query_params': query_params, 40 | 'headers': headers, 41 | }, 42 | deleted=False, 43 | ), 44 | } 45 | self.__cache = None 46 | 47 | def __enter__(self): 48 | self.__cache = copy.deepcopy(self.__vars) 49 | 50 | def __exit__(self, *args, **kwargs): 51 | self.__vars = self.__cache 52 | 53 | def __getitem__(self, key): 54 | v = self.__vars.get(key) 55 | if v is None: 56 | return None 57 | if v.deleted: 58 | return None 59 | return v.value 60 | 61 | def __setitem__(self, key, value): 62 | self.__vars[key] = Variable(deleted=False, value=value) 63 | 64 | def __delitem__(self, key): 65 | self.__vars[key] = Variable(deleted=True, value=self.__vars[key].value) 66 | 67 | def __del__(self): 68 | self.__vars = {} 69 | 70 | def copy(self): 71 | return copy.deepcopy(self) 72 | 73 | 74 | def _exception_validation(ctx: Context, *, pcode, icode, detail=True, msg=None): 75 | """ 76 | Exception of validator execution 77 | 78 | :param ctx: context's execution of runtime 79 | :param pcode: default public code for response (HTTP response status code) 80 | :param icode: default private code for response (internal code) 81 | :param detail: is need returns message in exception 82 | :param msg: message for response 83 | :return: 84 | """ 85 | log_msg = { 86 | 'handler': ctx['handler'], # handler of method 87 | 'request': ctx['request'], # request of method 88 | 'detail': msg, # default message from class 89 | 'comment': ctx['comment'], # comment from developer (not message for response) 90 | 'error_name': ctx['error_name'], # class name of error 91 | 'variables': ctx['variables'], # state's variables 92 | 'public_status_code': pcode, # status code of response (HTTP status code) 93 | 'internal_code': icode, # status code of response (internal code) 94 | } 95 | logger.exception(log_msg) 96 | msg = f'Code: {icode}. {msg}' 97 | if not detail: 98 | msg = f'Code: {icode}' 99 | raise HTTPException(pcode, detail=msg) 100 | 101 | 102 | class AssertorValidation: 103 | def __init__(self, pcode, icode, msg=None, detail=True): 104 | """ 105 | ServiceAssertor checks any conditions with context 106 | 107 | :param pcode: default public code for response (HTTP response status code) 108 | :param icode: default private code for response (internal code, internal system errors) 109 | :param msg: default message for response 110 | :param detail: show detail in HTTPException 111 | """ 112 | self.name = type(self).__name__ 113 | self.msg = msg 114 | self.pcode = pcode 115 | self.icode = icode 116 | self.detail = detail 117 | 118 | @staticmethod 119 | def __add_variables(ctx, kwargs): 120 | if ctx['variables'] is None: 121 | ctx['variables'] = {'_': kwargs} 122 | return 123 | ctx['variables']['_'] = kwargs 124 | 125 | def __call__(self, ctx: Context, **kwargs): 126 | """ 127 | :param ctx: dict with some fields. You can see these fields into `def exception(ctx, status_code, msg=None)` 128 | :param kwargs: params for validate 129 | :return: 130 | """ 131 | if self._validate(ctx, **kwargs): 132 | ctx_ = copy.deepcopy(ctx) 133 | self.__add_variables(ctx_, kwargs) 134 | ctx_['error_name'] = self.name 135 | _exception_validation(ctx=ctx_, msg=self.msg, detail=self.detail, pcode=self.pcode, icode=self.icode) 136 | 137 | def _validate(self, ctx: Context, **kwargs: typing.Dict): 138 | raise NotImplementedError() 139 | 140 | 141 | class AssertorOK(AssertorValidation): 142 | """ 143 | Check ok 144 | """ 145 | def _validate(self, ctx: Context, **kwargs: typing.Dict): 146 | return not kwargs['ok'] 147 | 148 | 149 | class AssertorNotNone(AssertorValidation): 150 | """ 151 | Check ok 152 | """ 153 | def _validate(self, ctx: Context, **kwargs: typing.Dict): 154 | return not (kwargs['ok'] is not None) 155 | 156 | 157 | class AssertorNone(AssertorValidation): 158 | """ 159 | Check ok 160 | """ 161 | def _validate(self, ctx: Context, **kwargs: typing.Dict): 162 | return not (kwargs['ok'] is None) 163 | 164 | 165 | class AssertorAEqB(AssertorValidation): 166 | """ 167 | Check a more b, where a is variables into context 168 | """ 169 | def _validate(self, ctx: Context, **kwargs: typing.Dict): 170 | """ 171 | :param ctx: context for validate 172 | :return: 173 | """ 174 | return not (kwargs['a'] == kwargs['b']) 175 | 176 | 177 | class AssertorALessB(AssertorValidation): 178 | """ 179 | Check a more b, where a is variables into context 180 | """ 181 | def _validate(self, ctx: Context, **kwargs: typing.Dict): 182 | """ 183 | :param ctx: context for validate 184 | :return: 185 | """ 186 | return not (kwargs['a'] < kwargs['b']) 187 | 188 | 189 | class AssertorALessEqB(AssertorValidation): 190 | """ 191 | Check a more b, where a is variables into context 192 | """ 193 | def _validate(self, ctx: Context, **kwargs: typing.Dict): 194 | """ 195 | :param ctx: context for validate 196 | :return: 197 | """ 198 | return not (kwargs['a'] <= kwargs['b']) 199 | 200 | 201 | def _exception_service(ctx: Context, *, pcode, icode, detail=True, msg=None): 202 | """ 203 | Exception of validator execution 204 | 205 | :param ctx: context's execution of runtime 206 | :param pcode: default public code for response (HTTP response status code) 207 | :param icode: default private code for response (internal code) 208 | :param wcode: wanted code from service 209 | :param detail: is need returns message in exception 210 | :param msg: message for response 211 | :return: 212 | """ 213 | log_msg = { 214 | 'handler': ctx['handler'], # handler of method 215 | 'request': ctx['request'], # request of method 216 | 'service_request': ctx['service_request'], # request to service 217 | 'service_response': ctx['service_response'], # response from service 218 | 'service_code': ctx['service_code'], # code from service 219 | 'detail': msg, # default message from class 220 | 'comment': ctx['comment'], # comment from developer (not message for response) 221 | 'error_name': ctx['error_name'], # class name of error 222 | 'variables': ctx['variables'], # state's variables 223 | 'public_status_code': pcode, # status code of response (HTTP status code) 224 | 'wanted_code': ctx['wcode'], # wanted code from service 225 | 'internal_code': pcode, # status code of response (internal code) 226 | } 227 | logger.exception(log_msg) 228 | msg = f'Code: {icode}. {msg}' 229 | if not detail: 230 | msg = f'Code: {icode}' 231 | raise HTTPException(pcode, detail=msg) 232 | 233 | 234 | class AssertorCode: 235 | def __init__(self, icode, pcode=520, msg=None, detail=True, wcode=None): 236 | """ 237 | ServiceAssertor checks any conditions with context 238 | 239 | :param pcode: default public code for response (HTTP response status code). Most probably, these codes are 5** 240 | :param icode: default private code for response (internal code, internal system errors) 241 | :param msg: default message for response 242 | :param detail: show detail in HTTPException 243 | :param wcode: wanted code from service. if wcode is not set, then pcode := wcode 244 | """ 245 | self.name = type(self).__name__ 246 | self.msg = msg 247 | self.pcode = pcode 248 | self.icode = icode 249 | self.detail = detail 250 | self.wcode = wcode 251 | if self.wcode is None: 252 | self.wcode = pcode 253 | 254 | def __call__(self, ctx: Context, *, req, resp, code): 255 | """ 256 | :param ctx: dict with some fields. You can see these fields into `def exception(ctx, status_code, msg=None)` 257 | :param req: params for validate 258 | :param resp: params for validate 259 | :return: 260 | """ 261 | if self._validate(ctx, req=req, resp=resp, code=code, wcode=self.wcode): 262 | ctx_ = copy.deepcopy(ctx) 263 | ctx_['error_name'] = self.name 264 | ctx_['service_request'] = req 265 | ctx_['service_response'] = resp 266 | ctx_['service_code'] = code 267 | ctx_['wcode'] = self.wcode 268 | _exception_service(ctx=ctx_, msg=self.msg, detail=self.detail, pcode=self.pcode, icode=self.icode) 269 | 270 | def _validate(self, ctx: Context, req, resp, code, wcode): 271 | """ 272 | :param ctx: context for validate 273 | :param req: request of service 274 | :param resp: response of service 275 | :param wcode: wanted code of service 276 | :param code: got code of service 277 | """ 278 | raise NotImplementedError() 279 | 280 | 281 | class AssertorEqCode(AssertorCode): 282 | """ 283 | Check a more b, where a is variables into context 284 | """ 285 | def _validate(self, ctx: Context, req, resp, code, wcode): 286 | """ 287 | :param ctx: context for validate 288 | :param req: request of service 289 | :param resp: response of service 290 | :param wcode: wanted code of service 291 | :param code: got code of service 292 | :return: 293 | """ 294 | return not (wcode == code) 295 | 296 | 297 | class AssertorNeqCode(AssertorCode): 298 | """ 299 | Check a more b, where a is variables into context 300 | """ 301 | def _validate(self, ctx: Context, req, resp, code, wcode): 302 | """ 303 | :param ctx: context for validate 304 | :param req: 305 | :param resp: 306 | :param wcode: int 307 | :param code: int 308 | :return: 309 | """ 310 | return not (wcode != code) 311 | 312 | 313 | Expected200 = AssertorEqCode(msg='Unexpected code 200', wcode=200, icode=-1, detail=False) 314 | Expected201 = AssertorEqCode(msg='Unexpected code 201', wcode=201, icode=-2, detail=False) 315 | Expected202 = AssertorEqCode(msg='Unexpected code 202', wcode=202, icode=-3, detail=False) 316 | Expected204 = AssertorEqCode(msg='Unexpected code 204', wcode=204, icode=-4, detail=False) 317 | 318 | ExtraFields = AssertorOK(msg='Consists extra fields', pcode=422, icode=1, detail=True) 319 | NotFoundFields = AssertorOK(msg='Not found some fields', pcode=422, icode=2, detail=True) 320 | --------------------------------------------------------------------------------