├── tests ├── __init__.py ├── unit │ ├── __init__.py │ ├── address_data_test.py │ ├── tornado_app_handlers_test.py │ ├── addressbook_db_test.py │ └── datamodel_test.py └── integration │ ├── __init__.py │ ├── addrservice_test.py │ └── tornado_app_addreservice_handlers_test.py ├── addrservice ├── tornado │ ├── __init__.py │ ├── server.py │ └── app.py ├── utils │ ├── __init__.py │ └── logutils.py ├── database │ ├── __init__.py │ ├── db_engines.py │ └── addressbook_db.py ├── __init__.py ├── service.py └── datamodel.py ├── requirements.txt ├── .vscode └── settings.json ├── data ├── __init__.py └── addresses │ ├── raga.json │ └── namo.json ├── LICENSE ├── configs └── addressbook-local.yaml ├── .gitignore ├── run.py ├── schema └── address-book-v1.0.json └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | -------------------------------------------------------------------------------- /addrservice/tornado/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | -------------------------------------------------------------------------------- /addrservice/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | -------------------------------------------------------------------------------- /addrservice/database/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiofiles==0.4.0 2 | aiohttp==3.7.4 3 | aiotask-context==0.6.1 4 | argparse==1.4.0 5 | asyncio==3.4.3 6 | asynctest==0.13.0 7 | coverage==5.0.3 8 | flake8==3.7.9 9 | jsonschema==3.2.0 10 | logfmt==0.4 11 | mypy==0.761 12 | PyYAML==5.4 13 | requests==2.22.0 14 | tornado==6.0.3 15 | -------------------------------------------------------------------------------- /addrservice/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import json 4 | import os 5 | 6 | ADDR_SERVICE_ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) 7 | 8 | ADDRESS_BOOK_SCHEMA_FILE = os.path.abspath(os.path.join( 9 | ADDR_SERVICE_ROOT_DIR, 10 | '../schema/address-book-v1.0.json' 11 | )) 12 | 13 | with open(ADDRESS_BOOK_SCHEMA_FILE, mode='r', encoding='utf-8') as f: 14 | ADDRESS_BOOK_SCHEMA = json.load(f) 15 | 16 | LOGGER_NAME = 'addrservice' 17 | -------------------------------------------------------------------------------- /addrservice/database/db_engines.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | from typing import Dict 4 | 5 | from addrservice.database.addressbook_db import ( 6 | AbstractAddressBookDB, InMemoryAddressBookDB, FilesystemAddressBookDB 7 | ) 8 | 9 | 10 | def create_addressbook_db(addr_db_config: Dict) -> AbstractAddressBookDB: 11 | db_type = list(addr_db_config.keys())[0] 12 | db_config = addr_db_config[db_type] 13 | 14 | return { 15 | 'memory': lambda cfg: InMemoryAddressBookDB(), 16 | 'fs': lambda cfg: FilesystemAddressBookDB(cfg), 17 | }[db_type](db_config) 18 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.pythonPath": ".venv/bin/python", 3 | "terminal.integrated.env.osx": { 4 | "PYTHONPATH": "${workspaceFolder}" 5 | }, 6 | "python.linting.flake8Enabled": true, 7 | "python.linting.pylintEnabled": false, 8 | "editor.formatOnSave": true, 9 | "cSpell.words": [ 10 | "addrservice" 11 | ], 12 | "python.testing.unittestEnabled": true, 13 | "python.testing.unittestArgs": [ 14 | "-v", 15 | "-s", 16 | "./tests/unit", 17 | "-p", 18 | "*_test.py" 19 | ], 20 | "python.testing.pytestEnabled": false, 21 | "python.testing.nosetestsEnabled": false 22 | } 23 | -------------------------------------------------------------------------------- /data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import glob 4 | import json 5 | import os 6 | from typing import Dict, Sequence 7 | 8 | ADDR_SERVICE_TEST_DATA_DIR = os.path.abspath(os.path.dirname(__file__)) 9 | 10 | 11 | ADDRESS_DATA_DIR = os.path.abspath(os.path.join( 12 | ADDR_SERVICE_TEST_DATA_DIR, 13 | 'addresses' 14 | )) 15 | 16 | ADDRESS_FILES = glob.glob(ADDRESS_DATA_DIR + '/*.json') 17 | 18 | 19 | def address_data_suite( 20 | json_files: Sequence[str] = ADDRESS_FILES 21 | ) -> Dict[str, Dict]: 22 | addr_data_suite = {} 23 | 24 | for fname in json_files: 25 | nickname = os.path.splitext(os.path.basename(fname))[0] 26 | with open(fname, mode='r', encoding='utf-8') as f: 27 | addr_json = json.load(f) 28 | addr_data_suite[nickname] = addr_json 29 | 30 | return addr_data_suite 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Satish Chandra Gupta 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/unit/address_data_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import jsonschema # type: ignore 4 | import unittest 5 | 6 | from addrservice import ADDRESS_BOOK_SCHEMA 7 | from data import address_data_suite 8 | import addrservice.datamodel as datamodel 9 | 10 | 11 | class AddressDataTest(unittest.TestCase): 12 | def setUp(self) -> None: 13 | super().setUp() 14 | self.address_data = address_data_suite() 15 | 16 | def test_json_schema(self) -> None: 17 | # Validate Address Schema 18 | jsonschema.Draft7Validator.check_schema(ADDRESS_BOOK_SCHEMA) 19 | 20 | def test_address_data_json(self) -> None: 21 | # Validate Address Test Data 22 | for nickname, addr in self.address_data.items(): 23 | # validate using application subschema 24 | jsonschema.validate(addr, ADDRESS_BOOK_SCHEMA) 25 | 26 | # Roundrtrip Test 27 | addr_obj = datamodel.AddressEntry.from_api_dm(addr) 28 | addr_dict = addr_obj.to_api_dm() 29 | self.assertEqual(addr, addr_dict) 30 | 31 | 32 | if __name__ == '__main__': 33 | unittest.main() 34 | -------------------------------------------------------------------------------- /data/addresses/raga.json: -------------------------------------------------------------------------------- 1 | { 2 | "full_name": "Rahul Gandhi", 3 | "addresses": [ 4 | { 5 | "kind": "work", 6 | "building_name": "Indian National Congress Office", 7 | "street_number": 24, 8 | "street_name": "Akbar Rd", 9 | "city": "New Delhi", 10 | "pincode": 110011, 11 | "country": "India" 12 | }, 13 | { 14 | "kind": "home", 15 | "street_number": 12, 16 | "street_name": "Tughlak Lane", 17 | "city": "New Delhi", 18 | "pincode": 110011, 19 | "country": "India" 20 | } 21 | ], 22 | "phone_numbers": [ 23 | { 24 | "kind": "work", 25 | "country_code": 91, 26 | "area_code": 11, 27 | "local_number": 23795161 28 | } 29 | ], 30 | "fax_numbers": [ 31 | { 32 | "kind": "work", 33 | "country_code": 91, 34 | "area_code": 11, 35 | "local_number": 23012410 36 | } 37 | ], 38 | "emails": [ 39 | { 40 | "kind": "work", 41 | "email": "office@rahulgandhi.in" 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /configs/addressbook-local.yaml: -------------------------------------------------------------------------------- 1 | service: 2 | name: Address Book 3 | 4 | addr-db: 5 | memory: null 6 | 7 | logging: 8 | version: 1 9 | formatters: 10 | brief: 11 | format: '%(asctime)s %(name)s %(levelname)s : %(message)s' 12 | detailed: 13 | format: 'time="%(asctime)s" logger="%(name)s" level="%(levelname)s" file="%(filename)s" lineno=%(lineno)d function="%(funcName)s" %(message)s' 14 | handlers: 15 | console: 16 | class: logging.StreamHandler 17 | level: INFO 18 | formatter: brief 19 | stream: ext://sys.stdout 20 | file: 21 | class : logging.handlers.RotatingFileHandler 22 | level: DEBUG 23 | formatter: detailed 24 | filename: /tmp/addrservice-app.log 25 | backupCount: 3 26 | loggers: 27 | addrservice: 28 | level: DEBUG 29 | handlers: 30 | - console 31 | - file 32 | propagate: no 33 | tornado.access: 34 | level: DEBUG 35 | handlers: 36 | - file 37 | tornado.application: 38 | level: DEBUG 39 | handlers: 40 | - file 41 | tornado.general: 42 | level: DEBUG 43 | handlers: 44 | - file 45 | root: 46 | level: WARNING 47 | handlers: 48 | - console 49 | -------------------------------------------------------------------------------- /addrservice/utils/logutils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import aiotask_context as context # type: ignore 4 | import logfmt # type: ignore 5 | import logging 6 | import re 7 | import traceback 8 | from typing import Dict 9 | 10 | LOG_CONTEXT = 'log_context' 11 | 12 | 13 | def get_log_context() -> Dict: 14 | log_context = context.get(LOG_CONTEXT) 15 | if log_context is None: 16 | log_context = {} 17 | context.set(LOG_CONTEXT, log_context) 18 | 19 | return log_context 20 | 21 | 22 | def set_log_context(**kwargs) -> None: 23 | log_context = get_log_context() 24 | log_context.update(kwargs) 25 | 26 | 27 | def clear_log_context() -> None: 28 | log_context = get_log_context() 29 | log_context.clear() 30 | 31 | 32 | def log( 33 | logger: logging.Logger, 34 | lvl: int, 35 | include_context: bool = False, 36 | **kwargs 37 | ) -> None: 38 | # Read https://docs.python.org/3/library/logging.html#logging.Logger.debug 39 | 40 | all_info = {**get_log_context(), **kwargs} if include_context else kwargs 41 | 42 | info = { 43 | k: v for k, v in all_info.items() 44 | if k not in ['exc_info', 'stack_info', 'extra'] 45 | } 46 | 47 | exc_info = all_info.get('exc_info') 48 | # stack_info = all_info.get('stack_info', False) 49 | # extra = all_info.get('extra', {}) 50 | 51 | if exc_info: # (typ, value, tb) 52 | trace = '\t'.join(traceback.format_exception(*exc_info)) 53 | info['trace'] = re.sub(r'[\r\n]+', '\t', trace) 54 | 55 | msg = next(logfmt.format(info)) 56 | logger.log( 57 | lvl, msg, 58 | # exc_info=exc_info, stack_info=stack_info, extra=extra 59 | ) 60 | -------------------------------------------------------------------------------- /data/addresses/namo.json: -------------------------------------------------------------------------------- 1 | { 2 | "full_name": "Narendra Modi", 3 | "addresses": [ 4 | { 5 | "kind": "work", 6 | "building_name": "Prime Minister’s Office", 7 | "street_name": "South Block", 8 | "locality": "Raisina Hill", 9 | "city": "New Delhi", 10 | "pincode": 110011, 11 | "country": "India" 12 | }, 13 | { 14 | "kind": "work", 15 | "building_name": "BJP Jansampark Karyalaya", 16 | "street_number": 77, 17 | "street_name": "Ravindrapuri Rd", 18 | "locality": "Bhelupur", 19 | "city": "Varanasi", 20 | "province": "Uttar Pradesh", 21 | "pincode": 221005, 22 | "country": "India" 23 | }, 24 | { 25 | "kind": "home", 26 | "street_number": 7, 27 | "street_name": "Race Course Road", 28 | "city": "New Delhi", 29 | "pincode": 110061, 30 | "country": "India" 31 | } 32 | ], 33 | "phone_numbers": [ 34 | { 35 | "kind": "work", 36 | "country_code": 91, 37 | "area_code": 11, 38 | "local_number": 23012312 39 | } 40 | ], 41 | "fax_numbers": [ 42 | { 43 | "kind": "work", 44 | "country_code": 91, 45 | "area_code": 11, 46 | "local_number": 23019545 47 | }, 48 | { 49 | "kind": "work", 50 | "country_code": 91, 51 | "area_code": 11, 52 | "local_number": 23016857 53 | } 54 | ], 55 | "emails": [ 56 | { 57 | "kind": "work", 58 | "email": "connect@mygov.nic.in" 59 | }, 60 | { 61 | "kind": "home", 62 | "email": "narendramodi1234@gmail.com" 63 | } 64 | ] 65 | } 66 | -------------------------------------------------------------------------------- /addrservice/service.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import jsonschema # type: ignore 4 | import logging 5 | from typing import AsyncIterator, Mapping, Tuple 6 | 7 | from addrservice import ADDRESS_BOOK_SCHEMA 8 | from addrservice.database.db_engines import create_addressbook_db 9 | from addrservice.datamodel import AddressEntry 10 | 11 | 12 | class AddressBookService: 13 | def __init__( 14 | self, 15 | config: Mapping, 16 | logger: logging.Logger 17 | ) -> None: 18 | self.addr_db = create_addressbook_db(config['addr-db']) 19 | self.logger = logger 20 | 21 | def start(self): 22 | self.addr_db.start() 23 | 24 | def stop(self): 25 | self.addr_db.stop() 26 | 27 | def validate_address(self, addr: Mapping) -> None: 28 | try: 29 | jsonschema.validate(addr, ADDRESS_BOOK_SCHEMA) 30 | except jsonschema.exceptions.ValidationError: 31 | raise ValueError('JSON Schema validation failed') 32 | 33 | async def create_address(self, value: Mapping) -> str: 34 | self.validate_address(value) 35 | addr = AddressEntry.from_api_dm(value) 36 | key = await self.addr_db.create_address(addr) 37 | return key 38 | 39 | async def get_address(self, key: str) -> Mapping: 40 | addr = await self.addr_db.read_address(key) 41 | return addr.to_api_dm() 42 | 43 | async def update_address(self, key: str, value: Mapping) -> None: 44 | self.validate_address(value) 45 | addr = AddressEntry.from_api_dm(value) 46 | await self.addr_db.update_address(key, addr) 47 | 48 | async def delete_address(self, key: str) -> None: 49 | await self.addr_db.delete_address(key) 50 | 51 | async def get_all_addresses(self) -> AsyncIterator[Tuple[str, Mapping]]: 52 | async for nickname, addr in self.addr_db.read_all_addresses(): 53 | yield nickname, addr.to_api_dm() 54 | -------------------------------------------------------------------------------- /tests/unit/tornado_app_handlers_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import aiotask_context as context # type: ignore 4 | import atexit 5 | from io import StringIO 6 | import json 7 | import logging 8 | import logging.config 9 | import yaml 10 | 11 | from tornado.ioloop import IOLoop 12 | import tornado.testing 13 | 14 | from addrservice import LOGGER_NAME 15 | from addrservice.tornado.app import make_addrservice_app 16 | 17 | from data import address_data_suite 18 | 19 | 20 | IN_MEMORY_CFG_TXT = ''' 21 | service: 22 | name: Address Book Test 23 | 24 | addr-db: 25 | memory: null 26 | 27 | logging: 28 | version: 1 29 | root: 30 | level: ERROR 31 | ''' 32 | 33 | with StringIO(IN_MEMORY_CFG_TXT) as f: 34 | TEST_CONFIG = yaml.load(f.read(), Loader=yaml.SafeLoader) 35 | 36 | 37 | class AddressServiceTornadoAppTestSetup(tornado.testing.AsyncHTTPTestCase): 38 | def setUp(self) -> None: 39 | super().setUp() 40 | self.headers = {'Content-Type': 'application/json; charset=UTF-8'} 41 | address_data = address_data_suite() 42 | keys = list(address_data.keys()) 43 | self.assertGreaterEqual(len(keys), 2) 44 | self.addr0 = address_data[keys[0]] 45 | self.addr1 = address_data[keys[1]] 46 | 47 | def get_app(self) -> tornado.web.Application: 48 | logging.config.dictConfig(TEST_CONFIG['logging']) 49 | logger = logging.getLogger(LOGGER_NAME) 50 | 51 | addr_service, app = make_addrservice_app( 52 | config=TEST_CONFIG, 53 | debug=True, 54 | logger=logger 55 | ) 56 | 57 | addr_service.start() 58 | atexit.register(lambda: addr_service.stop()) 59 | 60 | return app 61 | 62 | def get_new_ioloop(self): 63 | instance = IOLoop.current() 64 | instance.asyncio_loop.set_task_factory(context.task_factory) 65 | return instance 66 | 67 | 68 | class AddressServiceTornadoAppUnitTests(AddressServiceTornadoAppTestSetup): 69 | def test_default_handler(self): 70 | r = self.fetch( 71 | '/does-not-exist', 72 | method='GET', 73 | headers=None, 74 | ) 75 | info = json.loads(r.body.decode('utf-8')) 76 | 77 | self.assertEqual(r.code, 404, info) 78 | self.assertEqual(info['code'], 404) 79 | self.assertEqual(info['message'], 'Unknown Endpoint') 80 | 81 | 82 | if __name__ == '__main__': 83 | tornado.testing.main() 84 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /tests/integration/addrservice_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import asynctest # type: ignore 4 | from io import StringIO 5 | import logging 6 | import logging.config 7 | import unittest 8 | import yaml 9 | 10 | from addrservice import LOGGER_NAME 11 | from addrservice.datamodel import AddressEntry 12 | from addrservice.service import AddressBookService 13 | from data import address_data_suite 14 | 15 | IN_MEMORY_CFG_TXT = ''' 16 | service: 17 | name: Address Book Test 18 | 19 | addr-db: 20 | memory: null 21 | 22 | logging: 23 | version: 1 24 | root: 25 | level: ERROR 26 | ''' 27 | 28 | with StringIO(IN_MEMORY_CFG_TXT) as f: 29 | TEST_CONFIG = yaml.load(f.read(), Loader=yaml.SafeLoader) 30 | 31 | 32 | class AddressBookServiceWithInMemoryDBTest(asynctest.TestCase): 33 | async def setUp(self) -> None: 34 | logging.config.dictConfig(TEST_CONFIG['logging']) 35 | logger = logging.getLogger(LOGGER_NAME) 36 | 37 | self.service = AddressBookService( 38 | config=TEST_CONFIG, 39 | logger=logger 40 | ) 41 | self.service.start() 42 | 43 | self.address_data = address_data_suite() 44 | for nickname, val in self.address_data.items(): 45 | addr = AddressEntry.from_api_dm(val) 46 | await self.service.addr_db.create_address(addr, nickname) 47 | 48 | async def tearDown(self) -> None: 49 | self.service.stop() 50 | 51 | @asynctest.fail_on(active_handles=True) 52 | async def test_get_address(self) -> None: 53 | for nickname, addr in self.address_data.items(): 54 | value = await self.service.get_address(nickname) 55 | self.assertEqual(addr, value) 56 | 57 | @asynctest.fail_on(active_handles=True) 58 | async def test_get_all_addresses(self) -> None: 59 | all_addr = {} 60 | async for nickname, addr in self.service.get_all_addresses(): 61 | all_addr[nickname] = addr 62 | self.assertEqual(len(all_addr), 2) 63 | 64 | @asynctest.fail_on(active_handles=True) 65 | async def test_crud_address(self) -> None: 66 | nicknames = list(self.address_data.keys()) 67 | self.assertGreaterEqual(len(nicknames), 2) 68 | 69 | addr0 = self.address_data[nicknames[0]] 70 | key = await self.service.create_address(addr0) 71 | val = await self.service.get_address(key) 72 | self.assertEqual(addr0, val) 73 | 74 | addr1 = self.address_data[nicknames[1]] 75 | await self.service.update_address(key, addr1) 76 | val = await self.service.get_address(key) 77 | self.assertEqual(addr1, val) 78 | 79 | await self.service.delete_address(key) 80 | 81 | with self.assertRaises(KeyError): 82 | await self.service.get_address(key) 83 | 84 | 85 | if __name__ == '__main__': 86 | unittest.main() 87 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import os 5 | import subprocess 6 | from typing import List 7 | import unittest 8 | 9 | SOURCE_CODE = ['addrservice'] 10 | TEST_CODE = ['tests'] 11 | ALL_CODE = SOURCE_CODE + TEST_CODE 12 | 13 | 14 | def arg_parser() -> argparse.ArgumentParser: 15 | parser = argparse.ArgumentParser( 16 | description='Run linter, static type checker, tests' 17 | ) 18 | 19 | subparsers = parser.add_subparsers(dest='func', help='sub-commands') 20 | 21 | typechecker_cmd_parser = subparsers.add_parser('typecheck') 22 | typechecker_cmd_parser.add_argument( 23 | '-c', '--checker', 24 | default='mypy', 25 | help='specify static type checker, default: %(default)s' 26 | ) 27 | typechecker_cmd_parser.add_argument( 28 | 'paths', 29 | nargs='*', 30 | default=ALL_CODE, 31 | help='directories and files to check' 32 | ) 33 | 34 | lint_cmd_parser = subparsers.add_parser('lint') 35 | lint_cmd_parser.add_argument( 36 | '-l', '--linter', 37 | default='flake8', 38 | help='specify linter, default: %(default)s' 39 | ) 40 | lint_cmd_parser.add_argument( 41 | 'paths', 42 | nargs='*', 43 | default=ALL_CODE, 44 | help='directories and files to check' 45 | ) 46 | 47 | test_cmd_parser = subparsers.add_parser('test') 48 | test_cmd_parser.add_argument( 49 | '--suite', 50 | choices=['all', 'unit', 'integration'], 51 | default='all', 52 | type=str, 53 | help='test suite to run, default: %(default)s' 54 | ) 55 | test_cmd_parser.add_argument( 56 | '-v', '--verbose', 57 | action='store_true', 58 | help='turn on verbose output' 59 | ) 60 | 61 | return parser 62 | 63 | 64 | def run_checker(checker: str, paths: List[str]) -> None: 65 | if len(paths) != 0: 66 | subprocess.call([checker] + paths) 67 | 68 | 69 | def run_tests(suite_name: str, verbose: bool) -> None: 70 | test_suites = { 71 | 'all': 'tests', 72 | 'unit': 'tests/unit', 73 | 'integration': 'tests/integration' 74 | } 75 | suite = test_suites.get(suite_name, 'tests') 76 | 77 | verbosity = 2 if verbose else 1 78 | 79 | test_suite = unittest.TestLoader().discover(suite, pattern='*_test.py') 80 | unittest.TextTestRunner(verbosity=verbosity).run(test_suite) 81 | 82 | 83 | def main(args=None) -> None: 84 | os.chdir(os.path.abspath(os.path.dirname(__file__))) 85 | 86 | parser = arg_parser() 87 | args = parser.parse_args(args) 88 | # print(args) 89 | 90 | actions = { 91 | 'typecheck': lambda: run_checker(args.checker, args.paths), 92 | 'lint': lambda: run_checker(args.linter, args.paths), 93 | 'test': lambda: run_tests(args.suite, args.verbose), 94 | } 95 | 96 | actions.get(args.func, parser.print_help)() 97 | 98 | 99 | if __name__ == "__main__": 100 | main() 101 | -------------------------------------------------------------------------------- /addrservice/tornado/server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import aiotask_context as context # type: ignore 4 | import argparse 5 | import asyncio 6 | import logging 7 | import logging.config 8 | from typing import Dict 9 | import yaml 10 | 11 | import tornado.web 12 | 13 | from addrservice import LOGGER_NAME 14 | from addrservice.service import AddressBookService 15 | from addrservice.tornado.app import make_addrservice_app 16 | import addrservice.utils.logutils as logutils 17 | 18 | 19 | def parse_args(args=None): 20 | parser = argparse.ArgumentParser( 21 | description='Run Address Book Server' 22 | ) 23 | 24 | parser.add_argument( 25 | '-p', 26 | '--port', 27 | type=int, 28 | default=8080, 29 | help='port number for %(prog)s server to listen; ' 30 | 'default: %(default)s' 31 | ) 32 | 33 | parser.add_argument( 34 | '-d', 35 | '--debug', 36 | action='store_true', 37 | help='turn on debug logging' 38 | ) 39 | 40 | parser.add_argument( 41 | '-c', 42 | '--config', 43 | required=True, 44 | type=argparse.FileType('r'), 45 | help='config file for %(prog)s' 46 | ) 47 | 48 | args = parser.parse_args(args) 49 | return args 50 | 51 | 52 | def run_server( 53 | app: tornado.web.Application, 54 | service: AddressBookService, 55 | config: Dict, 56 | port: int, 57 | debug: bool, 58 | logger: logging.Logger 59 | ): 60 | name = config['service']['name'] 61 | loop = asyncio.get_event_loop() 62 | loop.set_task_factory(context.task_factory) 63 | 64 | # Start AddressBook service 65 | service.start() 66 | 67 | # Bind http server to port 68 | http_server_args = { 69 | 'decompress_request': True 70 | } 71 | http_server = app.listen(port, '', **http_server_args) 72 | logutils.log( 73 | logger, 74 | logging.INFO, 75 | message='STARTING', 76 | service_name=name, 77 | port=port 78 | ) 79 | 80 | try: 81 | # Start asyncio IO event loop 82 | loop.run_forever() 83 | except KeyboardInterrupt: 84 | # signal.SIGINT 85 | pass 86 | finally: 87 | loop.stop() 88 | logutils.log( 89 | logger, 90 | logging.INFO, 91 | message='SHUTTING DOWN', 92 | service_name=name 93 | ) 94 | http_server.stop() 95 | # loop.run_until_complete(asyncio.gather(*asyncio.Task.all_tasks())) 96 | loop.run_until_complete(loop.shutdown_asyncgens()) 97 | service.stop() 98 | loop.close() 99 | logutils.log( 100 | logger, 101 | logging.INFO, 102 | message='STOPPED', 103 | service_name=name 104 | ) 105 | 106 | 107 | def main(args=parse_args()): 108 | ''' 109 | Starts the Tornado server serving Address Book on the given port 110 | ''' 111 | 112 | config = yaml.load(args.config.read(), Loader=yaml.SafeLoader) 113 | 114 | # First thing: set logging config 115 | logging.config.dictConfig(config['logging']) 116 | logger = logging.getLogger(LOGGER_NAME) 117 | 118 | addr_service, addr_app = make_addrservice_app(config, args.debug, logger) 119 | 120 | run_server( 121 | app=addr_app, 122 | service=addr_service, 123 | config=config, 124 | port=args.port, 125 | debug=args.debug, 126 | logger=logger 127 | ) 128 | 129 | 130 | if __name__ == '__main__': 131 | main() 132 | -------------------------------------------------------------------------------- /schema/address-book-v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "title": "Schema of an address book", 4 | "definitions": { 5 | "kind": { 6 | "type": "string", 7 | "enum": [ 8 | "home", 9 | "work" 10 | ] 11 | }, 12 | "address": { 13 | "type": "object", 14 | "properties": { 15 | "kind": { 16 | "$ref": "#/definitions/kind" 17 | }, 18 | "building_name": { 19 | "type": "string" 20 | }, 21 | "unit_number": { 22 | "type": "number" 23 | }, 24 | "street_number": { 25 | "type": [ 26 | "number", 27 | "string" 28 | ] 29 | }, 30 | "street_name": { 31 | "type": "string" 32 | }, 33 | "locality": { 34 | "type": "string" 35 | }, 36 | "city": { 37 | "type": "string" 38 | }, 39 | "province": { 40 | "type": "string" 41 | }, 42 | "pincode": { 43 | "type": [ 44 | "number", 45 | "string" 46 | ] 47 | }, 48 | "country": { 49 | "type": "string" 50 | } 51 | }, 52 | "required": [ 53 | "kind", 54 | "street_name", 55 | "pincode", 56 | "country" 57 | ], 58 | "additionalProperties": false 59 | }, 60 | "phone": { 61 | "type": "object", 62 | "properties": { 63 | "kind": { 64 | "$ref": "#/definitions/kind" 65 | }, 66 | "country_code": { 67 | "type": "number" 68 | }, 69 | "area_code": { 70 | "type": "number" 71 | }, 72 | "local_number": { 73 | "type": "number" 74 | } 75 | }, 76 | "required": [ 77 | "kind", 78 | "country_code", 79 | "local_number" 80 | ], 81 | "additionalProperties": false 82 | }, 83 | "email": { 84 | "type": "object", 85 | "properties": { 86 | "kind": { 87 | "$ref": "#/definitions/kind" 88 | }, 89 | "email": { 90 | "type": "string", 91 | "format": "email" 92 | } 93 | }, 94 | "required": [ 95 | "kind", 96 | "email" 97 | ], 98 | "additionalProperties": false 99 | }, 100 | "addressEntry": { 101 | "$id": "#addressEntry", 102 | "type": "object", 103 | "properties": { 104 | "full_name": { 105 | "type": "string" 106 | }, 107 | "addresses": { 108 | "type": "array", 109 | "minItems": 0, 110 | "items": { 111 | "$ref": "#/definitions/address" 112 | } 113 | }, 114 | "phone_numbers": { 115 | "type": "array", 116 | "minItems": 0, 117 | "items": { 118 | "$ref": "#/definitions/phone" 119 | } 120 | }, 121 | "fax_numbers": { 122 | "type": "array", 123 | "minItems": 0, 124 | "items": { 125 | "$ref": "#/definitions/phone" 126 | } 127 | }, 128 | "emails": { 129 | "type": "array", 130 | "minItems": 0, 131 | "items": { 132 | "$ref": "#/definitions/email" 133 | } 134 | } 135 | }, 136 | "required": [ 137 | "full_name" 138 | ], 139 | "additionalProperties": false 140 | } 141 | }, 142 | "$ref": "#/definitions/addressEntry" 143 | } 144 | -------------------------------------------------------------------------------- /tests/integration/tornado_app_addreservice_handlers_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import json 4 | 5 | import tornado.testing 6 | 7 | from addrservice.tornado.app import ( 8 | ADDRESSBOOK_ENTRY_URI_FORMAT_STR 9 | ) 10 | 11 | from tests.unit.tornado_app_handlers_test import ( 12 | AddressServiceTornadoAppTestSetup 13 | ) 14 | 15 | 16 | class TestAddressServiceApp(AddressServiceTornadoAppTestSetup): 17 | def test_address_book_endpoints(self): 18 | # Get all addresses in the address book, must be ZERO 19 | r = self.fetch( 20 | ADDRESSBOOK_ENTRY_URI_FORMAT_STR.format(id=''), 21 | method='GET', 22 | headers=None, 23 | ) 24 | all_addrs = json.loads(r.body.decode('utf-8')) 25 | self.assertEqual(r.code, 200, all_addrs) 26 | self.assertEqual(len(all_addrs), 0, all_addrs) 27 | 28 | # Add an address 29 | r = self.fetch( 30 | ADDRESSBOOK_ENTRY_URI_FORMAT_STR.format(id=''), 31 | method='POST', 32 | headers=self.headers, 33 | body=json.dumps(self.addr0), 34 | ) 35 | self.assertEqual(r.code, 201) 36 | addr_uri = r.headers['Location'] 37 | 38 | # Get all addresses in the address book, must be ZERO 39 | r = self.fetch( 40 | ADDRESSBOOK_ENTRY_URI_FORMAT_STR.format(id=''), 41 | method='GET', 42 | headers=None, 43 | ) 44 | all_addrs = json.loads(r.body.decode('utf-8')) 45 | self.assertEqual(r.code, 200, all_addrs) 46 | self.assertEqual(len(all_addrs), 1, all_addrs) 47 | 48 | # POST: error cases 49 | r = self.fetch( 50 | ADDRESSBOOK_ENTRY_URI_FORMAT_STR.format(id=''), 51 | method='POST', 52 | headers=self.headers, 53 | body='it is not json', 54 | ) 55 | self.assertEqual(r.code, 400) 56 | self.assertEqual(r.reason, 'Invalid JSON body') 57 | r = self.fetch( 58 | ADDRESSBOOK_ENTRY_URI_FORMAT_STR.format(id=''), 59 | method='POST', 60 | headers=self.headers, 61 | body=json.dumps({}), 62 | ) 63 | self.assertEqual(r.code, 400) 64 | self.assertEqual(r.reason, 'JSON Schema validation failed') 65 | 66 | # Get the added address 67 | r = self.fetch( 68 | addr_uri, 69 | method='GET', 70 | headers=None, 71 | ) 72 | self.assertEqual(r.code, 200) 73 | self.assertEqual(self.addr0, json.loads(r.body.decode('utf-8'))) 74 | 75 | # GET: error cases 76 | r = self.fetch( 77 | ADDRESSBOOK_ENTRY_URI_FORMAT_STR.format(id='no-such-id'), 78 | method='GET', 79 | headers=None, 80 | ) 81 | self.assertEqual(r.code, 404) 82 | 83 | # Update that address 84 | r = self.fetch( 85 | addr_uri, 86 | method='PUT', 87 | headers=self.headers, 88 | body=json.dumps(self.addr1), 89 | ) 90 | self.assertEqual(r.code, 204) 91 | r = self.fetch( 92 | addr_uri, 93 | method='GET', 94 | headers=None, 95 | ) 96 | self.assertEqual(r.code, 200) 97 | self.assertEqual(self.addr1, json.loads(r.body.decode('utf-8'))) 98 | 99 | # PUT: error cases 100 | r = self.fetch( 101 | addr_uri, 102 | method='PUT', 103 | headers=self.headers, 104 | body='it is not json', 105 | ) 106 | self.assertEqual(r.code, 400) 107 | self.assertEqual(r.reason, 'Invalid JSON body') 108 | r = self.fetch( 109 | ADDRESSBOOK_ENTRY_URI_FORMAT_STR.format(id='1234'), 110 | method='PUT', 111 | headers=self.headers, 112 | body=json.dumps(self.addr1), 113 | ) 114 | self.assertEqual(r.code, 404) 115 | r = self.fetch( 116 | addr_uri, 117 | method='PUT', 118 | headers=self.headers, 119 | body=json.dumps({}), 120 | ) 121 | self.assertEqual(r.code, 400) 122 | self.assertEqual(r.reason, 'JSON Schema validation failed') 123 | 124 | # Delete that address 125 | r = self.fetch( 126 | addr_uri, 127 | method='DELETE', 128 | headers=None, 129 | ) 130 | self.assertEqual(r.code, 204) 131 | r = self.fetch( 132 | addr_uri, 133 | method='GET', 134 | headers=None, 135 | ) 136 | self.assertEqual(r.code, 404) 137 | 138 | # DELETE: error cases 139 | r = self.fetch( 140 | addr_uri, 141 | method='DELETE', 142 | headers=None, 143 | ) 144 | self.assertEqual(r.code, 404) 145 | 146 | # Get all addresses in the address book, must be ZERO 147 | r = self.fetch( 148 | ADDRESSBOOK_ENTRY_URI_FORMAT_STR.format(id=''), 149 | method='GET', 150 | headers=None, 151 | ) 152 | all_addrs = json.loads(r.body.decode('utf-8')) 153 | self.assertEqual(r.code, 200, all_addrs) 154 | self.assertEqual(len(all_addrs), 0, all_addrs) 155 | 156 | 157 | if __name__ == '__main__': 158 | tornado.testing.main() 159 | -------------------------------------------------------------------------------- /tests/unit/addressbook_db_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | from abc import ABCMeta, abstractmethod 4 | import asynctest # type: ignore 5 | from io import StringIO 6 | import os 7 | import tempfile 8 | from typing import Dict 9 | import unittest 10 | import yaml 11 | 12 | from addrservice.database.addressbook_db import ( 13 | AbstractAddressBookDB, InMemoryAddressBookDB, FilesystemAddressBookDB 14 | ) 15 | from addrservice.database.db_engines import create_addressbook_db 16 | from addrservice.datamodel import AddressEntry 17 | 18 | from data import address_data_suite 19 | 20 | 21 | class AbstractAddressBookDBTest(unittest.TestCase): 22 | def read_config(self, txt: str) -> Dict: 23 | with StringIO(txt) as f: 24 | cfg = yaml.load(f.read(), Loader=yaml.SafeLoader) 25 | return cfg 26 | 27 | def test_in_memory_db_config(self): 28 | cfg = self.read_config(''' 29 | addr-db: 30 | memory: null 31 | ''') 32 | 33 | self.assertIn('memory', cfg['addr-db']) 34 | db = create_addressbook_db(cfg['addr-db']) 35 | self.assertEqual(type(db), InMemoryAddressBookDB) 36 | 37 | def test_file_system_db_config(self): 38 | cfg = self.read_config(''' 39 | addr-db: 40 | fs: /tmp 41 | ''') 42 | 43 | self.assertIn('fs', cfg['addr-db']) 44 | db = create_addressbook_db(cfg['addr-db']) 45 | self.assertEqual(type(db), FilesystemAddressBookDB) 46 | self.assertEqual(db.store, '/tmp') 47 | 48 | 49 | class AbstractAddressBookDBTestCase(metaclass=ABCMeta): 50 | def setUp(self) -> None: 51 | self.address_data = { 52 | k: AddressEntry.from_api_dm(v) 53 | for k, v in address_data_suite().items() 54 | } 55 | self.addr_db = self.make_addr_db() 56 | 57 | @abstractmethod 58 | def make_addr_db(self) -> AbstractAddressBookDB: 59 | raise NotImplementedError() 60 | 61 | @abstractmethod 62 | def addr_count(self) -> int: 63 | raise NotImplementedError() 64 | 65 | @asynctest.fail_on(active_handles=True) 66 | async def test_crud_lifecycle(self) -> None: 67 | # Nothing in the database 68 | for nickname in self.address_data: 69 | with self.assertRaises(KeyError): # type: ignore 70 | await self.addr_db.read_address(nickname) 71 | 72 | # Create then Read, again Create(fail) 73 | for nickname, addr in self.address_data.items(): 74 | await self.addr_db.create_address(addr, nickname) 75 | await self.addr_db.read_address(nickname) 76 | with self.assertRaises(KeyError): # type: ignore 77 | await self.addr_db.create_address(addr, nickname) 78 | 79 | self.assertEqual(self.addr_count(), 2) # type: ignore 80 | 81 | # First data in test set 82 | first_nickname = list(self.address_data.keys())[0] 83 | first_addr = self.address_data[first_nickname] 84 | 85 | # Update 86 | await self.addr_db.update_address(first_nickname, first_addr) 87 | with self.assertRaises(KeyError): # type: ignore 88 | await self.addr_db.update_address('does not exist', first_addr) 89 | 90 | # Create without giving nickname 91 | new_nickname = await self.addr_db.create_address(addr) 92 | self.assertIsNotNone(new_nickname) # type: ignore 93 | self.assertEqual(self.addr_count(), 3) # type: ignore 94 | 95 | # Get All Addresses 96 | addresses = {} 97 | async for nickname, addr in self.addr_db.read_all_addresses(): 98 | addresses[nickname] = addr 99 | 100 | self.assertEqual(len(addresses), 3) # type: ignore 101 | 102 | # Delete then Read, and the again Delete 103 | for nickname in self.address_data: 104 | await self.addr_db.delete_address(nickname) 105 | with self.assertRaises(KeyError): # type: ignore 106 | await self.addr_db.read_address(nickname) 107 | with self.assertRaises(KeyError): # type: ignore 108 | await self.addr_db.delete_address(nickname) 109 | 110 | self.assertEqual(self.addr_count(), 1) # type: ignore 111 | 112 | await self.addr_db.delete_address(new_nickname) 113 | self.assertEqual(self.addr_count(), 0) # type: ignore 114 | 115 | 116 | class InMemoryAddressBookDBTest( 117 | AbstractAddressBookDBTestCase, 118 | asynctest.TestCase 119 | ): 120 | def make_addr_db(self) -> AbstractAddressBookDB: 121 | self.mem_db = InMemoryAddressBookDB() 122 | return self.mem_db 123 | 124 | def addr_count(self) -> int: 125 | return len(self.mem_db.db) 126 | 127 | 128 | class FilesystemAddressBookDBTest( 129 | AbstractAddressBookDBTestCase, 130 | asynctest.TestCase 131 | ): 132 | def make_addr_db(self) -> AbstractAddressBookDB: 133 | self.tmp_dir = tempfile.TemporaryDirectory(prefix='addrbook-fsdb') 134 | self.store_dir = self.tmp_dir.name 135 | self.fs_db = FilesystemAddressBookDB(self.store_dir) 136 | return self.fs_db 137 | 138 | def addr_count(self) -> int: 139 | return len([ 140 | name for name in os.listdir(self.store_dir) 141 | if os.path.isfile(os.path.join(self.store_dir, name)) 142 | ]) 143 | return len(self.addr_db.db) 144 | 145 | def tearDown(self): 146 | self.tmp_dir.cleanup() 147 | super().tearDown() 148 | 149 | async def test_db_creation(self): 150 | with tempfile.TemporaryDirectory(prefix='addrbook-fsdb') as tempdir: 151 | store_dir = os.path.join(tempdir, 'abc') 152 | FilesystemAddressBookDB(store_dir) 153 | tmpfilename = os.path.join(tempdir, 'xyz.txt') 154 | with open(tmpfilename, 'w') as f: 155 | f.write('this is a file and not a dir') 156 | with self.assertRaises(ValueError): 157 | FilesystemAddressBookDB(tmpfilename) 158 | 159 | 160 | if __name__ == '__main__': 161 | unittest.main() 162 | -------------------------------------------------------------------------------- /addrservice/database/addressbook_db.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | from abc import ABCMeta, abstractmethod 4 | import aiofiles # type: ignore 5 | import json 6 | import os 7 | from typing import AsyncIterator, Dict, Mapping, Tuple 8 | import uuid 9 | 10 | from addrservice.datamodel import AddressEntry 11 | 12 | 13 | class AbstractAddressBookDB(metaclass=ABCMeta): 14 | def start(self): 15 | pass 16 | 17 | def stop(self): 18 | pass 19 | 20 | # CRUD 21 | 22 | @abstractmethod 23 | async def create_address( 24 | self, 25 | addr: AddressEntry, 26 | nickname: str = None 27 | ) -> str: 28 | raise NotImplementedError() 29 | 30 | @abstractmethod 31 | async def read_address(self, nickname: str) -> AddressEntry: 32 | raise NotImplementedError() 33 | 34 | @abstractmethod 35 | async def update_address(self, nickname: str, addr: AddressEntry) -> None: 36 | raise NotImplementedError() 37 | 38 | @abstractmethod 39 | async def delete_address(self, nickname: str) -> None: 40 | raise NotImplementedError() 41 | 42 | @abstractmethod 43 | def read_all_addresses(self) -> AsyncIterator[Tuple[str, AddressEntry]]: 44 | raise NotImplementedError() 45 | 46 | 47 | class InMemoryAddressBookDB(AbstractAddressBookDB): 48 | def __init__(self): 49 | self.db: Dict[str, AddressEntry] = {} 50 | 51 | async def create_address( 52 | self, 53 | addr: AddressEntry, 54 | nickname: str = None 55 | ) -> str: 56 | if nickname is None: 57 | nickname = uuid.uuid4().hex 58 | 59 | if nickname in self.db: 60 | raise KeyError('{} already exists'.format(nickname)) 61 | 62 | self.db[nickname] = addr 63 | return nickname 64 | 65 | async def read_address(self, nickname: str) -> AddressEntry: 66 | return self.db[nickname] 67 | 68 | async def update_address(self, nickname: str, addr: AddressEntry) -> None: 69 | if nickname is None or nickname not in self.db: 70 | raise KeyError('{} does not exist'.format(nickname)) 71 | 72 | self.db[nickname] = addr 73 | 74 | async def delete_address(self, nickname: str) -> None: 75 | if nickname is None or nickname not in self.db: 76 | raise KeyError('{} does not exist'.format(nickname)) 77 | 78 | del self.db[nickname] 79 | 80 | async def read_all_addresses( 81 | self 82 | ) -> AsyncIterator[Tuple[str, AddressEntry]]: 83 | for nickname, addr in self.db.items(): 84 | yield nickname, addr 85 | 86 | 87 | class FilesystemAddressBookDB(AbstractAddressBookDB): 88 | def __init__(self, store_dir_path: str): 89 | store_dir = os.path.abspath(store_dir_path) 90 | if not os.path.exists(store_dir): 91 | os.makedirs(store_dir) 92 | if not (os.path.isdir(store_dir) and os.access(store_dir, os.W_OK)): 93 | raise ValueError( 94 | 'String store "{}" is not a writable directory'.format( 95 | store_dir 96 | ) 97 | ) 98 | self._store = store_dir 99 | 100 | @property 101 | def store(self) -> str: 102 | return self._store 103 | 104 | def _file_name(self, nickname: str) -> str: 105 | return os.path.join( 106 | self.store, 107 | nickname + '.json' 108 | ) 109 | 110 | def _file_exists(self, nickname: str) -> bool: 111 | return os.path.exists(self._file_name(nickname)) 112 | 113 | async def _file_read(self, nickname: str) -> Dict: 114 | try: 115 | async with aiofiles.open( 116 | self._file_name(nickname), 117 | encoding='utf-8', 118 | mode='r' 119 | ) as f: 120 | contents = await f.read() 121 | return json.loads(contents) 122 | except FileNotFoundError: 123 | raise KeyError(nickname) 124 | 125 | async def _file_write(self, nickname: str, addr: Mapping) -> None: 126 | async with aiofiles.open( 127 | self._file_name(nickname), 128 | mode='w', 129 | encoding='utf-8' 130 | ) as f: 131 | await f.write(json.dumps(addr)) 132 | 133 | async def _file_delete(self, nickname: str) -> None: 134 | os.remove(self._file_name(nickname)) 135 | 136 | async def _file_read_all(self) -> AsyncIterator[Tuple[str, Dict]]: 137 | all_files = os.listdir(self.store) 138 | extn_end = '.json' 139 | extn_len = len(extn_end) 140 | for f in all_files: 141 | if f.endswith(extn_end): 142 | nickname = f[:-extn_len] 143 | addr = await self._file_read(nickname) 144 | yield nickname, addr 145 | 146 | async def create_address( 147 | self, 148 | addr: AddressEntry, 149 | nickname: str = None 150 | ) -> str: 151 | if nickname is None: 152 | nickname = uuid.uuid4().hex 153 | 154 | if self._file_exists(nickname): 155 | raise KeyError('{} already exists'.format(nickname)) 156 | 157 | await self._file_write(nickname, addr.to_api_dm()) 158 | return nickname 159 | 160 | async def read_address(self, nickname: str) -> AddressEntry: 161 | addr = await self._file_read(nickname) 162 | return AddressEntry.from_api_dm(addr) 163 | 164 | async def update_address(self, nickname: str, addr: AddressEntry) -> None: 165 | if self._file_exists(nickname): 166 | await self._file_write(nickname, addr.to_api_dm()) 167 | else: 168 | raise KeyError(nickname) 169 | 170 | async def delete_address(self, nickname: str) -> None: 171 | if self._file_exists(nickname): 172 | await self._file_delete(nickname) 173 | else: 174 | raise KeyError(nickname) 175 | 176 | async def read_all_addresses( 177 | self 178 | ) -> AsyncIterator[Tuple[str, AddressEntry]]: 179 | async for nickname, addr in self._file_read_all(): 180 | yield nickname, AddressEntry.from_api_dm(addr) 181 | -------------------------------------------------------------------------------- /addrservice/tornado/app.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import json 4 | import logging 5 | from types import TracebackType 6 | from typing import ( 7 | Any, 8 | Awaitable, 9 | Dict, 10 | Optional, 11 | Tuple, 12 | Type, 13 | ) 14 | import traceback 15 | import uuid 16 | 17 | import tornado.web 18 | 19 | from addrservice import LOGGER_NAME 20 | from addrservice.service import AddressBookService 21 | import addrservice.utils.logutils as logutils 22 | 23 | ADDRESSBOOK_REGEX = r'/addresses/?' 24 | ADDRESSBOOK_ENTRY_REGEX = r'/addresses/(?P[a-zA-Z0-9-]+)/?' 25 | ADDRESSBOOK_ENTRY_URI_FORMAT_STR = r'/addresses/{id}' 26 | 27 | 28 | class BaseRequestHandler(tornado.web.RequestHandler): 29 | def initialize( 30 | self, 31 | service: AddressBookService, 32 | config: Dict, 33 | logger: logging.Logger 34 | ) -> None: 35 | self.service = service 36 | self.config = config 37 | self.logger = logger 38 | 39 | def prepare(self) -> Optional[Awaitable[None]]: 40 | req_id = uuid.uuid4().hex 41 | logutils.set_log_context( 42 | req_id=req_id, 43 | method=self.request.method, 44 | uri=self.request.uri, 45 | ip=self.request.remote_ip 46 | ) 47 | 48 | logutils.log( 49 | self.logger, 50 | logging.DEBUG, 51 | include_context=True, 52 | message='REQUEST' 53 | ) 54 | 55 | return super().prepare() 56 | 57 | def on_finish(self) -> None: 58 | super().on_finish() 59 | 60 | def write_error(self, status_code: int, **kwargs: Any) -> None: 61 | self.set_header('Content-Type', 'application/json; charset=UTF-8') 62 | body = { 63 | 'method': self.request.method, 64 | 'uri': self.request.path, 65 | 'code': status_code, 66 | 'message': self._reason 67 | } 68 | 69 | logutils.set_log_context(reason=self._reason) 70 | 71 | if 'exc_info' in kwargs: 72 | exc_info = kwargs['exc_info'] 73 | logutils.set_log_context(exc_info=exc_info) 74 | if self.settings.get('serve_traceback'): 75 | # in debug mode, send a traceback 76 | trace = '\n'.join(traceback.format_exception(*exc_info)) 77 | body['trace'] = trace 78 | 79 | self.finish(body) 80 | 81 | def log_exception( 82 | self, 83 | typ: Optional[Type[BaseException]], 84 | value: Optional[BaseException], 85 | tb: Optional[TracebackType], 86 | ) -> None: 87 | # https://www.tornadoweb.org/en/stable/web.html#tornado.web.RequestHandler.log_exception 88 | if isinstance(value, tornado.web.HTTPError): 89 | if value.log_message: 90 | msg = value.log_message % value.args 91 | logutils.log( 92 | tornado.log.gen_log, 93 | logging.WARNING, 94 | status=value.status_code, 95 | request_summary=self._request_summary(), 96 | message=msg 97 | ) 98 | else: 99 | logutils.log( 100 | tornado.log.app_log, 101 | logging.ERROR, 102 | message='Uncaught exception', 103 | request_summary=self._request_summary(), 104 | request=repr(self.request), 105 | exc_info=(typ, value, tb) 106 | ) 107 | 108 | 109 | class DefaultRequestHandler(BaseRequestHandler): 110 | def initialize( # type: ignore 111 | self, 112 | status_code: int, 113 | message: str, 114 | logger: logging.Logger 115 | ): 116 | self.logger = logger 117 | self.set_status(status_code, reason=message) 118 | 119 | def prepare(self) -> Optional[Awaitable[None]]: 120 | raise tornado.web.HTTPError( 121 | self._status_code, 122 | 'request uri: %s', 123 | self.request.uri, 124 | reason=self._reason 125 | ) 126 | 127 | 128 | class AddressBookRequestHandler(BaseRequestHandler): 129 | async def get(self): 130 | all_addrs = {} 131 | async for nickname, addr in self.service.get_all_addresses(): 132 | all_addrs[nickname] = addr 133 | 134 | self.set_status(200) 135 | self.finish(all_addrs) 136 | 137 | async def post(self): 138 | try: 139 | addr = json.loads(self.request.body.decode('utf-8')) 140 | id = await self.service.create_address(addr) 141 | addr_uri = ADDRESSBOOK_ENTRY_URI_FORMAT_STR.format(id=id) 142 | self.set_status(201) 143 | self.set_header('Location', addr_uri) 144 | self.finish() 145 | except (json.decoder.JSONDecodeError, TypeError): 146 | raise tornado.web.HTTPError( 147 | 400, reason='Invalid JSON body' 148 | ) 149 | except ValueError as e: 150 | raise tornado.web.HTTPError(400, reason=str(e)) 151 | 152 | 153 | class AddressBookEntryRequestHandler(BaseRequestHandler): 154 | async def get(self, id): 155 | try: 156 | addr = await self.service.get_address(id) 157 | self.set_status(200) 158 | self.finish(addr) 159 | except KeyError as e: 160 | raise tornado.web.HTTPError(404, reason=str(e)) 161 | 162 | async def put(self, id): 163 | try: 164 | addr = json.loads(self.request.body.decode('utf-8')) 165 | await self.service.update_address(id, addr) 166 | self.set_status(204) 167 | self.finish() 168 | except (json.decoder.JSONDecodeError, TypeError): 169 | raise tornado.web.HTTPError( 170 | 400, reason='Invalid JSON body' 171 | ) 172 | except KeyError as e: 173 | raise tornado.web.HTTPError(404, reason=str(e)) 174 | except ValueError as e: 175 | raise tornado.web.HTTPError(400, reason=str(e)) 176 | 177 | async def delete(self, id): 178 | try: 179 | await self.service.delete_address(id) 180 | self.set_status(204) 181 | self.finish() 182 | except KeyError as e: 183 | raise tornado.web.HTTPError(404, reason=str(e)) 184 | 185 | 186 | def log_function(handler: tornado.web.RequestHandler) -> None: 187 | # https://www.tornadoweb.org/en/stable/web.html#tornado.web.Application.settings 188 | 189 | logger = getattr(handler, 'logger', logging.getLogger(LOGGER_NAME)) 190 | 191 | if handler.get_status() < 400: 192 | level = logging.INFO 193 | elif handler.get_status() < 500: 194 | level = logging.WARNING 195 | else: 196 | level = logging.ERROR 197 | 198 | logutils.log( 199 | logger, 200 | level, 201 | include_context=True, 202 | message='RESPONSE', 203 | status=handler.get_status(), 204 | time_ms=(1000.0 * handler.request.request_time()) 205 | ) 206 | 207 | logutils.clear_log_context() 208 | 209 | 210 | def make_addrservice_app( 211 | config: Dict, 212 | debug: bool, 213 | logger: logging.Logger 214 | ) -> Tuple[AddressBookService, tornado.web.Application]: 215 | service = AddressBookService(config, logger) 216 | 217 | app = tornado.web.Application( 218 | [ 219 | # Address Book endpoints 220 | (ADDRESSBOOK_REGEX, AddressBookRequestHandler, 221 | dict(service=service, config=config, logger=logger)), 222 | (ADDRESSBOOK_ENTRY_REGEX, AddressBookEntryRequestHandler, 223 | dict(service=service, config=config, logger=logger)) 224 | ], 225 | compress_response=True, # compress textual responses 226 | log_function=log_function, # log_request() uses it to log results 227 | serve_traceback=debug, # it is passed on as setting to write_error() 228 | default_handler_class=DefaultRequestHandler, 229 | default_handler_args={ 230 | 'status_code': 404, 231 | 'message': 'Unknown Endpoint', 232 | 'logger': logger 233 | } 234 | ) 235 | 236 | return service, app 237 | -------------------------------------------------------------------------------- /tests/unit/datamodel_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | import unittest 4 | 5 | from addrservice.datamodel import ( 6 | AddressType, Address, Phone, Email, AddressEntry 7 | ) 8 | 9 | 10 | class DataModelTest(unittest.TestCase): 11 | def test_data_model(self) -> None: 12 | address = Address( 13 | kind=AddressType.work, 14 | building_name='Data Model', 15 | unit_number=101, 16 | street_number=4, 17 | street_name='Microservices Ave', 18 | locality='Python', 19 | city='Bangalore', 20 | province='Karnataka', 21 | pincode=560001, 22 | country='India' 23 | ) 24 | self.assertEqual(address.kind, AddressType.work) 25 | self.assertEqual(address.building_name, 'Data Model') 26 | self.assertEqual(address.unit_number, 101) 27 | self.assertEqual(address.street_number, 4) 28 | self.assertEqual(address.street_name, 'Microservices Ave') 29 | self.assertEqual(address.locality, 'Python') 30 | self.assertEqual(address.city, 'Bangalore') 31 | self.assertEqual(address.province, 'Karnataka') 32 | self.assertEqual(address.pincode, 560001) 33 | self.assertEqual(address.country, 'India') 34 | 35 | phone = Phone( 36 | kind=AddressType.home, 37 | country_code=91, 38 | local_number=9876543210 39 | ) 40 | self.assertEqual(phone.kind, AddressType.home) 41 | self.assertEqual(phone.country_code, 91) 42 | self.assertEqual(phone.area_code, None) 43 | self.assertEqual(phone.local_number, 9876543210) 44 | 45 | fax = Phone( 46 | kind=AddressType.work, 47 | country_code=91, 48 | area_code=80, 49 | local_number=12345678 50 | ) 51 | self.assertEqual(fax.kind, AddressType.work) 52 | self.assertEqual(fax.country_code, 91) 53 | self.assertEqual(fax.area_code, 80) 54 | self.assertEqual(fax.local_number, 12345678) 55 | 56 | email = Email( 57 | kind=AddressType.work, 58 | email='datamodel@microservices.py' 59 | ) 60 | 61 | address_entry = AddressEntry( 62 | full_name='Data Model', 63 | addresses=[address], 64 | phone_numbers=[phone], 65 | fax_numbers=[fax], 66 | emails=[email] 67 | ) 68 | self.assertEqual(address_entry.full_name, 'Data Model') 69 | self.assertEqual(len(address_entry.addresses), 1) 70 | self.assertEqual(address_entry.addresses[0], address) 71 | self.assertEqual(len(address_entry.phone_numbers), 1) 72 | self.assertEqual(address_entry.phone_numbers[0], phone) 73 | self.assertEqual(len(address_entry.fax_numbers), 1) 74 | self.assertEqual(address_entry.fax_numbers[0], fax) 75 | self.assertEqual(len(address_entry.emails), 1) 76 | self.assertEqual(address_entry.emails[0], email) 77 | 78 | address_dict_1 = address_entry.to_api_dm() 79 | address_dict_2 = AddressEntry.from_api_dm(address_dict_1).to_api_dm() 80 | self.assertEqual(address_dict_1, address_dict_2) 81 | 82 | # Setters 83 | 84 | address.kind = AddressType.home 85 | address.building_name = 'Abc' 86 | address.unit_number = 1 87 | address.street_number = 2 88 | address.street_name = 'Xyz' 89 | address.locality = 'Pqr' 90 | address.city = 'Nowhere' 91 | address.province = 'Wierd' 92 | address.pincode = '0X01' 93 | address.country = 'Forsaken' 94 | 95 | phone.kind = AddressType.work 96 | phone.country_code = 1 97 | phone.area_code = 123 98 | phone.local_number = 4567890 99 | 100 | email.kind = AddressType.home 101 | email.email = 'abc@example.com' 102 | 103 | address_entry.full_name = 'Abc Xyz' 104 | address_entry.addresses = [] 105 | address_entry.phone_numbers = [] 106 | address_entry.fax_numbers = [] 107 | address_entry.emails = [] 108 | 109 | # Exceptions 110 | 111 | with self.assertRaises(ValueError): 112 | Address( 113 | kind=None, # type: ignore 114 | street_name='Microservices Ave', 115 | pincode=560001, 116 | country='India' 117 | ) 118 | 119 | with self.assertRaises(ValueError): 120 | Address( 121 | kind=AddressType.work, 122 | street_name=None, # type: ignore 123 | pincode=560001, 124 | country='India' 125 | ) 126 | 127 | with self.assertRaises(ValueError): 128 | Address( 129 | kind=AddressType.work, 130 | street_name='Microservices Ave', 131 | pincode=None, # type: ignore 132 | country='India' 133 | ) 134 | 135 | with self.assertRaises(ValueError): 136 | Address( 137 | kind=AddressType.work, 138 | street_name='Microservices Ave', 139 | pincode=560001, 140 | country=None # type: ignore 141 | ) 142 | 143 | addr = Address( 144 | kind=AddressType.work, 145 | street_name='Microservices Ave', 146 | pincode=560001, 147 | country='India' 148 | ) 149 | 150 | with self.assertRaises(ValueError): 151 | addr.kind = None # type: ignore 152 | 153 | with self.assertRaises(ValueError): 154 | addr.street_name = None # type: ignore 155 | 156 | with self.assertRaises(ValueError): 157 | addr.pincode = None # type: ignore 158 | 159 | with self.assertRaises(ValueError): 160 | addr.country = None # type: ignore 161 | 162 | with self.assertRaises(ValueError): 163 | Phone( 164 | kind=None, # type: ignore 165 | country_code=1, 166 | area_code=234, 167 | local_number=5678900 168 | ) 169 | 170 | with self.assertRaises(ValueError): 171 | Phone( 172 | kind=AddressType.work, 173 | country_code=None, # type: ignore 174 | area_code=234, 175 | local_number=5678900 176 | ) 177 | 178 | with self.assertRaises(ValueError): 179 | Phone( 180 | kind=AddressType.work, 181 | country_code=1, 182 | area_code=234, 183 | local_number=None # type: ignore 184 | ) 185 | 186 | p = Phone( 187 | kind=AddressType.work, 188 | country_code=1, 189 | area_code=234, 190 | local_number=5678900 191 | ) 192 | 193 | with self.assertRaises(ValueError): 194 | p.kind = None # type: ignore 195 | 196 | with self.assertRaises(ValueError): 197 | p.country_code = None # type: ignore 198 | 199 | with self.assertRaises(ValueError): 200 | p.local_number = None # type: ignore 201 | 202 | with self.assertRaises(ValueError): 203 | Email( 204 | kind=None, # type: ignore 205 | email='datamodel@microservices.py' 206 | ) 207 | 208 | with self.assertRaises(ValueError): 209 | Email( 210 | kind=AddressType.work, 211 | email=None # type: ignore 212 | ) 213 | 214 | e = Email( 215 | kind=AddressType.work, 216 | email='datamodel@microservices.py' 217 | ) 218 | 219 | with self.assertRaises(ValueError): 220 | e.kind = None # type: ignore 221 | 222 | with self.assertRaises(ValueError): 223 | e.email = None # type: ignore 224 | 225 | with self.assertRaises(ValueError): 226 | AddressEntry(full_name=None) # type: ignore 227 | 228 | a = AddressEntry(full_name='abc') 229 | 230 | with self.assertRaises(ValueError): 231 | a.full_name = None # type: ignore 232 | 233 | 234 | if __name__ == '__main__': 235 | unittest.main() 236 | -------------------------------------------------------------------------------- /addrservice/datamodel.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020. All rights reserved. 2 | 3 | from enum import Enum, unique 4 | from typing import ( 5 | Any, 6 | Mapping, 7 | Optional, 8 | Sequence, 9 | Union 10 | ) 11 | 12 | 13 | VALUE_ERR_MSG = '{} has invalid value {}' 14 | 15 | 16 | @unique 17 | class AddressType(Enum): 18 | home = 1, 19 | work = 2 20 | 21 | 22 | class Address: 23 | def __init__( 24 | self, 25 | kind: AddressType, 26 | street_name: str, 27 | pincode: Union[int, str], 28 | country: str, 29 | building_name: str = None, 30 | unit_number: int = None, 31 | street_number: Union[int, str] = None, 32 | locality: str = None, 33 | city: str = None, 34 | province: str = None, 35 | ): 36 | if kind is None: 37 | raise ValueError(VALUE_ERR_MSG.format('kind', kind)) 38 | if not street_name: 39 | raise ValueError(VALUE_ERR_MSG.format('street_name', street_name)) 40 | if not pincode: 41 | raise ValueError(VALUE_ERR_MSG.format('pincode', pincode)) 42 | if not country: 43 | raise ValueError(VALUE_ERR_MSG.format('country', country)) 44 | 45 | self._kind = kind 46 | self._building_name = building_name 47 | self._unit_number = unit_number 48 | self._street_number = street_number 49 | self._street_name = street_name 50 | self._locality = locality 51 | self._city = city 52 | self._province = province 53 | self._pincode = pincode 54 | self._country = country 55 | 56 | @classmethod 57 | def from_api_dm(cls, vars: Mapping[str, Any]) -> 'Address': 58 | return Address( 59 | kind=AddressType[vars['kind']], 60 | street_name=vars['street_name'], 61 | pincode=vars['pincode'], 62 | country=vars['country'], 63 | building_name=vars.get('building_name'), 64 | unit_number=vars.get('unit_number'), 65 | street_number=vars.get('street_number'), 66 | locality=vars.get('locality'), 67 | city=vars.get('city'), 68 | province=vars.get('province'), 69 | ) 70 | 71 | @property 72 | def kind(self) -> AddressType: 73 | return self._kind 74 | 75 | @kind.setter 76 | def kind(self, value: AddressType) -> None: 77 | if value is None: 78 | raise ValueError(VALUE_ERR_MSG.format('value', value)) 79 | 80 | self._kind = value 81 | 82 | @property 83 | def building_name(self) -> Optional[str]: 84 | return self._building_name 85 | 86 | @building_name.setter 87 | def building_name(self, value: str) -> None: 88 | self._building_name = value 89 | 90 | @property 91 | def unit_number(self) -> Optional[int]: 92 | return self._unit_number 93 | 94 | @unit_number.setter 95 | def unit_number(self, value: int) -> None: 96 | self._unit_number = value 97 | 98 | @property 99 | def street_number(self) -> Optional[Union[int, str]]: 100 | return self._street_number 101 | 102 | @street_number.setter 103 | def street_number(self, value: Union[int, str]) -> None: 104 | self._street_number = value 105 | 106 | @property 107 | def street_name(self) -> str: 108 | return self._street_name 109 | 110 | @street_name.setter 111 | def street_name(self, value: str) -> None: 112 | if not value: 113 | raise ValueError(VALUE_ERR_MSG.format('value', value)) 114 | 115 | self._street_name = value 116 | 117 | @property 118 | def locality(self) -> Optional[str]: 119 | return self._locality 120 | 121 | @locality.setter 122 | def locality(self, value: str) -> None: 123 | self._locality = value 124 | 125 | @property 126 | def city(self) -> Optional[str]: 127 | return self._city 128 | 129 | @city.setter 130 | def city(self, value: str) -> None: 131 | self._ = value 132 | 133 | @property 134 | def province(self) -> Optional[str]: 135 | return self._province 136 | 137 | @province.setter 138 | def province(self, value: str) -> None: 139 | self._province = value 140 | 141 | @property 142 | def pincode(self) -> Union[int, str]: 143 | return self._pincode 144 | 145 | @pincode.setter 146 | def pincode(self, value: Union[int, str]) -> None: 147 | if not value: 148 | raise ValueError(VALUE_ERR_MSG.format('value', value)) 149 | 150 | self._pincode = value 151 | 152 | @property 153 | def country(self) -> str: 154 | return self._country 155 | 156 | @country.setter 157 | def country(self, value: str) -> None: 158 | if not value: 159 | raise ValueError(VALUE_ERR_MSG.format('value', value)) 160 | 161 | self._country = value 162 | 163 | def to_api_dm(self) -> Mapping[str, Any]: 164 | d = { 165 | 'kind': self.kind.name, 166 | 'building_name': self.building_name, 167 | 'unit_number': self.unit_number, 168 | 'street_number': self.street_number, 169 | 'street_name': self.street_name, 170 | 'locality': self.locality, 171 | 'city': self.city, 172 | 'province': self.province, 173 | 'pincode': self.pincode, 174 | 'country': self.country, 175 | } 176 | 177 | return {k: v for k, v in d.items() if v is not None} 178 | 179 | 180 | class Phone: 181 | def __init__( 182 | self, 183 | kind: AddressType, 184 | country_code: int, 185 | local_number: int, 186 | area_code: int = None, 187 | ): 188 | if kind is None: 189 | raise ValueError(VALUE_ERR_MSG.format('kind', kind)) 190 | if not country_code: 191 | raise ValueError(VALUE_ERR_MSG.format( 192 | 'country_code', 193 | country_code 194 | )) 195 | if not local_number: 196 | raise ValueError(VALUE_ERR_MSG.format( 197 | 'local_number', 198 | local_number 199 | )) 200 | 201 | self._kind = kind 202 | self._country_code = country_code 203 | self._area_code = area_code 204 | self._local_number = local_number 205 | 206 | @classmethod 207 | def from_api_dm(cls, vars: Mapping[str, Any]) -> 'Phone': 208 | return Phone( 209 | kind=AddressType[vars['kind']], 210 | country_code=vars['country_code'], 211 | local_number=vars['local_number'], 212 | area_code=vars.get('area_code'), 213 | ) 214 | 215 | @property 216 | def kind(self) -> AddressType: 217 | return self._kind 218 | 219 | @kind.setter 220 | def kind(self, value: AddressType) -> None: 221 | if value is None: 222 | raise ValueError(VALUE_ERR_MSG.format('value', value)) 223 | 224 | self._kind = value 225 | 226 | @property 227 | def country_code(self) -> int: 228 | return self._country_code 229 | 230 | @country_code.setter 231 | def country_code(self, value: int) -> None: 232 | if not value: 233 | raise ValueError(VALUE_ERR_MSG.format('value', value)) 234 | 235 | self._country_code = value 236 | 237 | @property 238 | def area_code(self) -> Optional[int]: 239 | return self._area_code 240 | 241 | @area_code.setter 242 | def area_code(self, value: int) -> None: 243 | self._ = value 244 | 245 | @property 246 | def local_number(self) -> int: 247 | return self._local_number 248 | 249 | @local_number.setter 250 | def local_number(self, value: int) -> None: 251 | if not value: 252 | raise ValueError(VALUE_ERR_MSG.format('value', value)) 253 | 254 | self._local_number = value 255 | 256 | def to_api_dm(self) -> Mapping[str, Any]: 257 | d = { 258 | 'kind': self.kind.name, 259 | 'country_code': self.country_code, 260 | 'area_code': self.area_code, 261 | 'local_number': self.local_number, 262 | } 263 | 264 | return {k: v for k, v in d.items() if v is not None} 265 | 266 | 267 | class Email: 268 | def __init__( 269 | self, 270 | kind: AddressType, 271 | email: str, 272 | ): 273 | if kind is None: 274 | raise ValueError(VALUE_ERR_MSG.format('kind', kind)) 275 | if not email: 276 | raise ValueError(VALUE_ERR_MSG.format('value', email)) 277 | 278 | self._kind = kind 279 | self._email = email 280 | 281 | @classmethod 282 | def from_api_dm(cls, vars: Mapping[str, Any]) -> 'Email': 283 | return Email( 284 | kind=AddressType[vars['kind']], 285 | email=vars['email'], 286 | ) 287 | 288 | @property 289 | def kind(self) -> AddressType: 290 | return self._kind 291 | 292 | @kind.setter 293 | def kind(self, value: AddressType) -> None: 294 | if value is None: 295 | raise ValueError(VALUE_ERR_MSG.format('value', value)) 296 | 297 | self._kind = value 298 | 299 | @property 300 | def email(self) -> str: 301 | return self._email 302 | 303 | @email.setter 304 | def email(self, value: str) -> None: 305 | if not value: 306 | raise ValueError(VALUE_ERR_MSG.format('value', value)) 307 | 308 | self._email = value 309 | 310 | def to_api_dm(self) -> Mapping[str, Any]: 311 | d = { 312 | 'kind': self.kind.name, 313 | 'email': self.email, 314 | } 315 | 316 | return {k: v for k, v in d.items() if v is not None} 317 | 318 | 319 | class AddressEntry: 320 | def __init__( 321 | self, 322 | full_name: str, 323 | addresses: Sequence[Address] = [], 324 | phone_numbers: Sequence[Phone] = [], 325 | fax_numbers: Sequence[Phone] = [], 326 | emails: Sequence[Email] = [], 327 | ): 328 | if not full_name: 329 | raise ValueError(VALUE_ERR_MSG.format('full_name', full_name)) 330 | 331 | self._full_name = full_name 332 | self._addresses = list(addresses) 333 | self._phone_numbers = list(phone_numbers) 334 | self._fax_numbers = list(fax_numbers) 335 | self._emails = list(emails) 336 | 337 | @classmethod 338 | def from_api_dm(cls, vars: Mapping[str, Any]) -> 'AddressEntry': 339 | return AddressEntry( 340 | full_name=vars['full_name'], 341 | addresses=[ 342 | Address.from_api_dm(x) for x in vars.get('addresses', []) 343 | ], 344 | phone_numbers=[ 345 | Phone.from_api_dm(x) for x in vars.get('phone_numbers', []) 346 | ], 347 | fax_numbers=[ 348 | Phone.from_api_dm(x) for x in vars.get('fax_numbers', []) 349 | ], 350 | emails=[ 351 | Email.from_api_dm(x) for x in vars.get('emails', []) 352 | ], 353 | ) 354 | 355 | @property 356 | def full_name(self) -> str: 357 | return self._full_name 358 | 359 | @full_name.setter 360 | def full_name(self, value: str) -> None: 361 | if not value: 362 | raise ValueError(VALUE_ERR_MSG.format('full_name', value)) 363 | 364 | self._full_name = value 365 | 366 | @property 367 | def addresses(self) -> Sequence[Address]: 368 | return self._addresses 369 | 370 | @addresses.setter 371 | def addresses(self, value: Sequence[Address]) -> None: 372 | self._addresses = list(value) 373 | 374 | @property 375 | def phone_numbers(self) -> Sequence[Phone]: 376 | return self._phone_numbers 377 | 378 | @phone_numbers.setter 379 | def phone_numbers(self, value: Sequence[Phone]) -> None: 380 | self._phone_numbers = list(value) 381 | 382 | @property 383 | def fax_numbers(self) -> Sequence[Phone]: 384 | return self._fax_numbers 385 | 386 | @fax_numbers.setter 387 | def fax_numbers(self, value: Sequence[Phone]) -> None: 388 | self._fax_numbers = list(value) 389 | 390 | @property 391 | def emails(self) -> Sequence[Email]: 392 | return self._emails 393 | 394 | @emails.setter 395 | def emails(self, value: Sequence[Email]) -> None: 396 | self._emails = list(value) 397 | 398 | def to_api_dm(self) -> Mapping[str, Any]: 399 | d = { 400 | 'full_name': self.full_name, 401 | 'addresses': [x.to_api_dm() for x in self.addresses], 402 | 'phone_numbers': [x.to_api_dm() for x in self.phone_numbers], 403 | 'fax_numbers': [x.to_api_dm() for x in self.fax_numbers], 404 | 'emails': [x.to_api_dm() for x in self.emails], 405 | } 406 | 407 | return {k: v for k, v in d.items() if v is not None} 408 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tutorial: Building and testing micro-services using Tornado 2 | 3 | Blog Posts: 4 | 5 | 1. [Choices, Key Concepts, and Project setup](https://www.ml4devs.com/articles/python-microservices-tornado-01-asyncio-lint-test-coverage-project-setup/) 6 | 2. [Build and Test REST Endpoints With Tornado](https://www.ml4devs.com/articles/python-microservices-tornado-02-rest-unit-integration-tests/) 7 | 3. [Effective Canonical Logging Across Services](https://www.ml4devs.com/articles/python-microservices-tornado-03-effective-canonical-logging-across-services/) 8 | 4. [API, Object, and Storage Data Models](https://www.ml4devs.com/articles/python-microservices-tornado-04-api-object-and-physical-storage-data-models/) 9 | 10 | ## 0. Get the source code 11 | 12 | Get the source code for the tutorial: 13 | 14 | ``` bash 15 | $ git clone https://github.com/scgupta/tutorial-python-microservice-tornado.git 16 | $ cd tutorial-python-microservice-tornado 17 | 18 | $ tree . 19 | . 20 | ├── LICENSE 21 | ├── README.md 22 | ├── addrservice 23 | │ ├── __init__.py 24 | │ ├── database 25 | │ │ ├── __init__.py 26 | │ │ ├── addressbook_db.py 27 | │ │ └── db_engines.py 28 | │ ├── datamodel.py 29 | │ ├── service.py 30 | │ ├── tornado 31 | │ │ ├── __init__.py 32 | │ │ ├── app.py 33 | │ │ └── server.py 34 | │ └── utils 35 | │ ├── __init__.py 36 | │ └── logutils.py 37 | ├── configs 38 | │ └── addressbook-local.yaml 39 | ├── data 40 | │ ├── __init__.py 41 | │ └── addresses 42 | │ ├── namo.json 43 | │ └── raga.json 44 | ├── requirements.txt 45 | ├── run.py 46 | ├── schema 47 | │ └── address-book-v1.0.json 48 | └── tests 49 | ├── __init__.py 50 | ├── integration 51 | │ ├── __init__.py 52 | │ ├── addrservice_test.py 53 | │ └── tornado_app_addreservice_handlers_test.py 54 | └── unit 55 | ├── __init__.py 56 | ├── address_data_test.py 57 | ├── addressbook_db_test.py 58 | ├── datamodel_test.py 59 | └── tornado_app_handlers_test.py 60 | ``` 61 | 62 | The directory `addrservice` is for the source code of the service, and the directory `test` is for keeping the tests. 63 | 64 | ## 1. Project Setup 65 | 66 | Setup Virtual Environment: 67 | 68 | ``` bash 69 | $ python3 -m venv .venv 70 | $ source ./.venv/bin/activate 71 | $ pip install --upgrade pip 72 | $ pip3 install -r ./requirements.txt 73 | ``` 74 | 75 | Let's start from scratch: 76 | 77 | ``` bash 78 | $ git checkout -b tag-01-project-setup 79 | ``` 80 | 81 | You can run static type checker, linter, unit tests, and code coverage by either executing the tool directly or through `run.py` script. In each of the following, In each of the following, you can use either of the commands. 82 | 83 | Static Type Checker: 84 | 85 | ``` bash 86 | $ mypy ./addrservice ./tests 87 | 88 | $ ./run.py typecheck 89 | ``` 90 | 91 | Linter: 92 | 93 | ``` bash 94 | $ flake8 ./addrservice ./tests 95 | 96 | $ ./run.py lint 97 | ``` 98 | 99 | Unit Tests: 100 | 101 | ``` bash 102 | $ python -m unittest discover tests -p '*_test.py' 103 | 104 | $ ./run.py test 105 | ``` 106 | 107 | Code Coverage: 108 | 109 | ``` bash 110 | $ coverage run --source=addrservice --branch -m unittest discover tests -p '*_test.py' 111 | 112 | $ coverage run --source=addrservice --branch ./run.py test 113 | ``` 114 | 115 | After running tests with code coverage, you can get the report: 116 | 117 | ``` bash 118 | $ coverage report 119 | Name Stmts Miss Branch BrPart Cover 120 | ----------------------------------------------------------- 121 | addrservice/__init__.py 2 2 0 0 0% 122 | ``` 123 | 124 | You can also generate HTML report: 125 | 126 | ``` bash 127 | $ coverage html 128 | $ open htmlcov/index.html 129 | ``` 130 | 131 | If you are able to run all these commands, your project setup has no error and you are all set for coding. 132 | 133 | --- 134 | 135 | ## 2. Microservice 136 | 137 | Checkout the code: 138 | 139 | ``` bash 140 | $ git checkout -b tag-02-microservice 141 | ``` 142 | 143 | File `addrservice/service.py` has business logic for CRUD operations for the address-book. This file is indpendent of any web service framework. 144 | It currenly has just stubs with rudimentry implementation keeing addresses in a dictionary. It is sufficint to implement and test the REST service endpoints. 145 | 146 | [Tornado](https://www.tornadoweb.org/) is a framework to develop Python web/microservices. It uses async effectively to achieve high number of open connections. In this tutorial, we create a `tornado.web.Application` and add `tornado.web.RequestHandlers` in file `addrservice/tornado/app.py` to serve various API endpoints for this address service. Tornado also has a rich framework for testing. 147 | 148 | Web services return HTML back. In address book microservice, API data interface is JSON. We will examine key Tornado APIs of `Application`, `RequestHandler` and `tornado.testing` to develop it. 149 | 150 | But first, let's run the server and test it: 151 | 152 | ``` bash 153 | $ python3 addrservice/tornado/server.py --port 8080 --config ./configs/addressbook-local.yaml --debug 154 | 155 | Starting Address Book on port 8080 ... 156 | ``` 157 | 158 | Also run lint, typecheck and test to verify nothing is broken, and also code coverage: 159 | 160 | ``` bash 161 | $ ./run.py lint 162 | $ ./run.py typecheck 163 | $ ./run.py test -v 164 | $ coverage run --source=addrservice --omit="addrservice/tornado/server.py" --branch ./run.py test 165 | $ coverage report 166 | Name Stmts Miss Branch BrPart Cover 167 | ------------------------------------------------------------------- 168 | addrservice/__init__.py 2 0 0 0 100% 169 | addrservice/service.py 23 1 0 0 96% 170 | addrservice/tornado/__init__.py 0 0 0 0 100% 171 | addrservice/tornado/app.py 83 4 8 3 92% 172 | ------------------------------------------------------------------- 173 | TOTAL 108 5 8 3 93% 174 | ``` 175 | 176 | The `addrservice/tornado/server.py` has been omitted from coverage. This is the file used to start the server. Since Torando test framework has a mechanism to start the server in the same process where tests are running, this file does not get tested by unit and integration tests. 177 | 178 | These are the addressbook API endpoints, implemented through two Request Handlers: 179 | 180 | `AddressBookRequestHandler`: 181 | 182 | - `GET /addresses`: gets all addresses in the address book 183 | - `POST /addresses`: create an entry in the addressbook 184 | 185 | `AddressBookEntryRequestHandler`: 186 | 187 | - `GET /addresses/{id}`: get the address book entry with given id 188 | - `PUT /addresses/{id}`: update the address book entry with given id 189 | - `DELETE /addresses/{id}`: delete the address book entry with given id 190 | 191 | Here is a sample session exercising all endpoints (notice the POST response has Location in the Headers containing the URI/id `66fdbb78e79846849608b2cfe244a858` of the entry that gets created): 192 | 193 | ``` bash 194 | # Create an address entry 195 | 196 | $ curl -i -X POST http://localhost:8080/addresses -d '{"full_name": "Bill Gates"}' 197 | 198 | HTTP/1.1 201 Created 199 | Server: TornadoServer/6.0.3 200 | Content-Type: text/html; charset=UTF-8 201 | Date: Tue, 10 Mar 2020 14:40:01 GMT 202 | Location: /addresses/66fdbb78e79846849608b2cfe244a858 203 | Content-Length: 0 204 | Vary: Accept-Encoding 205 | 206 | # Read the address entry 207 | 208 | $ curl -i -X GET http://localhost:8080/addresses/66fdbb78e79846849608b2cfe244a858 209 | 210 | HTTP/1.1 200 OK 211 | Server: TornadoServer/6.0.3 212 | Content-Type: application/json; charset=UTF-8 213 | Date: Tue, 10 Mar 2020 14:44:26 GMT 214 | Etag: "5496aee01a83cf2386641b2c43540fc5919d621e" 215 | Content-Length: 22 216 | Vary: Accept-Encoding 217 | {"full_name": "Bill Gates"} 218 | 219 | # Update the address entry 220 | 221 | $ curl -i -X PUT http://localhost:8080/addresses/66fdbb78e79846849608b2cfe244a858 -d '{"full_name": "William Henry Gates III"}' 222 | 223 | HTTP/1.1 204 No Content 224 | Server: TornadoServer/6.0.3 225 | Date: Tue, 10 Mar 2020 14:48:04 GMT 226 | Vary: Accept-Encoding 227 | 228 | # List all addresses 229 | 230 | $ curl -i -X GET http://localhost:8080/addresses 231 | 232 | HTTP/1.1 200 OK 233 | Server: TornadoServer/6.0.3 234 | Content-Type: application/json; charset=UTF-8 235 | Date: Tue, 10 Mar 2020 14:49:10 GMT 236 | Etag: "5601e676f3fa4447feaa8d2dd960be163af7570a" 237 | Content-Length: 73 238 | Vary: Accept-Encoding 239 | {"66fdbb78e79846849608b2cfe244a858": {"full_name": "William Henry Gates III"}} 240 | 241 | # Delete the address 242 | 243 | $ curl -i -X DELETE http://localhost:8080/addresses/66fdbb78e79846849608b2cfe244a858 244 | 245 | HTTP/1.1 204 No Content 246 | Server: TornadoServer/6.0.3 247 | Date: Tue, 10 Mar 2020 14:50:38 GMT 248 | Vary: Accept-Encoding 249 | 250 | # Verify address is deleted 251 | 252 | $ curl -i -X GET http://localhost:8080/addresses 253 | 254 | HTTP/1.1 200 OK 255 | Server: TornadoServer/6.0.3 256 | Content-Type: application/json; charset=UTF-8 257 | Date: Tue, 10 Mar 2020 14:52:01 GMT 258 | Etag: "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f" 259 | Content-Length: 2 260 | Vary: Accept-Encoding 261 | {} 262 | 263 | $ curl -i -X GET http://localhost:8080/addresses/66fdbb78e79846849608b2cfe244a858 264 | 265 | HTTP/1.1 404 '66fdbb78e79846849608b2cfe244a858' 266 | Server: TornadoServer/6.0.3 267 | Content-Type: application/json; charset=UTF-8 268 | Date: Tue, 10 Mar 2020 14:53:06 GMT 269 | Content-Length: 1071 270 | Vary: Accept-Encoding 271 | {"method": "GET", "uri": "/addresses/66fdbb78e79846849608b2cfe244a858", "code": 404, "message": "'66fdbb78e79846849608b2cfe244a858'", "trace": "Traceback (most recent call last):\n\n File \"... redacted call stack trace ... addrservice/tornado/app.py\", line 100, in get\n raise tornado.web.HTTPError(404, reason=str(e))\n\ntornado.web.HTTPError: HTTP 404: '66fdbb78e79846849608b2cfe244a858'\n"} 272 | ``` 273 | 274 | --- 275 | 276 | ## 3. Logging 277 | 278 | Checkout the code: 279 | 280 | ``` bash 281 | $ git checkout -b tag-03-logging 282 | ``` 283 | 284 | Effective logs can cut down diagnosis time and facilitate monitoring and altering. 285 | 286 | ### Log Format 287 | 288 | [Logfmt](https://pypi.org/project/logfmt/) log format consists of *key-value* pairs. 289 | It offers good balance between processing using standard tools and human readibility. 290 | 291 | ### Canonical Logs 292 | 293 | Emiting one canonical log line](https://brandur.org/canonical-log-lines) for each request makes manual inspection easier. 294 | Assigning and logging a *request id* to each request, and passing that id to all called service helps correlate logs across services. 295 | The *key-value* pairs for the log are stored in a [task context](https://github.com/Skyscanner/aiotask-context), which is maintained across asyncio task interleaving. 296 | 297 | ### Log Configuration 298 | 299 | Logging are useful in diagnosing services, more so when async is involved. Python has a standard [logging](https://docs.python.org/3/library/logging.html) package, and its documentation includes an excellent [HOWTO](https://docs.python.org/3/howto/logging.html) guide and [Cookbook](https://docs.python.org/3/howto/logging-cookbook.html). These are rich source of information, and leave nothoing much to add. Following are some of the best practices in my opinion: 300 | 301 | - Do NOT use ROOT logger directly throgh `logging.debug()`, `logging.error()` methods directly because it is easy to overlook their default behavior. 302 | - Do NOT use module level loggers of variety `logging.getLogger(__name__)` because any complex project will require controlling logging through configuration (see next point). These may cause surprise if you forget to set `disable_existing_loggers` to false or overlook how modules are loaded and initialized. If use at all, call `logging.getLogger(__name__)` inside function, rather than outside at the beginning of a module. 303 | - `dictConfig` (in `yaml`) offers right balance of versatility and flexibility compared to `ini` based `fileConfig`or doing it in code. Specifying logger in config files allows you to use different logging levels and infra in prod deployment, stage deployments, and local debugging (with increasingly more logs). 304 | 305 | Sending logs to multiple data stores and tools for processing can be controled by a [log configuration](https://docs.python.org/3/library/logging.config.html). Each logger has a format and multiple handlers can be associated with a logger. Here is a part of `configs/addressbook-local.yaml`: 306 | 307 | ``` yaml 308 | logging: 309 | version: 1 310 | formatters: 311 | brief: 312 | format: '%(asctime)s %(name)s %(levelname)s : %(message)s' 313 | detailed: 314 | format: 'time="%(asctime)s" logger="%(name)s" level="%(levelname)s" file="%(filename)s" lineno=%(lineno)d function="%(funcName)s" %(message)s' 315 | handlers: 316 | console: 317 | class: logging.StreamHandler 318 | level: INFO 319 | formatter: brief 320 | stream: ext://sys.stdout 321 | file: 322 | class : logging.handlers.RotatingFileHandler 323 | level: DEBUG 324 | formatter: detailed 325 | filename: /tmp/addrservice-app.log 326 | backupCount: 3 327 | loggers: 328 | addrservice: 329 | level: DEBUG 330 | handlers: 331 | - console 332 | - file 333 | propagate: no 334 | tornado.access: 335 | level: DEBUG 336 | handlers: 337 | - file 338 | tornado.application: 339 | level: DEBUG 340 | handlers: 341 | - file 342 | tornado.general: 343 | level: DEBUG 344 | handlers: 345 | - file 346 | root: 347 | level: WARNING 348 | handlers: 349 | - console 350 | ``` 351 | 352 | Notice that this configuration not just defines a logger `addrservice` for this service, but also modifies behavior of Tornado's general logger. There are several pre-defined [handlers](https://docs.python.org/3/library/logging.handlers.html). Here the SteamHandler and RotatingFileHandler are being used to write to console and log files respectively. 353 | 354 | ### Tornado 355 | 356 | Tornado has several hooks to control when and how logging is done: 357 | 358 | - [`log_function`](https://www.tornadoweb.org/en/stable/web.html#tornado.web.Application.settings): function Tornado calls at the end of every request to log the result. 359 | - [`write_error`](https://www.tornadoweb.org/en/stable/web.html#tornado.web.RequestHandler.write_error): to customize the error response. Information about the error is added to the log context. 360 | - [`log_exception`](): to log uncaught exceptions. It can be overwritten to log in logfmt format. 361 | 362 | ### Log Inspection 363 | 364 | **Start the server:** 365 | 366 | It will show the console log: 367 | 368 | ``` bash 369 | $ python3 addrservice/tornado/server.py --port 8080 --config ./configs/addressbook-local.yaml --debug 370 | 371 | 2020-03-17 12:54:15,198 addrservice INFO : message="STARTING" service_name="Address Book" port=8080 372 | ``` 373 | 374 | **Watch the logs:** 375 | 376 | ``` bash 377 | $ tail -f /tmp/addrservice-app.log 378 | 379 | time="2020-03-17 12:54:15,198" logger="addrservice" level="INFO" file="logutils.py" lineno=57 function="log" message="STARTING" service_name="Address Book" port=8080 380 | ``` 381 | 382 | **Send a request:** 383 | 384 | ```bash 385 | $ curl -i -X POST http://localhost:8080/addresses -d '{"name": "Bill Gates"}' 386 | 387 | HTTP/1.1 201 Created 388 | Server: TornadoServer/6.0.3 389 | Content-Type: text/html; charset=UTF-8 390 | Date: Tue, 17 Mar 2020 07:26:32 GMT 391 | Location: /addresses/7feec2df29fd4b039028ad351bafe422 392 | Content-Length: 0 393 | Vary: Accept-Encoding 394 | ``` 395 | 396 | The console log will show brief log entries: 397 | 398 | ``` log 399 | 2020-03-17 12:56:32,784 addrservice INFO : req_id="e6cd3072530f46b9932946fd65a13779" method="POST" uri="/addresses" ip="::1" message="RESPONSE" status=201 time_ms=1.2888908386230469 400 | ``` 401 | 402 | The log file will show logfmt-style one-line detailed canonical log entries: 403 | 404 | ``` log 405 | time="2020-03-17 12:56:32,784" logger="addrservice" level="INFO" file="logutils.py" lineno=57 function="log" req_id="e6cd3072530f46b9932946fd65a13779" method="POST" uri="/addresses" ip="::1" message="RESPONSE" status=201 time_ms=1.2888908386230469 406 | ``` 407 | 408 | ### Unit and Integration Tests 409 | 410 | Tests are quiet by default: 411 | 412 | ``` bash 413 | $ ./run.py lint 414 | $ ./run.py typecheck 415 | 416 | $ ./run.py test -v 417 | 418 | test_address_book_endpoints (integration.tornado_app_addreservice_handlers_test.TestAddressServiceApp) ... ok 419 | test_default_handler (unit.tornado_app_handlers_test.AddressServiceTornadoAppUnitTests) ... ok 420 | 421 | ---------------------------------------------------------------------- 422 | Ran 2 tests in 0.049s 423 | 424 | OK 425 | 426 | $ coverage run --source=addrservice --omit="addrservice/tornado/server.py" --branch ./run.py test 427 | 428 | .. 429 | ---------------------------------------------------------------------- 430 | Ran 2 tests in 0.049s 431 | OK 432 | 433 | $ coverage report 434 | 435 | Name Stmts Miss Branch BrPart Cover 436 | ------------------------------------------------------------------- 437 | addrservice/__init__.py 3 0 0 0 100% 438 | addrservice/service.py 25 1 0 0 96% 439 | addrservice/tornado/__init__.py 0 0 0 0 100% 440 | addrservice/tornado/app.py 105 6 18 6 90% 441 | addrservice/utils/__init__.py 0 0 0 0 100% 442 | addrservice/utils/logutils.py 28 0 6 0 100% 443 | ------------------------------------------------------------------- 444 | TOTAL 161 7 24 6 93% 445 | ``` 446 | 447 | If you want to change the log message during tests, change log level from ERROR to INFO: 448 | 449 | ``` python 450 | # tests/unit/tornado_app_handlers_test.py 451 | 452 | IN_MEMORY_CFG_TXT = ''' 453 | service: 454 | name: Address Book Test 455 | logging: 456 | version: 1 457 | root: 458 | level: INFO 459 | ''' 460 | ``` 461 | 462 | With that change, if you run the tests, you can examine the logs: 463 | 464 | ``` log 465 | $ ./run.py tests 466 | 467 | INFO:addrservice:req_id="a100e35140604d72930cb16c9eed8e8a" method="GET" uri="/addresses/" ip="127.0.0.1" message="RESPONSE" status=200 time_ms=1.232147216796875 468 | INFO:addrservice:req_id="29b08c81acbd403b89f007ba03b5fee7" method="POST" uri="/addresses/" ip="127.0.0.1" message="RESPONSE" status=201 time_ms=0.9398460388183594 469 | WARNING:addrservice:req_id="1c959a77f9de4f7e87e384a174fb6fbe" method="POST" uri="/addresses/" ip="127.0.0.1" reason="Invalid JSON body" message="RESPONSE" status=400 time_ms=1.7652511596679688 trace="Traceback..... 470 | ``` 471 | 472 | --- 473 | 474 | ## 4. Data Model 475 | 476 | Get the code: 477 | 478 | ``` bash 479 | $ git checkout -b tag-04-datamodel 480 | ``` 481 | 482 | ### API Data Model 483 | 484 | Also known as communication or exchange data model 485 | The data model for interacting with a microservice. It is designed for efficiently exchanging (sending and receiving) data with the service. 486 | 487 | The address book service uses JSON for exchanging data. The [JSON schema](https://json-schema.org/) for the data model is in `schema/address-book-v1.0.json`, and test data in `data/addresses/*.json`. Even the data must be tested to be correct. So there is a test `tests/unit/address_data_test.py` to check whether data files conform to the JSON schema. 488 | 489 | ``` bash 490 | $ python3 tests/unit/address_data_test.py 491 | .. 492 | ---------------------------------------------------------------------- 493 | Ran 2 tests in 0.006s 494 | 495 | OK 496 | ``` 497 | 498 | ### Object Data Model 499 | 500 | Also known as application data model or data structures. 501 | It is designed for efficiently performing business logic (algorithms) of an application / service. 502 | 503 | There are tools like [Python JSON Schema Objects](https://github.com/cwacek/python-jsonschema-objects), [Warlock](https://github.com/bcwaldon/warlock), [Valideer](https://github.com/podio/valideer), that generate POPO (Plain Old Python Object) classes from a JSON schema. These tools do simple structural mapping from JSON schema elements to classes. However, there are validation checks, inheritance, and polymorphism that can't be expressed in JSON schema. So it may require hand-crafting a data model suitable for business logic. 504 | 505 | The logical data model is implemented in `addrservice/datamodel.py`. 506 | 507 | ### Storage Data Model 508 | 509 | Also known as Physical Data Model. 510 | It is designed for efficient storage, retrieval, and search. There are several kinds of data stores: relational, hierarchical, graph. A combination of these storage is picked depending upon the structure of the persistent data, and retrieval and search requirements. 511 | 512 | The `addrservice/database/addressbook_db.py` defines an `AbstractAddressBookDB`, which the service interacts with. This decouples the storage choice, and allows changing the storage model without affecting rest of the code. For example, it defines an `InMemoryAddressBookDB` and `FileAddressBookDB`. The in-memory data store is useful in unit/integration tests as it facilitates deep asserts for the state of the store. The file backed storage persists the data in files, and useful for debugging. 513 | 514 | The storage can be swapped by setting the configuration, for example: 515 | 516 | ``` yaml 517 | addr-db: 518 | memory: null 519 | ``` 520 | 521 | Based on the config, an appropriate DB engine is set up by `addrservice/database/db_engines.py:create_addressbook_db()`. 522 | 523 | Following is need to implement a SQL data store: 524 | 525 | - Implement a sub-class of `AbstractAddressBookDB` that stores data in a RDBMS 526 | - Add a case in `create_addressbook_db()` 527 | - Change config (`configs/addressbook-local.yaml`) 528 | 529 | It does not require touch any of the business logic. 530 | 531 | ### Running the Service 532 | 533 | Start the service: 534 | 535 | ``` bash 536 | $ python3 addrservice/tornado/server.py --port 8080 --config ./configs/addressbook-local.yaml --debug 537 | 538 | 2020-03-30 06:46:29,641 addrservice INFO : message="STARTING" service_name="Address Book" port=8080 539 | ``` 540 | 541 | Test CRUD with curl command: 542 | 543 | ``` bash 544 | $ curl -X 'GET' http://localhost:8080/addresses 545 | 546 | {} 547 | 548 | $ curl -i -X 'POST' -H "Content-Type: application/json" -d "@data/addresses/namo.json" http://localhost:8080/addresses 549 | 550 | HTTP/1.1 100 (Continue) 551 | 552 | HTTP/1.1 201 Created 553 | Server: TornadoServer/6.0.3 554 | Content-Type: text/html; charset=UTF-8 555 | Date: Mon, 30 Mar 2020 01:22:10 GMT 556 | Location: /addresses/0bc13fbf2db54ef392a08a37378afa7f 557 | Content-Length: 0 558 | Vary: Accept-Encoding 559 | 560 | $ curl -X 'GET' http://localhost:8080/addresses/0bc13fbf2db54ef392a08a37378afa7f 561 | 562 | {"full_name": "Narendra Modi", "addresses": ...} 563 | 564 | $ curl -i -X 'PUT' -H "Content-Type: application/json" -d "@data/addresses/raga.json" http://localhost:8080/addresses/0bc13fbf2db54ef392a08a37378afa7f 565 | 566 | HTTP/1.1 204 No Content 567 | Server: TornadoServer/6.0.3 568 | Date: Mon, 30 Mar 2020 01:24:51 GMT 569 | Vary: Accept-Encoding 570 | 571 | $ curl -X 'GET' http://localhost:8080/addresses/0bc13fbf2db54ef392a08a37378afa7f 572 | 573 | {"full_name": "Rahul Gandhi", "addresses": ...} 574 | 575 | $ curl -i -X 'DELETE' http://localhost:8080/addresses/0bc13fbf2db54ef392a08a37378afa7f 576 | 577 | HTTP/1.1 204 No Content 578 | Server: TornadoServer/6.0.3 579 | Date: Mon, 30 Mar 2020 01:26:34 GMT 580 | Vary: Accept-Encoding 581 | 582 | $ curl -X 'GET' http://localhost:8080/addresses 583 | 584 | {} 585 | ``` 586 | 587 | Change `addr-db` in `configs/addressbook-local.yaml` from memory to file system: 588 | 589 | ``` yaml 590 | addr-db: 591 | fs: /tmp/addrservice-db 592 | ``` 593 | 594 | Run all the commands again. You will see records being stored in `/tmp/addrservice-db` as json files. 595 | 596 | ### Tests and Code Coverage 597 | 598 | Run tests and check code coverage: 599 | 600 | ``` bash 601 | $ coverage run --source=addrservice --omit="addrservice/tornado/server.py" --branch ./run.py test 602 | 603 | .......... 604 | ---------------------------------------------------------------------- 605 | Ran 10 tests in 0.148s 606 | 607 | OK 608 | 609 | $ coverage report 610 | 611 | Name Stmts Miss Branch BrPart Cover 612 | -------------------------------------------------------------------------- 613 | addrservice/__init__.py 7 0 0 0 100% 614 | addrservice/database/__init__.py 0 0 0 0 100% 615 | addrservice/database/addressbook_db.py 107 5 28 1 96% 616 | addrservice/database/db_engines.py 6 0 2 0 100% 617 | addrservice/datamodel.py 226 0 54 0 100% 618 | addrservice/service.py 36 0 2 0 100% 619 | addrservice/tornado/__init__.py 0 0 0 0 100% 620 | addrservice/tornado/app.py 107 2 20 4 95% 621 | addrservice/utils/__init__.py 0 0 0 0 100% 622 | addrservice/utils/logutils.py 28 0 6 0 100% 623 | -------------------------------------------------------------------------- 624 | TOTAL 517 7 112 5 98% 625 | ``` 626 | 627 | --- 628 | --------------------------------------------------------------------------------