├── python_template_with_config ├── __init__.py ├── tests │ ├── __init__.py │ └── test_start_here.py ├── another_module.py ├── start_here.py └── config.py ├── .gitignore ├── setup.py ├── LICENSE └── README.md /python_template_with_config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python_template_with_config/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python_template_with_config/another_module.py: -------------------------------------------------------------------------------- 1 | """1 liner to explain this module""" 2 | import numpy as np 3 | 4 | 5 | def some_other_function(): 6 | """Dummy function""" 7 | return None 8 | 9 | 10 | def a_math_function(): 11 | """Example function that uses numpy""" 12 | return np.sin(2*np.pi) 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | .ropeproject 3 | log.log 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Packages 9 | *.egg 10 | *.egg-info 11 | dist 12 | build 13 | eggs 14 | parts 15 | bin 16 | var 17 | sdist 18 | develop-eggs 19 | .installed.cfg 20 | lib 21 | lib64 22 | 23 | # Installer logs 24 | pip-log.txt 25 | 26 | # Unit test / coverage reports 27 | .coverage 28 | .tox 29 | nosetests.xml 30 | 31 | # Translations 32 | *.mo 33 | 34 | # Mr Developer 35 | .mr.developer.cfg 36 | .project 37 | .pydevproject 38 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='python_template_with_config', 5 | version='0.1.0', 6 | description='Template of a Python module', 7 | long_description="", 8 | author='Written by Ian Ozsvald', 9 | author_email='ian@ianozsvald.com', 10 | url='https://github.com/ianozsvald/', 11 | license='', 12 | packages=find_packages(), 13 | include_package_data=True, 14 | zip_safe=False, 15 | install_requires=["numpy>=1.10"], 16 | classifiers=[ 17 | 'Environment :: Console', 18 | 'Intended Audience :: Developers', 19 | 'Operating System :: OS Independent', 20 | 'Programming Language :: Python'] 21 | ) 22 | -------------------------------------------------------------------------------- /python_template_with_config/tests/test_start_here.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Tests for start_here""" 3 | 4 | import unittest 5 | import logging 6 | from python_template_with_config import start_here 7 | from python_template_with_config import config 8 | 9 | # Run this using: 10 | # $ nosetests 11 | # or 12 | # $ nosetests -s # show stdout for the print 13 | 14 | A_PARAMETER = 99 15 | 16 | 17 | class Test(unittest.TestCase): 18 | def setUp(self): 19 | # get a config object (this is optional, remove if you don't need it) 20 | self.conf = config.get('test') 21 | 22 | def test1(self): 23 | print(self.conf) 24 | logging.info("Testing some very basic things") 25 | self.assertEqual(start_here.dummy_function(), "Hello") 26 | self.assertEqual(self.conf.name, "test") 27 | self.assertEqual(self.conf.a_parameter, A_PARAMETER) 28 | 29 | 30 | if __name__ == "__main__": 31 | unittest.main() 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Ian Ozsvald 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /python_template_with_config/start_here.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """1 liner to explain this project""" 3 | import argparse 4 | import logging 5 | from python_template_with_config import config 6 | from python_template_with_config import another_module # just to show how to import another module in this package 7 | 8 | 9 | # See the README for notes on running this using either: 10 | # arguments via __main__ 11 | # a configuration environment variable 12 | 13 | 14 | def dummy_function(): 15 | """Delete this dummy function, it is here for the unittests""" 16 | return "Hello" 17 | 18 | 19 | if __name__ == "__main__": 20 | # Create an argument parser which also shows the defaults when --help is 21 | # called 22 | parser = argparse.ArgumentParser(description='Project description for this prototype...', 23 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 24 | # These two lines show a positional (mandatory) argument and an optional argument 25 | # parser.add_argument('positional_arg', type=str, help='required positional argument') 26 | parser.add_argument('--env', default="dev", help='optional configuration argument') 27 | parser.add_argument('--a_parameter', type=int, default=None, help='some other parameter that is already in the Conf') 28 | args = parser.parse_args() 29 | 30 | conf = config.get(args.env, overrides=vars(args)) # Note that the environment is optional (it can use a config environment instead) 31 | logger = logging.getLogger("logger_for_template") 32 | logger.info("These are our args: {}".format(args)) 33 | logger.info("This is our configuration: {}".format(conf)) 34 | logger.info("This is an example log message, logging is configured once config.get() has been called") 35 | logger.info("The value of a_parameter={}".format(conf.a_parameter)) 36 | 37 | config.configure_logging(logging.CRITICAL) 38 | logger.info("This log message won't be shown") 39 | logger.critical("This critical message will now be shown!") 40 | -------------------------------------------------------------------------------- /python_template_with_config/config.py: -------------------------------------------------------------------------------- 1 | """""" 2 | import os 3 | import abc 4 | import sys 5 | import logging 6 | 7 | 8 | # environment variable for configuration 9 | CONFIG_ENV_VAR = "CONFIG" 10 | LOGGER_NAME = "logger_for_template" # CHOOSE YOUR OWN NAME FOR YOUR APP 11 | 12 | 13 | def configure_logging(log_level): 14 | """Configure a logger with sane datetime and path info for the calling function""" 15 | # configure logging to stdout 16 | logger = logging.getLogger(LOGGER_NAME) 17 | logger.setLevel(log_level) 18 | 19 | # remove any existing handlers 20 | for h in logger.handlers: 21 | logger.removeHandler(h) 22 | 23 | # only add a new handler if we've not set one yet 24 | if len(logger.handlers) == 0: 25 | fmt = '%(asctime)s.%(msecs)d p%(process)s {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s' 26 | datefmt = '%Y-%m-%d %H:%M:%S' 27 | 28 | ch = logging.StreamHandler(sys.stdout) 29 | ch.setLevel(log_level) 30 | 31 | formatter = logging.Formatter(fmt, datefmt=datefmt) 32 | ch.setFormatter(formatter) 33 | logger.addHandler(ch) 34 | 35 | 36 | class ConfBasic(abc.ABC): 37 | """Abstract Base Class that defines some of the basics for our configuration""" 38 | name = "conf_basic_ABC" 39 | 40 | def __init__(self): 41 | configure_logging(logging.DEBUG) 42 | 43 | def __repr__(self): 44 | """Simplest representation of what a configuration looks like""" 45 | return self.name 46 | 47 | 48 | class ConfDev(ConfBasic): 49 | """Configuration for development scenario""" 50 | name = "dev" 51 | 52 | def __init__(self, overrides): 53 | super().__init__() 54 | self.a_parameter = overrides.get('a_parameter') or 42 55 | 56 | 57 | class ConfTest(ConfBasic): 58 | """Example 2nd configuration, a test scenario""" 59 | name = "test" 60 | 61 | def __init__(self, overrides): 62 | super().__init__() 63 | self.a_parameter = 99 64 | 65 | available_configurations = [ConfDev, ConfTest] 66 | 67 | 68 | def get(configuration=None, overrides={}): 69 | """Return a configuration based on name or environment variable""" 70 | if configuration is None: 71 | configuration = os.getenv(CONFIG_ENV_VAR) 72 | 73 | # look through the available configurations, find the 74 | # match and instantiate it 75 | for c in available_configurations: 76 | if c.name == configuration: 77 | conf = c(overrides) 78 | return conf 79 | 80 | configuration_names = [c.name for c in available_configurations] 81 | print("No configuration matches to '{}', you must pass in a configuration from {}".format(configuration, configuration_names)) 82 | raise ValueError("No matching configuration") 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Example Python project in a package using modules, tests, logging and configurations 2 | 3 | This is a template for a Python 3.4+ project including: 4 | 5 | * module structure with a `setup.py` and a `tests/` folder 6 | * unit test with test environment (via environment variable) allowing configuration via a function call 7 | * main file with argparse template for command line arguments 8 | * config module with inherited configurations so you can easily add your own 9 | * logging configured with ISO-8601 date (parseable by dateutil.parser), process number, file and line number 10 | 11 | ## Installation 12 | 13 | $ python setup.py develop # install locally, it symlinks back to this folder 14 | $ (python setup.py install) # probably you *don't want to do this* as this project is useless by itself (once you've customised it, maybe this is what you want to do...) 15 | 16 | ## Usage 17 | 18 | Python coding template including ENV environment variable configuration:: 19 | 20 | $ python start_here.py --env=dev 21 | $ CONFIG=dev python start_here.py 22 | 23 | Using `--help` will show the default arguments and all your options: 24 | 25 | $ python start_here.py --help 26 | usage: start_here.py [-h] [--env ENV] 27 | 28 | Project description for this prototype... 29 | 30 | optional arguments: 31 | -h, --help show this help message and exit 32 | --env ENV optional configuration argument (default: dev) 33 | 34 | ## Customisation 35 | 36 | * You probably want to edit `logger = logging.getLogger(LOGGER_NAME)` in `config.py` to use your own name (see how it is used in `start_here.py`) 37 | * In `config.py` edit at least `ConfDev` (the default configuration) so it has a useful basic configuration for your problem 38 | * Add some tests in `tests/` 39 | 40 | ## Overriding the configuration 41 | 42 | I've built and used a variety of configuration systems over the years, this is my 'best guess' as to one that works for web microservices (I tend to use Flask), console scripts (e.g. for local usage and Docker) and hacky idea testing. 43 | 44 | As it stands if you don't provide any arguments, the Conf will use whatever's coded in its class (see `config.py`): 45 | 46 | $ python start_here.py 47 | 2015-05-20 16:31:11.625 p16016 {start_here.py:31} INFO - These are our args: Namespace(a_parameter=None, env='dev') 48 | 2015-05-20 16:31:11.625 p16016 {start_here.py:32} INFO - This is our configuration: dev 49 | 2015-05-20 16:31:11.625 p16016 {start_here.py:33} INFO - This is an example log message, logging is configured once config.get() has been called 50 | 2015-05-20 16:31:11.626 p16016 {start_here.py:34} INFO - The value of a_parameter=42 51 | 52 | If you pass in an override parameter, that'll be used in preference: 53 | 54 | $ python start_here.py --a_parameter=1 55 | 2015-05-20 16:32:46.517 p16038 {start_here.py:31} INFO - These are our args: Namespace(a_parameter=1, env='dev') 56 | 2015-05-20 16:32:46.517 p16038 {start_here.py:32} INFO - This is our configuration: dev 57 | 2015-05-20 16:32:46.517 p16038 {start_here.py:33} INFO - This is an example log message, logging is configured once config.get() has been called 58 | 2015-05-20 16:32:46.517 p16038 {start_here.py:34} INFO - The value of a_parameter=1 59 | 60 | Bare in mind that this assumes that `None` (i.e. not provided) signifies that the parameter isn't being overridden, this might not be the case in your application. 61 | 62 | NOTE this override mechanism might be overkill for your needs! Feel free to strip out the `overrides` parameter to simplify everything! Alternatively you might want to pick-up an environment variable for overrides instead of passing them in at the command line. 63 | 64 | ## Further reading 65 | 66 | * http://docs.python-guide.org/en/latest/writing/structure/ How to structure a Python project 67 | * https://python-packaging-user-guide.readthedocs.org/en/latest/ Packaging how-to 68 | 69 | ## Tests 70 | 71 | Use `py.test` (or `nosetests` if you're old-skool) to run tests, remember that you can see stdout if you do `py.test -s` which'll include any logging messages. 72 | 73 | ## Some notes: 74 | 75 | * for unit testing it is useful to add coverage with: https://pypi.python.org/pypi/pytest-cov 76 | * pip is useful for installing, use `pip freeze > requirements.txt` to free versions of the libraries you're using 77 | * when logging prefer to use %r or %s in the message rather than more specific types (e.g. %d) as e.g. a %d will raise another exception if None is received but %r can handle the None and a decimal number 78 | * you might want to add a `utilities` folder for command-line utilities, alongside the `tests` 79 | * https://github.com/heynemann/generator-python-package is a package-generator, that might give you more ideas (and, frnakly, maybe it is a whole pile better - I'm open to input!) thx @zemanel 80 | 81 | # TODO ? 82 | 83 | * in config.py we could auto-discover the available configurations by looking at who inherits from ConfBasic rather than building the list manually 84 | * it would be nice to check that all configurations have a unique `name` and none are in conflict 85 | --------------------------------------------------------------------------------