├── .envrc ├── SECRET ├── requirements.txt ├── setup.cfg ├── Makefile ├── .gitignore ├── .editorconfig ├── setup.py ├── varyaml.py ├── test └── test_varyaml.py └── README.md /.envrc: -------------------------------------------------------------------------------- 1 | layout python3 2 | -------------------------------------------------------------------------------- /SECRET: -------------------------------------------------------------------------------- 1 | secret-content 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml>=3 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | test: 3 | PYTHONPATH=. pytest test/ 4 | 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | env/ 2 | .cache/ 3 | __pycache__/ 4 | *.egg-info/ 5 | dist/ 6 | *.pyc 7 | /.direnv/ 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.{py,yml,md}] 4 | indent_style = space 5 | indent_size = 4 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup( 4 | name='varyaml', 5 | version='0.0.5', 6 | description='yaml parser with environment interpolation -- safe secrets in configs', 7 | url='https://github.com/abe-winter/varyaml', 8 | author='Abe Winter', 9 | author_email='awinter.public@gmail.com', 10 | license='MIT', 11 | classifiers=[ 12 | 'Development Status :: 4 - Beta', 13 | 'Intended Audience :: Developers', 14 | 'License :: OSI Approved :: MIT License', 15 | 'Programming Language :: Python :: 3.5', 16 | ], 17 | keywords='yaml environment config configuration', 18 | py_modules=['varyaml'], 19 | install_requires=['pyyaml>=3'], 20 | extras_require={'test': ['pytest']}, 21 | ) 22 | -------------------------------------------------------------------------------- /varyaml.py: -------------------------------------------------------------------------------- 1 | import yaml, os 2 | 3 | class Omit: pass 4 | 5 | def get_overrides(settings): 6 | "takes the varyaml section of a conf and the passed-in tags; returns matched override section or {}" 7 | overrides = settings.get('overrides', []) 8 | for section in overrides: 9 | assert '__filter__' in section 10 | if any(os.environ.get(key) == val for key, val in section['__filter__'].items()): 11 | return section 12 | return {} 13 | 14 | class Settings: 15 | "some kind of helper to manage interpolation & precedence" 16 | def __init__(self, settings): 17 | self.defaults = settings.get('defaults', {}) 18 | self.overrides = get_overrides(settings) 19 | self.path = settings.get('path') 20 | self.parse_env = settings.get('parse_env') 21 | 22 | def substitute(self, var, path): 23 | if var in os.environ: 24 | return yaml.safe_load(os.environ[var]) if self.parse_env else os.environ[var] 25 | elif self.path and os.path.exists(os.path.join(self.path, var)): 26 | return open(os.path.join(self.path, var)).read() 27 | elif var in self.overrides: 28 | return self.overrides[var] 29 | elif var in self.defaults: 30 | return self.defaults[var] 31 | else: 32 | raise KeyError(var, path) 33 | 34 | def process(item, settings, path=''): 35 | "recursively processes containers. mutates input." 36 | iter_ = None 37 | if isinstance(item, dict): 38 | iter_ = item.items() 39 | elif isinstance(item, list): 40 | iter_ = enumerate(item) 41 | else: 42 | return item 43 | 44 | any_omit = False 45 | pops = [] 46 | for index, val in iter_: 47 | if isinstance(val, (dict, list)): 48 | process(val, settings, '%s/%s' % (path, index)) 49 | elif isinstance(val, str) and val.startswith('$'): 50 | newval = settings.substitute(val[1:], '%s/%s' % (path, index)) 51 | if newval == '__omit__': 52 | if isinstance(item, list): 53 | item[index] = Omit 54 | any_omit = True 55 | elif isinstance(item, dict): 56 | pops.append(index) 57 | else: 58 | item[index] = newval 59 | 60 | if pops: 61 | assert isinstance(item, dict) 62 | for k in pops: 63 | item.pop(k) 64 | if any_omit: 65 | assert isinstance(item, list) 66 | item[:] = (x for x in item if x is not Omit) 67 | 68 | return item 69 | 70 | def load(*args, **kwargs): 71 | data = yaml.safe_load(*args, **kwargs) 72 | settings = Settings(data.get('varyaml', {}) if isinstance(data, dict) else {}) 73 | return process(data, settings) 74 | -------------------------------------------------------------------------------- /test/test_varyaml.py: -------------------------------------------------------------------------------- 1 | import varyaml, pytest, itertools, os, yaml 2 | 3 | os.environ['VY_KEY'] = 'hello' 4 | 5 | def load(raw, **kwargs): 6 | "helper to dump & parse" 7 | return varyaml.load(yaml.dump(raw), **kwargs) 8 | 9 | def test_dict(): 10 | ret = load({'a':'$VY_KEY', 'b':1, 'c':'ok'}) 11 | assert ret['a'] == os.environ['VY_KEY'] 12 | assert ret['b'] == 1 13 | assert ret['c'] == 'ok' 14 | 15 | def test_list(): 16 | ret = load(['$VY_KEY', 1, 'ok']) 17 | assert ret[0] == os.environ['VY_KEY'] 18 | assert ret[1] == 1 19 | assert ret[2] == 'ok' 20 | 21 | def test_dict_default(): 22 | assert 'X1' not in os.environ 23 | assert 'X3' not in os.environ 24 | os.environ['X0'] = '50' 25 | os.environ['X2'] = '60' 26 | ret = load({ 27 | 'omittable':'$X0', 'omitted':'$X1', 'present':'$X2', 'default':'$X3', 28 | 'varyaml':{'defaults':{'X0':'__omit__', 'X1':'__omit__', 'X2':1, 'X3':1}}, 29 | }) 30 | assert ret['omittable'] == '50' 31 | assert 'omitted' not in ret 32 | assert ret['present'] == '60' 33 | assert ret['default'] == 1 34 | 35 | def test_file_lookup(): 36 | assert 'SECRET' not in os.environ 37 | ret = load({'fromfile':'$SECRET', 'varyaml':{'path':'.'}}) 38 | assert ret['fromfile'] == 'secret-content\n' 39 | 40 | def test_missing_error(): 41 | "also test path here" 42 | assert 'MISSING' not in os.environ 43 | with pytest.raises(KeyError) as exc: 44 | load([{'missing':'$MISSING'}]) 45 | assert exc.value.args == ('MISSING', '/0/missing') 46 | 47 | def test_nest_cases(): 48 | assert load({'a':{'b':'$VY_KEY'}})['a']['b'] == 'hello' 49 | assert load({'a':['$VY_KEY']})['a'][0] == 'hello' 50 | assert load([['$VY_KEY']])[0][0] == 'hello' 51 | assert load([{'b':'$VY_KEY'}])[0]['b'] == 'hello' 52 | 53 | def test_overrides(): 54 | conf = { 55 | 'host': '$HOST', 56 | 'varyaml': { 57 | 'defaults': {'HOST': '127.0.0.1'}, 58 | 'overrides': [ 59 | { 60 | '__filter__': {'ENV': 'prod'}, 61 | 'HOST': 'managed-db.cloudhost.com', 62 | }, { 63 | '__filter__': {'ENV': 'test', 'ENV': 'staging'}, 64 | 'HOST': 'db.local', 65 | }, 66 | ], 67 | }, 68 | } 69 | assert load(conf)['host'] == '127.0.0.1' 70 | os.environ['ENV'] = 'prod' 71 | assert load(conf)['host'] == 'managed-db.cloudhost.com' 72 | os.environ['ENV'] = 'staging' 73 | assert load(conf)['host'] == 'db.local' 74 | os.environ['ENV'] = 'nosuchenv' 75 | assert load(conf)['host'] == '127.0.0.1' 76 | 77 | def test_env_parsing(): 78 | "make sure env values are parsed as yaml when parse_env = True" 79 | conf = { 80 | 'yes': '$YES', 81 | 'one': '$ONE', 82 | 'deep': '$DEEP', 83 | 'varyaml': { 84 | 'defaults': {'YES': True, 'ONE': 1, 'DEEP': {'hey': 'ok'}}, 85 | 'parse_env': True, 86 | }, 87 | } 88 | assert load(conf) == {'yes': True, 'one': 1, 'deep': {'hey': 'ok'}, 'varyaml': conf['varyaml']} 89 | # todo: here and elsewhere, reset os.environ overrides after tests 90 | os.environ['YES'] = 'false' 91 | os.environ['ONE'] = '2' 92 | os.environ['DEEP'] = yaml.dump({'hey': 'whatup'}) 93 | assert load(conf) == {'yes': False, 'one': 2, 'deep': {'hey': 'whatup'}, 'varyaml': conf['varyaml']} 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## varyaml 2 | 3 | Ever wanted to reference environment arguments inside your config file? 4 | 5 | ```yaml 6 | db: 7 | host: database.internal.dns 8 | port: $DBPORT 9 | username: $DBUSER 10 | password: rosebud 11 | # todo: make password an env arg for security? nah 12 | ``` 13 | 14 | When you load this file with varyaml, DBPORT and DBUSER will be read from your environment and the loader will **crash** (yes) if they're missing. 15 | 16 | Why is this important? Orchestration frameworks prefer to pass args in as env vars, but programs are easier to understand (and type-check) if they have config files. This tool lets you have both. 17 | 18 | ### Usage 19 | 20 | ```python 21 | import varyaml 22 | CONF = varyaml.load('{}') 23 | # or 24 | CONF = varyaml.load(open('config.yml')) 25 | 26 | print(CONF['db']['host']) 27 | ``` 28 | 29 | ### Installation 30 | 31 | ```bash 32 | pip install varyaml 33 | # or 34 | pip install git+git://github.com/abe-winter/varyaml 35 | ``` 36 | 37 | ### Reading vars from disk 38 | 39 | Some orchestration frameworks put secrets on disk. You can get at them like this: 40 | 41 | ```yaml 42 | db: 43 | password: $DB_PASS 44 | varyaml: 45 | path: /run/secrets 46 | ``` 47 | 48 | In this example, varyaml will check first for an environment var named DB_PASS, then it will read the contents /run/secrets/DB_PASS if the file exists, then it will go to defaults (not provided in this sample), then crash if not found. 49 | 50 | ### Default values 51 | 52 | You can specify defaults (i.e. make the environment var optional). 53 | 54 | (todo: what if the top-level object isn't a dictionary?) 55 | 56 | The special string value `__omit__` says to pop the key if not found. 57 | 58 | ```yaml 59 | db: 60 | port: $DBPORT 61 | username: $DBUSER 62 | database: $DBNAME 63 | varyaml: 64 | defaults: 65 | DBPORT: 5432 66 | DBNAME: __omit__ 67 | path: /run/secrets 68 | ``` 69 | 70 | ### Parsing environment vars as yaml 71 | 72 | When `parse_env` is true in the `varyaml` section, the engine will parse all environment vars as yaml. This is useful for bools, integers, and deep values (i.e. dictionaries) that you want to override. 73 | 74 | Example: 75 | 76 | ```yaml 77 | yes: $YES 78 | varyaml: 79 | defaults: 80 | YES: true 81 | parse_env: true 82 | ``` 83 | 84 | With `parse_env`, setting `YES=false` in the shell will result in a python bool False. Without `parse_env`, you'd get the string `'false'`. 85 | 86 | ### Environment overrides & merging 87 | 88 | The parser takes an `overrides` section which can be used to set different defaults for different environments where your code runs. The order of precedence is: 89 | 90 | * env vars take highest priority 91 | * then on-disk in varyaml.path 92 | * then check for overrides in the current env 93 | * then look in the defaults section 94 | 95 | Overrides are controlled with environment vars. Example (controlled with `APP_ENV`): 96 | 97 | ```yaml 98 | db: 99 | host: $DBHOST 100 | varyaml: 101 | overrides: 102 | - __filter__: {APP_ENV: prod} 103 | DBHOST: managed-db.cloudhost.com 104 | # note: these are 'or' conditions. (env=staging or env=test) 105 | - __filter__: {APP_ENV: staging, APP_ENV: test} 106 | DBHOST: db.local 107 | ``` 108 | 109 | There's a working example in [`test_overrides` in the test suite](https://github.com/abe-winter/varyaml/search?q=test_overrides). 110 | 111 | Design guidance for overrides: 112 | 113 | * Put per-environment overrides in your checked-in config file 114 | * Except anything that needs to be changed a lot, which you should make an env var or fetch from a live config system 115 | * And except secrets, which you should pass as environment vars or on disk 116 | 117 | ### Python versions & release status 118 | 119 | This is in beta release as of June 2017. The test suite is limited and not informed by any real-world snafus. 120 | 121 | I think pyyaml doesn't support yaml aliases, but if it does is there a possibility of circular references? If that happens `varyaml.process` (called by `load`) will hang. 122 | 123 | Tested on py3.5. Probably works on anything recent that supports pyyaml. 124 | 125 | ### Contributions 126 | 127 | * Blog posts on using this in the wild (a) will be appreciated and (b) may be linked to from this spot if they're good. 128 | * I'm not sure if I need env var redirection (i.e. `DEFAULT_HOST=localhost DBHOST='$DEFAULT_HOST'`). If you have a use case and you want to add the feature (disabled by default), send a pull request with tests. 129 | * Escaping for dollar signs in config values (currently they're always converted to environment vars) 130 | --------------------------------------------------------------------------------