├── VERSION ├── test ├── __init__.py ├── __main__.py ├── main_test.py ├── util_test.py ├── s3_test.py ├── cache_test.py └── settings_test.py ├── requirements ├── test.txt └── core.txt ├── MANIFEST.in ├── .gitignore ├── pistachio ├── __main__.py ├── __init__.py ├── util.py ├── main.py ├── cache.py ├── s3.py └── settings.py ├── Makefile ├── .travis.yml ├── setup.py ├── README.md ├── README.rst └── LICENSE /VERSION: -------------------------------------------------------------------------------- 1 | 2.1.7 2 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | mock>=1.0.1 2 | -------------------------------------------------------------------------------- /requirements/core.txt: -------------------------------------------------------------------------------- 1 | PyYAML>=3.10 2 | boto3>=1.2.4 3 | botocore>=1.3.29 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include pistachio * 2 | recursive-include requirements * 3 | include * 4 | global-exclude *.pyc 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.py[co] 3 | __pycache__ 4 | MANIFEST 5 | dist/ 6 | build/ 7 | .idea 8 | .DS_Store 9 | .virtualenv 10 | -------------------------------------------------------------------------------- /pistachio/__main__.py: -------------------------------------------------------------------------------- 1 | from . import main 2 | import yaml 3 | 4 | if __name__ == '__main__': 5 | print(yaml.dump(main.load(), default_flow_style=False)), 6 | -------------------------------------------------------------------------------- /pistachio/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .main import * 4 | 5 | if __name__ == '__main__': 6 | logging.basicConfig(format='%(levelname)s:Pistachio: %(message)s', level=logging.INFO) 7 | -------------------------------------------------------------------------------- /test/__main__.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from test.cache_test import * 4 | from test.main_test import * 5 | from test.settings_test import * 6 | from test.s3_test import * 7 | from test.util_test import * 8 | 9 | if __name__ == '__main__': 10 | unittest.main() 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY : test 2 | 3 | test: 4 | python -m test 5 | 6 | upload: docs 7 | python setup.py sdist upload 8 | 9 | docs: 10 | pandoc --from=markdown --to=rst --output=README.rst README.md 11 | 12 | virtualenv: 13 | virtualenv .virtualenv 14 | .virtualenv/bin/pip install -e . 15 | .virtualenv/bin/pip install -r requirements/test.txt 16 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.3" 5 | - "3.4" 6 | - "nightly" 7 | # command to install dependencies 8 | install: 9 | - "pip install ." 10 | - "pip install -r requirements/test.txt" 11 | # command to run tests 12 | script: make test 13 | # Only run on master and PRs 14 | branches: 15 | only: 16 | - master 17 | -------------------------------------------------------------------------------- /pistachio/util.py: -------------------------------------------------------------------------------- 1 | def merge_dicts(d1, d2): 2 | """ 3 | Recursively update a dict 4 | Merges d2 into d1 5 | Any conflicts override d1 6 | """ 7 | for key in d2: 8 | if key in d1 and isinstance(d1[key], dict) and isinstance(d2[key], dict): 9 | merge_dicts(d1[key], d2[key]) 10 | else: 11 | d1[key] = d2[key] 12 | return d1 13 | 14 | def truthy(string_or_bool): 15 | """ Takes a string or bool as an argument, and returns a bool """ 16 | if string_or_bool == True: 17 | return True 18 | elif string_or_bool == False: 19 | return False 20 | else: 21 | return str(string_or_bool).lower() in ['true'] 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | VERSION = open('VERSION').read().strip() 4 | with open('requirements/core.txt') as f: 5 | INSTALL_REQUIRES = f.read().splitlines() 6 | with open('requirements/test.txt') as f: 7 | TEST_REQUIRES = f.read().splitlines() 8 | 9 | setup( 10 | name='pistachio', 11 | version=VERSION, 12 | author='Jon San Miguel', 13 | author_email='jon.sanmiguel@optimizely.com', 14 | packages=['pistachio'], 15 | url='https://github.com/optimizely/pistachio', 16 | download_url='https://github.com/optimizely/pistachio/tarball/%s' % VERSION, 17 | license=open('LICENSE').read(), 18 | description='Credential Loader for S3 Stored Credentials', 19 | long_description=open('README.rst').read(), 20 | install_requires=INSTALL_REQUIRES, 21 | tests_require=TEST_REQUIRES, 22 | test_suite='test', 23 | ) 24 | -------------------------------------------------------------------------------- /pistachio/main.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from . import cache 4 | from . import s3 5 | from . import settings 6 | 7 | logger = logging.getLogger(__name__) 8 | SETTINGS = settings.load() 9 | memo = None 10 | 11 | 12 | def load(s=SETTINGS): 13 | # Use memoized if available 14 | global memo 15 | if memo: 16 | return memo 17 | 18 | # Validate the settings 19 | s = settings.validate(s) 20 | 21 | # Attempt to load from cache unless disabled 22 | loaded_cache = cache.load(s) 23 | if loaded_cache is not None: 24 | return loaded_cache 25 | 26 | # Otherwise, download from s3, and save to cache 27 | session = s3.create_connection(s) 28 | loaded = s3.download(session, s) 29 | cache.write(s, loaded) 30 | 31 | # Memoize 32 | memo = loaded 33 | 34 | return loaded 35 | 36 | 37 | def attempt_reload(s=SETTINGS): 38 | # Validate the settings 39 | s = settings.validate(s) 40 | 41 | # Attempt to download from s3 and save to cache 42 | try: 43 | session = s3.create_connection(s) 44 | loaded = s3.download(session, s) 45 | cache.write(s, loaded) 46 | # Memoize 47 | global memo 48 | memo = loaded 49 | logger.info('Successfully reloaded cache') 50 | except: 51 | logger.error('Failed to reload cache') 52 | raise 53 | -------------------------------------------------------------------------------- /test/main_test.py: -------------------------------------------------------------------------------- 1 | # Util Test Module 2 | import mock 3 | import unittest 4 | 5 | import pistachio.main 6 | 7 | TEST_CONFIG = { 8 | 'test': 'config', 9 | 'pistachio': {} 10 | } 11 | TEST_SETTINGS = { 12 | 'bucket': 'test', 13 | 'profile': 'test', 14 | 'path': 'test', 15 | } 16 | 17 | # Tests the settings.validate function 18 | class TestMemoization(unittest.TestCase): 19 | 20 | def setUp(self): 21 | pass 22 | 23 | def tearDown(self): 24 | pass 25 | 26 | # Test the memo is returned when set 27 | def test_returns_memo(self): 28 | pistachio.main.memo = TEST_CONFIG 29 | self.assertEqual(pistachio.main.load(TEST_SETTINGS), TEST_CONFIG) 30 | 31 | # Test that memo is set when unset 32 | @mock.patch('pistachio.cache.load', mock.Mock(return_value = None)) 33 | @mock.patch('pistachio.s3.create_connection', mock.Mock(return_value = {})) 34 | @mock.patch('pistachio.s3.download', mock.Mock(return_value = TEST_CONFIG)) 35 | def test_memo_set_on_load(self): 36 | pistachio.main.load(TEST_SETTINGS) 37 | self.assertEqual(pistachio.main.memo, TEST_CONFIG) 38 | 39 | # Test that memo is set when on reload 40 | @mock.patch('pistachio.s3.create_connection', mock.Mock(return_value = {})) 41 | @mock.patch('pistachio.s3.download', mock.Mock(return_value = TEST_CONFIG)) 42 | def test_returns_memo_on_reload(self): 43 | pistachio.main.attempt_reload(TEST_SETTINGS) 44 | self.assertEqual(pistachio.main.memo, TEST_CONFIG) 45 | 46 | # Test that memo is not loaded on a reload 47 | @mock.patch('pistachio.s3.create_connection', mock.Mock(return_value = {})) 48 | @mock.patch('pistachio.s3.download', mock.Mock(return_value = {'fraudulent': 'config', 'pistachio': {}})) 49 | def test_reload_ignores_memo(self): 50 | pistachio.main.attempt_reload(TEST_SETTINGS) 51 | self.assertNotEqual(pistachio.main.memo, TEST_CONFIG) 52 | 53 | 54 | if __name__ == '__main__': 55 | unittest.main() 56 | -------------------------------------------------------------------------------- /test/util_test.py: -------------------------------------------------------------------------------- 1 | # Util Test Module 2 | import unittest 3 | import pistachio.util as util 4 | 5 | # Tests the util.merge_dicts function 6 | class TestMergeDicts(unittest.TestCase): 7 | 8 | def setUp(self): 9 | pass 10 | 11 | # Test the recursive case: both d1 and d2 have nested dicts 12 | def test_recursive_case(self): 13 | d1 = { 'test': { 'overlap': 'this will be overridden', 'd1': 'this should be unaffected' }} 14 | d2 = { 'test': { 'overlap': 'this will override', 'd2': 'this should be merged in' }} 15 | merged = { 16 | 'test': { 17 | 'overlap': 'this will override', 18 | 'd1': 'this should be unaffected', 19 | 'd2': 'this should be merged in' }} 20 | self.assertEqual(util.merge_dicts(d1, d2), merged) 21 | 22 | # Test the base case: d1 is a nested dict 23 | def test_base_case_d1(self): 24 | d1 = { 'test': { 'inner': 'dict' } } 25 | d2 = { 'test': 'string' } 26 | self.assertEqual(util.merge_dicts(d1, d2), d2) 27 | 28 | # Test the base case: d2 is a nested dict 29 | def test_base_case_d2(self): 30 | d1 = { 'test': 'string' } 31 | d2 = { 'test': { 'inner': 'dict' } } 32 | self.assertEqual(util.merge_dicts(d1, d2), d2) 33 | 34 | # Test the base case: neither d1 or d2 have nested dicts 35 | def test_base_case(self): 36 | d1 = { 'test': 'string' } 37 | d2 = { 'test': 'alsostring' } 38 | self.assertEqual(util.merge_dicts(d1, d2), d2) 39 | 40 | 41 | # Tests the util.truthy function 42 | class TestTruthy(unittest.TestCase): 43 | 44 | def setUp(self): 45 | self.truth_map = [ 46 | ('True', True), 47 | ('true', True), 48 | (True, True), 49 | ('False', False), 50 | ('false', False), 51 | (False, False), 52 | ('', False), 53 | (None, False), 54 | ] 55 | 56 | def test_truth_map(self): 57 | for arg, output in self.truth_map: 58 | self.assertEqual(util.truthy(arg), output) 59 | 60 | 61 | if __name__ == '__main__': 62 | unittest.main() 63 | -------------------------------------------------------------------------------- /pistachio/cache.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import yaml 4 | 5 | opened_cache = None 6 | 7 | 8 | def load(settings): 9 | """Attempt to load cache from cache_path""" 10 | if not settings['cache']: 11 | return None 12 | 13 | # Load the file from a cache if one exists and not expired 14 | if is_valid(settings): 15 | return read(settings['cache']) 16 | 17 | # Otherwise return None 18 | return None 19 | 20 | 21 | def write(settings, config): 22 | """Write cache to cache_path""" 23 | if not settings.get('cache'): 24 | return None 25 | 26 | cache = settings['cache'] 27 | cache_disabled = set(cache.get('disable', [])) & set(settings.get('path', [])) 28 | 29 | if settings.get('path') and config.get('pistachio'): 30 | config['pistachio']['path'] = settings['path'] 31 | if cache.get('path') and cache['enabled'] and not cache_disabled: 32 | with open(cache['path'], 'w') as pistachio_cache: 33 | pistachio_cache.write(yaml.safe_dump(config, default_flow_style=False)) 34 | os.chmod(cache['path'], 0o600) 35 | 36 | 37 | def read(cache): 38 | global opened_cache 39 | if not opened_cache: 40 | opened_cache = yaml.load(open(cache['path'], 'r')) 41 | return opened_cache 42 | 43 | 44 | def is_valid(settings): 45 | """Check if cache exists and is valid""" 46 | cache = settings['cache'] 47 | if exists(cache) and is_enabled(cache) and not is_expired(cache): 48 | loaded = read(cache) 49 | 50 | # Check if cache matches the path we want to load from 51 | if settings['path'] == loaded.get('pistachio',{}).get('path', None): 52 | return True 53 | 54 | return False 55 | 56 | 57 | def exists(cache): 58 | return os.path.isfile(cache['path']) 59 | 60 | 61 | def is_enabled(cache): 62 | return cache['enabled'] 63 | 64 | 65 | def is_expired(cache): 66 | """Check if cache is expired. 'expires' in minutes""" 67 | if 'expires' in cache and time.time() - os.path.getmtime(cache['path']) > cache['expires']*60: 68 | return True 69 | return False 70 | -------------------------------------------------------------------------------- /test/s3_test.py: -------------------------------------------------------------------------------- 1 | # S3 Test Module 2 | import boto3 3 | import botocore 4 | import mock 5 | import unittest 6 | 7 | import pistachio.s3 as s3 8 | 9 | 10 | class TestCreateConnection(unittest.TestCase): 11 | """ Tests the s3.create_connection function """ 12 | 13 | def setUp(self): 14 | self.new_valid_settings = { # Pistaciho VERSION 2.0 > pistachio settings 15 | 'profile': 'exists', 16 | 'bucket': 'exists', 17 | } 18 | self.old_valid_settings = { # Pistachio VERSION 1.0 > pistachio settings 19 | 'key': 'exists', 20 | 'secret': 'exists', 21 | 'bucket': 'exists', 22 | } 23 | 24 | def test_using_no_profile_uses_boto3_credentials(self): 25 | test_settings = self.new_valid_settings 26 | del test_settings['profile'] 27 | with mock.patch('boto3.session.Session') as session: 28 | try: 29 | s3.create_connection(test_settings) 30 | except Exception as e: 31 | self.fail(e) 32 | session.assert_called_with() 33 | 34 | def test_using_unknown_profile_fails(self): 35 | test_settings = self.new_valid_settings 36 | test_settings['profile'] = 'anonymous123+_not_a_profile$$$$' 37 | with self.assertRaises(botocore.exceptions.ProfileNotFound): 38 | s3.create_connection(test_settings) 39 | 40 | def test_using_specified_profile_uses_specified_profile(self): 41 | test_settings = self.new_valid_settings 42 | test_settings['profile'] = 'not default' 43 | with mock.patch('boto3.session.Session') as session: 44 | try: 45 | s3.create_connection(test_settings) 46 | except Exception as e: 47 | self.fail(e) 48 | session.assert_called_with(profile_name='not default') 49 | 50 | def test_using_key_and_secret(self): 51 | test_settings = self.old_valid_settings 52 | with mock.patch('boto3.session.Session') as session: 53 | try: 54 | s3.create_connection(test_settings) 55 | except Exception as e: 56 | self.fail(e) 57 | session.assert_called_with(aws_access_key_id='exists', aws_secret_access_key='exists') 58 | 59 | def test_using_deprecated_key_only_uses_boto3_credentials(self): 60 | test_settings = self.old_valid_settings 61 | del test_settings['secret'] 62 | with mock.patch('boto3.session.Session') as session: 63 | try: 64 | s3.create_connection(test_settings) 65 | except Exception as e: 66 | self.fail(e) 67 | session.assert_called_with() 68 | 69 | def test_using_deprecated_secret_only_uses_boto3_credentials(self): 70 | test_settings = self.old_valid_settings 71 | del test_settings['key'] 72 | with mock.patch('boto3.session.Session') as session: 73 | try: 74 | s3.create_connection(test_settings) 75 | except Exception as e: 76 | self.fail(e) 77 | session.assert_called_with() 78 | 79 | def test_using_nothing_uses_boto3_credentials(self): 80 | with mock.patch('boto3.session.Session') as session: 81 | try: 82 | s3.create_connection({}) 83 | except Exception as e: 84 | self.fail(e) 85 | session.assert_called_with() 86 | 87 | 88 | class TestDownload(unittest.TestCase): 89 | """ Tests the s3.download function """ 90 | pass # TODO 91 | 92 | 93 | if __name__ == '__main__': 94 | unittest.main() 95 | -------------------------------------------------------------------------------- /pistachio/s3.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import boto3 3 | import botocore 4 | import logging 5 | import threading 6 | import yaml 7 | 8 | from . import util 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | # This is module level so it can be appended to by multiple threads 13 | config_partials = None 14 | 15 | # rate limit 16 | maxconn = 100 17 | pool = threading.BoundedSemaphore(value=maxconn) 18 | 19 | def create_connection(settings): 20 | """ Creates an S3 connection using AWS credentials """ 21 | # Keys and secrets defined by environment variables 22 | if settings.get('key') and settings.get('secret'): 23 | logger.debug('Using your PISTACHIO_KEY and PISTACHIO_SECRET environment variables') 24 | session = boto3.session.Session(aws_access_key_id=settings['key'], 25 | aws_secret_access_key=settings['secret']) 26 | else: 27 | # Set up session with specified profile or 'default' 28 | if not settings.get('profile'): 29 | session = boto3.session.Session() 30 | logger.debug('Did not specify AWS profile. Defaulting to boto3 credentials.') 31 | else: 32 | session = boto3.session.Session(profile_name=settings['profile']) 33 | logger.debug('Specified AWS profile. Using profile: {}'.format(session.profile_name)) 34 | return session 35 | 36 | 37 | def download(session, settings): 38 | """ Downloads the configs from S3, merges them, and returns a dict """ 39 | # Use Amazon S3 40 | conn = session.resource('s3') 41 | # Specify bucket being accessed 42 | Bucket = conn.Bucket(settings['bucket']) 43 | 44 | # Initialize the config and pistachio config 45 | config = {} 46 | pistachio_config = {'pistachio': { 47 | 'key': settings.get('key'), 48 | 'secret': settings.get('secret'), 49 | 'profile': settings.get('profile'), 50 | 'bucket': Bucket.name, 51 | }} 52 | 53 | # Reset the config_partials array 54 | global config_partials 55 | config_partials = {} 56 | # For each folder 57 | # Must store partials by folder, so that we guarantee folder hierarchy when merging them 58 | for folder in settings['path']: 59 | config_partials[folder] = [] 60 | 61 | # Create thread store if we're running in parallel 62 | if settings['parallel']: 63 | threads = [] 64 | 65 | # Iterate through the folders in the path 66 | for folder in reversed(settings['path']): 67 | # Iterate through yaml files in the set folder 68 | for key in Bucket.objects.filter(Prefix=folder + '/', Delimiter='/'): 69 | if key.key.endswith('.yaml'): 70 | # Download and store 71 | if settings['parallel']: 72 | thread = threading.Thread(target=fetch_config_partial, args=(folder, key)) 73 | thread.start() 74 | threads.append(thread) 75 | else: 76 | fetch_config_partial(folder, key) 77 | 78 | # Wait for the threads to finish if we're running in parallel 79 | if settings['parallel']: 80 | for thread in threads: 81 | # Timeout in 5 seconds 82 | thread.join(5) 83 | 84 | # Merge them together 85 | for folder in reversed(settings['path']): 86 | for config_partial in config_partials[folder]: 87 | util.merge_dicts(config, config_partial) 88 | 89 | if not config: 90 | raise Exception('No credentials were downloaded') 91 | 92 | config.update(pistachio_config) 93 | return config 94 | 95 | 96 | def fetch_config_partial(folder, key): 97 | """ Downloads contents of an S3 file given an S3 key object """ 98 | try: 99 | pool.acquire() 100 | contents = key.get()['Body'].read() 101 | 102 | # Append the config_partials with the downloaded content 103 | global config_partials 104 | config_partials[folder].append(yaml.load(contents)) 105 | 106 | except botocore.exceptions.ClientError as e: 107 | error_code = e.response.get('Error', {}).get('Code', 'Unknown') 108 | if error_code == 'ExpiredToken': 109 | logger.error('Your AWS credentials are expired. Please fetch a new set of credentials.') 110 | raise 111 | else: 112 | logger.warning("S3 exception on %s: %s" % (key, e)) 113 | except: 114 | logger.error("Unexpected error: %s" % sys.exc_info()[0]) 115 | raise 116 | finally: 117 | pool.release() 118 | 119 | -------------------------------------------------------------------------------- /pistachio/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | import stat 3 | import yaml 4 | 5 | from . import cache 6 | from . import util 7 | 8 | """ Special Variables """ 9 | PISTACHIO_FILE_NAME = '.pistachio' 10 | PISTACHIO_ALTERNATIVE_FILE_NAME = 'pistachio.yaml' 11 | 12 | def load(): 13 | """ 14 | Configure pistachio settings. 15 | 16 | Settings are retrieved by walking backwards from the cwd, to the $HOME dir. 17 | Any file in that walk path named PISTACHIO_FILE_NAME or 18 | PISTACHIO_ALTERNATIVE_FILE_NAME is parsed for settings. 19 | """ 20 | settings = {} # Settings 21 | pistachio_files = [] # Pistachio specific files 22 | 23 | # Search bottom up from the current directory for settings files 24 | path = os.getcwd() 25 | 26 | while True: 27 | # Check for PISTACHIO_ALTERNATIVE_FILE_NAME and PISTACHIO_FILE_NAME 28 | for filename in (PISTACHIO_ALTERNATIVE_FILE_NAME, PISTACHIO_FILE_NAME): 29 | file_path = os.path.join(path, filename) 30 | if os.path.isfile(file_path): 31 | pistachio_files.append(file_path) 32 | 33 | # Break out if we're at the root directory 34 | if path == '/': 35 | break 36 | # Otherwise, iterate up to the parent directory 37 | path = os.path.abspath(os.path.join(path, os.pardir)) 38 | 39 | # Check for a PISTACHIO_FILE_NAME file in the HOME directory 40 | if os.getenv('HOME'): 41 | pistachio_settings_path = os.path.abspath( 42 | os.path.join(os.getenv('HOME'), PISTACHIO_FILE_NAME) 43 | ) 44 | if os.path.isfile(pistachio_settings_path): 45 | pistachio_files.append(pistachio_settings_path) 46 | 47 | # Load settings from files 48 | for pistachio_file in reversed(pistachio_files): 49 | settings.update(validate_pistachio_file(pistachio_file)) 50 | 51 | # Override settings from any PISTACHIO environment variables 52 | for var, val in os.environ.items(): 53 | if var == 'PISTACHIO_PATH': 54 | # When pistachio path is set through environment variables, folders are ':' delimited 55 | settings['path'] = val.split(':') 56 | elif var.startswith('PISTACHIO_'): 57 | key = var.split('PISTACHIO_', 1)[1] 58 | settings[key.lower()] = val 59 | 60 | return settings 61 | 62 | 63 | def validate_pistachio_file(file): 64 | with open(file, 'r') as _file: 65 | contents = _file.read().strip() 66 | loaded = yaml.load(contents) 67 | 68 | if not contents or not loaded or loaded == contents: 69 | # If it's still just a regular string, then it's not yaml 70 | raise Exception('%s is not a proper yaml file.' % file) 71 | 72 | # Expand the fullpath of the cache, if set 73 | if 'cache' in loaded: 74 | loaded['cache']['path'] = os.path.abspath(os.path.join(os.path.dirname(file), loaded['cache']['path'])) 75 | 76 | # Warn about open pistachio keys or secrets 77 | if 'key' in loaded or 'secret' in loaded: 78 | mode = oct(stat.S_IMODE(os.stat(file).st_mode)) 79 | if mode not in ['0o600', '0600']: 80 | raise Exception('Pistachio settings file "{0}" contains a key/secret. Mode must be set to "0600" or "0o600", not "{1}"'.format(file, mode)) 81 | print('"{0}" contains key/secret. Please remove key/secret. Using AWS credentials instead...'.format(file)) 82 | loaded.pop('key', None) 83 | loaded.pop('secret', None) 84 | 85 | return loaded 86 | 87 | 88 | # Set the default values for missing fields 89 | def set_defaults(settings): 90 | # Default settings 91 | if 'path' not in settings or settings['path'] is None: 92 | settings['path'] = [''] 93 | 94 | if 'cache' not in settings: 95 | settings['cache'] = {} 96 | else: 97 | settings['cache'].setdefault('enabled', True) 98 | 99 | if 'parallel' not in settings: 100 | settings['parallel'] = False 101 | 102 | return settings 103 | 104 | 105 | # Validate settings and set defaults 106 | def validate(settings): 107 | validation_message = """ 108 | For the settings to be valid it must fulfill any of the following: 109 | 1. Have a valid cache file 110 | 2. Have a bucket defined 111 | """ 112 | 113 | settings = set_defaults(settings) 114 | if 'bucket' not in settings and not (settings['cache'] and cache.is_valid(settings)): 115 | raise ValueError(validation_message) 116 | 117 | # Type conversions 118 | if not isinstance(settings.get('path', []), list): 119 | settings['path'] = [settings['path']] 120 | if not isinstance(settings.get('cache', {}).get('disable', []), list): 121 | settings['cache']['disable'] = [settings['cache']['disable']] 122 | if not isinstance(settings.get('parallel', False), bool): 123 | settings['parallel'] = util.truthy(settings['parallel']) 124 | 125 | return settings 126 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | S3 stored Credential loader module. 2 | ================ 3 | [![Build Status](https://travis-ci.org/optimizely/pistachio.svg?branch=master)](https://travis-ci.org/optimizely/pistachio) 4 | 5 | Copyright Optimizely, Inc., All Rights Reserved. 6 | 7 | The `pistachio` module exists to load credentials stored on S3. 8 | This package works in conjunction with [boto3](https://github.com/boto/boto3) to seamlessly connect you to your Amazon S3 bucket. 9 | This package understands nothing about how your S3 security is managed. 10 | This package assumes it has access to the S3 bucket/folder(s) you set it to use. 11 | 12 | 13 | ## Quickstart 14 | 15 | #### Prerequisite 16 | Set up an AWS profile by running `aws configure` or writing to `~/.aws/credentials`. 17 | More instructions on how to do this are here: 18 | 19 | [Configuring the AWS Command Line Interface](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) 20 | 21 | #### Setup 22 | Put a `pistachio.yaml` or `.pistachio` file in your project repo with the following content: 23 | ``` 24 | bucket: 25 | ``` 26 | Add `pistachio.cache` to your `.gitignore`, so you don't track cache files. 27 | 28 | #### Accessing the loaded config 29 | ``` 30 | import pistachio 31 | 32 | config = pistachio.load() 33 | print config # Print the results 34 | 35 | value = config[] # Access a value 36 | ``` 37 | 38 | #### Under the Hood 39 | When you run `pistachio.load()` it: 40 | - Checks if you have a 'cache' setting 41 | - If so, checks that `path` setting matches Pistachio's `path` value in the cache file, if it exists 42 | - If so, attempts to load from the cache file, if it exists 43 | - Otherwise, loads the config by merging yaml files from specified bucket/folders 44 | - This can be slow, as it has to download each file from S3 over the network 45 | - If 'cache' is set, saves the cache 46 | - Saves the loaded config to `pistachio.CONFIG` 47 | 48 | ## Settings 49 | This is loaded from files named `pistachio.yaml` and `.pistachio`. 50 | Keys set in higher priority files receive precedence and override lower priority files. 51 | 52 | #### Load priority from highest to lowest: 53 | 54 | ##### 1. Environment variables prefixed with `PISTACHIO_` 55 | 56 | `export PISTACHIO_=` would override any keys set in the `pistachio.yaml` or `.pistachio` files 57 | 58 | ##### 2. The `pistachio.yaml` or `.pistachio` files starting from the current working directory, up to the root of the filesystem. 59 | 60 | ``` 61 | ./src/www/pistachio.yaml # Would Override... 62 | ./src/pistachio.yaml # Which Would Override... 63 | ./pistachio.yaml # 64 | ``` 65 | 66 | ##### 3. Lastly the `pistachio.yaml` or `.pistachio` from your $HOME directory if one exists 67 | 68 | This is a good place to set your global pistachio configs 69 | 70 | #### Format of `pistachio.yaml`/`.pistachio` files 71 | 72 | ``` 73 | bucket: STRING 74 | # REQUIRED 75 | # Bucket to load Configs from. 76 | ``` 77 | ``` 78 | profile: STRING 79 | # OPTIONAL 80 | # DEFAULT: 'default' 81 | # AWS Profile containing your key and secret 82 | ``` 83 | ``` 84 | parallel: STRING 85 | # OPTIONAL 86 | # DEFAULT: 'false' 87 | # When set to 'true', s3 downloads run in parallel 88 | ``` 89 | ``` 90 | path: STRING/LIST 91 | # OPTIONAL 92 | # DEFAULT: [''] # All contents of bucket 93 | # Folder(s) within the bucket to load from. 94 | # Can be string or array. 95 | # When an array, folders listed first have higher precedence. 96 | # When setting through ENV variable, folders are ':' delimited. E.g. `PISTACHIO_PATH=folder1:folder2` 97 | # When unset, looks in the root of the bucket. 98 | ``` 99 | ``` 100 | cache: HASH 101 | # OPTIONAL 102 | # DEFAULT: {} 103 | # When unset, does not attempt to load from cache, or save from cache. 104 | path: STRING 105 | # REQUIRED 106 | # Not required if no cache hash is set 107 | # Relative to the pistachio.yaml file, to save/load cache from 108 | expires: INT 109 | # OPTIONAL 110 | # Time in minutes until cache will expire 111 | # When unset, cache will not expire 112 | enabled: BOOLEAN 113 | # OPTIONAL 114 | # DEFAULT: True 115 | # When False, will disable cache 116 | disable: STRING/ARRAY 117 | # OPTIONAL 118 | # Takes in a path, or list of paths. Whenever pistachio loads any 119 | # of those paths, cache will be disabled 120 | ``` 121 | 122 | #### Example pistachio.yaml or .pistachio file 123 | ``` 124 | 125 | # pistachio.yaml 126 | bucket: MyBucket 127 | path: www 128 | ``` 129 | 130 | #### Example environment variables 131 | ``` 132 | $ export PISTACHIO_PROFILE=default 133 | $ export PISTACHIO_BUCKET=MyBucket 134 | $ export PISTACHIO_PATH=www:common 135 | ``` 136 | 137 | #### Example pistachio.yaml or .pistachio file with extra configurations 138 | ``` 139 | # pistachio.yaml 140 | profile: default 141 | bucket: MyBucket 142 | path: 143 | - www 144 | - common 145 | cache: 146 | path: ./pistachio.cache 147 | expires: 60 # minutes 148 | disable: 149 | - prod 150 | ``` 151 | 152 | ## Storing Credentials 153 | Credentials should be uploaded to the respective bucket, and optionally folder, that you are setting pistachio to load from. All files within the specified bucket/folder(s) ending in .yaml will be merged together in alphabetical order. 154 | 155 | Example: 156 | ``` 157 | MyBucket/ 158 | common/ 159 | jenkins.yaml 160 | github.yaml 161 | frontend/ 162 | highcharts.yaml 163 | backend/ 164 | aws.yaml 165 | ``` 166 | 167 | ## Running tests 168 | All tests are in the test/ directory. To run them do the following: 169 | 170 | ``` 171 | python -m test 172 | ``` 173 | -------------------------------------------------------------------------------- /test/cache_test.py: -------------------------------------------------------------------------------- 1 | # Util Test Module 2 | import mock 3 | import io 4 | import sys 5 | import unittest 6 | 7 | import pistachio.cache as cache 8 | 9 | # Get builtins based on python version 10 | if sys.version_info.major == 2: 11 | builtins = '__builtin__' 12 | import __builtin__ as builtins_module 13 | else: 14 | builtins = 'builtins' 15 | import builtins as builtins_module 16 | 17 | 18 | # Tests the cache.load 19 | class TestLoad(unittest.TestCase): 20 | 21 | def setUp(self): 22 | self.default_settings = {'path': '', 'cache': {'enabled': True, 'path': 'exists'}} 23 | self.default_cache = { 24 | 'foo': 'bar', 25 | 'pistachio': { 26 | 'path': '' 27 | } 28 | } 29 | 30 | # Mock isfile to return True 31 | self.isfile_patch = mock.patch('os.path.isfile', mock.Mock(return_value = True)) 32 | self.isfile_patch.start() 33 | 34 | # Freeze the time to two minutes 35 | self.timenow_patch = mock.patch('time.time', mock.Mock(return_value = 120)) # 2 minutes since modified 36 | self.timenow_patch.start() 37 | 38 | # Set modified time to zero 39 | self.modifiedtime_patch = mock.patch('os.path.getmtime', mock.Mock(return_value = 0)) 40 | self.modifiedtime_patch.start() 41 | 42 | # Mock read method to return without reading an actual file 43 | self.read_patch = mock.patch('pistachio.cache.read', mock.Mock(return_value = self.default_cache)) 44 | self.read_patch.start() 45 | 46 | def tearDown(self): 47 | self.isfile_patch.stop() 48 | self.timenow_patch.stop() 49 | self.modifiedtime_patch.stop() 50 | self.read_patch.stop() 51 | 52 | # Test that cache is ignored when empty 53 | def test_cache_no_settings(self): 54 | test_settings = {'cache': {}} 55 | self.assertEqual(cache.load(test_settings), None) 56 | 57 | # Test that cache is ignored when expired 58 | def test_cache_expired(self): 59 | test_settings = self.default_settings 60 | test_settings['cache']['expires'] = 1 # 1 minute 61 | self.assertEqual(cache.load(test_settings), None) 62 | 63 | # Test that cache is loaded when not expired 64 | def test_cache_not_expired(self): 65 | test_settings = self.default_settings 66 | test_settings['cache']['expires'] = 3 # 3 minute 67 | self.assertEqual(cache.load(test_settings), self.default_cache) 68 | 69 | # Test that cache is loaded when no expired is specified 70 | def test_cache_no_expires_setting(self): 71 | test_settings = self.default_settings 72 | self.assertEqual(cache.load(test_settings), self.default_cache) 73 | 74 | # Test that cache is ignored when no disabled 75 | def test_cache_not_enabled(self): 76 | test_settings = self.default_settings 77 | test_settings['cache']['enabled'] = False # 1 minute 78 | self.assertEqual(cache.load(test_settings), None) 79 | 80 | 81 | class TestRead(unittest.TestCase): 82 | 83 | TEST_CACHE_FILE = "'test': 'cache'\n" 84 | 85 | def setUp(self): 86 | self.test_cache_settings = { 87 | 'path': 'fakepath' 88 | } 89 | self.test_cache_dict = {'test': 'cache'} 90 | 91 | def tearDown(self): 92 | pass 93 | 94 | @mock.patch.object(builtins_module, 'open', return_value=TEST_CACHE_FILE) 95 | def test_properly_loads_cache(self, open_mock): 96 | self.assertDictEqual(cache.read(self.test_cache_settings), self.test_cache_dict) 97 | 98 | @mock.patch.object(builtins_module, 'open', return_value=TEST_CACHE_FILE) 99 | def test_memoization(self, open_mock): 100 | for _ in range(2): 101 | cache.read(self.test_cache_settings) 102 | open_mock.assert_called_once_with(self.test_cache_settings['path'], 'r') 103 | 104 | 105 | # Tests the cache.load 106 | class TestWrite(unittest.TestCase): 107 | 108 | def setUp(self): 109 | # Test loaded config 110 | self.test_config = {'test': 'config'} 111 | 112 | # Mock chmod to return True 113 | self.chmod_patch = mock.patch('os.chmod', mock.Mock(return_value = True)) 114 | self.chmod_patch.start() 115 | 116 | def tearDown(self): 117 | self.chmod_patch.stop() 118 | 119 | # Test that cache is not written when no path is set 120 | @mock.patch.object(builtins_module, 'open') 121 | def test_cache_not_set(self, open_mock): 122 | test_settings = {'cache': {}} 123 | cache.write(test_settings, self.test_config) 124 | self.assertFalse(open_mock.called) 125 | 126 | # Test that cache is written when path is set and enabled is set to true 127 | @mock.patch.object(builtins_module, 'open') 128 | def test_cache_enabled_true(self, open_mock): 129 | test_settings = {'cache': {'path': 'exists', 'enabled': True}} 130 | cache.write(test_settings, self.test_config) 131 | self.assertTrue(open_mock.called) 132 | 133 | # Test that cache is not written when path is set and enabled is set to false 134 | @mock.patch.object(builtins_module, 'open') 135 | def test_cache_enabled_false(self, open_mock): 136 | test_settings = {'cache': {'path': 'exists', 'enabled': False}} 137 | cache.write(test_settings, self.test_config) 138 | self.assertFalse(open_mock.called) 139 | 140 | # Test that cache is not written when pistacho loads a path that is disabled in cache 141 | @mock.patch.object(builtins_module, 'open') 142 | def test_cache_disable(self, open_mock): 143 | test_settings = {'cache': {'path': 'exists', 'enabled': True, 'disable': ['prod']}, 'path': ['prod', 'dev']} 144 | cache.write(test_settings, self.test_config) 145 | self.assertFalse(open_mock.called) 146 | 147 | # Test that cache is written when no disabled paths are included within settings 148 | @mock.patch.object(builtins_module, 'open') 149 | def test_cache_not_disable(self, open_mock): 150 | test_settings = {'cache': {'path': 'exists', 'enabled': True, 'disable': ['athena']}, 'path': ['prod', 'dev']} 151 | cache.write(test_settings, self.test_config) 152 | self.assertTrue(open_mock.called) 153 | 154 | if __name__ == '__main__': 155 | unittest.main() 156 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | S3 stored Credential loader module. 2 | =================================== 3 | 4 | |Build Status| 5 | 6 | Copyright Optimizely, Inc., All Rights Reserved. 7 | 8 | | The ``pistachio`` module exists to load credentials stored on S3. 9 | | This package works in conjunction with 10 | `boto3 `__ to seamlessly connect you to 11 | your Amazon S3 bucket. 12 | | This package understands nothing about how your S3 security is 13 | managed. 14 | | This package assumes it has access to the S3 bucket/folder(s) you set 15 | it to use. 16 | 17 | Quickstart 18 | ---------- 19 | 20 | Prerequisite 21 | ^^^^^^^^^^^^ 22 | 23 | Set up an AWS profile by running ``aws configure`` or writing to 24 | ``~/.aws/credentials``. More instructions on how to do this are here: 25 | 26 | `Configuring the AWS Command Line 27 | Interface `__ 28 | 29 | Setup 30 | ^^^^^ 31 | 32 | Put a ``pistachio.yaml`` or ``.pistachio`` file in your project repo 33 | with the following content: 34 | 35 | :: 36 | 37 | bucket: 38 | 39 | Add ``pistachio.cache`` to your ``.gitignore``, so you don't track cache 40 | files. 41 | 42 | Accessing the loaded config 43 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ 44 | 45 | :: 46 | 47 | import pistachio 48 | 49 | config = pistachio.load() 50 | print config # Print the results 51 | 52 | value = config[] # Access a value 53 | 54 | Under the Hood 55 | ^^^^^^^^^^^^^^ 56 | 57 | | When you run ``pistachio.load()`` it: 58 | | - Checks if you have a 'cache' setting - If so, checks that ``path`` 59 | setting matches Pistachio's ``path`` value in the cache file, if it 60 | exists - If so, attempts to load from the cache file, if it exists - 61 | Otherwise, loads the config by merging yaml files from specified 62 | bucket/folders - This can be slow, as it has to download each file 63 | from S3 over the network - If 'cache' is set, saves the cache - Saves 64 | the loaded config to ``pistachio.CONFIG`` 65 | 66 | Settings 67 | -------- 68 | 69 | | This is loaded from files named ``pistachio.yaml`` and ``.pistachio``. 70 | | Keys set in higher priority files receive precedence and override 71 | lower priority files. 72 | 73 | Load priority from highest to lowest: 74 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 75 | 76 | 1. Environment variables prefixed with ``PISTACHIO_`` 77 | ''''''''''''''''''''''''''''''''''''''''''''''''''''' 78 | 79 | ``export PISTACHIO_=`` would override any keys set 80 | in the ``pistachio.yaml`` or ``.pistachio`` files 81 | 82 | 2. The ``pistachio.yaml`` or ``.pistachio`` files starting from the current working directory, up to the root of the filesystem. 83 | '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' 84 | 85 | :: 86 | 87 | ./src/www/pistachio.yaml # Would Override... 88 | ./src/pistachio.yaml # Which Would Override... 89 | ./pistachio.yaml # 90 | 91 | 3. Lastly the ``pistachio.yaml`` or ``.pistachio`` from your $HOME directory if one exists 92 | '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' 93 | 94 | This is a good place to set your global pistachio configs 95 | 96 | Format of ``pistachio.yaml``/``.pistachio`` files 97 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 98 | 99 | :: 100 | 101 | bucket: STRING 102 | # REQUIRED 103 | # Bucket to load Configs from. 104 | 105 | :: 106 | 107 | profile: STRING 108 | # OPTIONAL 109 | # DEFAULT: 'default' 110 | # AWS Profile containing your key and secret 111 | 112 | :: 113 | 114 | parallel: STRING 115 | # OPTIONAL 116 | # DEFAULT: 'false' 117 | # When set to 'true', s3 downloads run in parallel 118 | 119 | :: 120 | 121 | path: STRING/LIST 122 | # OPTIONAL 123 | # DEFAULT: [''] # All contents of bucket 124 | # Folder(s) within the bucket to load from. 125 | # Can be string or array. 126 | # When an array, folders listed first have higher precedence. 127 | # When setting through ENV variable, folders are ':' delimited. E.g. `PISTACHIO_PATH=folder1:folder2` 128 | # When unset, looks in the root of the bucket. 129 | 130 | :: 131 | 132 | cache: HASH 133 | # OPTIONAL 134 | # DEFAULT: {} 135 | # When unset, does not attempt to load from cache, or save from cache. 136 | path: STRING 137 | # REQUIRED 138 | # Not required if no cache hash is set 139 | # Relative to the pistachio.yaml file, to save/load cache from 140 | expires: INT 141 | # OPTIONAL 142 | # Time in minutes until cache will expire 143 | # When unset, cache will not expire 144 | enabled: BOOLEAN 145 | # OPTIONAL 146 | # DEFAULT: True 147 | # When False, will disable cache 148 | disable: STRING/ARRAY 149 | # OPTIONAL 150 | # Takes in a path, or list of paths. Whenever pistachio loads any 151 | # of those paths, cache will be disabled 152 | 153 | Example pistachio.yaml or .pistachio file 154 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 155 | 156 | :: 157 | 158 | 159 | # pistachio.yaml 160 | bucket: MyBucket 161 | path: www 162 | 163 | Example environment variables 164 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 165 | 166 | :: 167 | 168 | $ export PISTACHIO_PROFILE=default 169 | $ export PISTACHIO_BUCKET=MyBucket 170 | $ export PISTACHIO_PATH=www:common 171 | 172 | Example pistachio.yaml or .pistachio file with extra configurations 173 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 174 | 175 | :: 176 | 177 | # pistachio.yaml 178 | profile: default 179 | bucket: MyBucket 180 | path: 181 | - www 182 | - common 183 | cache: 184 | path: ./pistachio.cache 185 | expires: 60 # minutes 186 | disable: 187 | - prod 188 | 189 | Storing Credentials 190 | ------------------- 191 | 192 | Credentials should be uploaded to the respective bucket, and optionally 193 | folder, that you are setting pistachio to load from. All files within 194 | the specified bucket/folder(s) ending in .yaml will be merged together 195 | in alphabetical order. 196 | 197 | Example: 198 | 199 | :: 200 | 201 | MyBucket/ 202 | common/ 203 | jenkins.yaml 204 | github.yaml 205 | frontend/ 206 | highcharts.yaml 207 | backend/ 208 | aws.yaml 209 | 210 | Running tests 211 | ------------- 212 | 213 | All tests are in the test/ directory. To run them do the following: 214 | 215 | :: 216 | 217 | python -m test 218 | 219 | .. |Build Status| image:: https://travis-ci.org/optimizely/pistachio.svg?branch=master 220 | :target: https://travis-ci.org/optimizely/pistachio 221 | -------------------------------------------------------------------------------- /test/settings_test.py: -------------------------------------------------------------------------------- 1 | # Settings Test Module 2 | import mock 3 | import unittest 4 | 5 | import pistachio.settings as settings 6 | 7 | # Tests the settings.validate function 8 | class TestValidate(unittest.TestCase): 9 | 10 | def setUp(self): 11 | self.new_valid_settings = { # Pistaciho VERSION 2.0 > pistachio settings 12 | 'profile': 'exists', 13 | 'bucket': 'exists', 14 | } 15 | self.old_valid_settings = { # Pistachio VERSION 1.0 > pistachio settings 16 | 'key': 'exists', 17 | 'secret': 'exists', 18 | 'bucket': 'exists', 19 | } 20 | self.timenow_patch = mock.patch('time.time', mock.Mock(return_value = 120)) # 2 minutes since modified 21 | self.timenow_patch.start() 22 | self.modifiedtime_patch = mock.patch('os.path.getmtime', mock.Mock(return_value = 0)) 23 | self.modifiedtime_patch.start() 24 | 25 | def tearDown(self): 26 | self.timenow_patch.stop() 27 | self.modifiedtime_patch.stop() 28 | 29 | # Test for the new valid settings 30 | def test_new_valid_settings(self): 31 | test_settings = self.new_valid_settings 32 | try: 33 | settings.validate(test_settings) 34 | except: 35 | self.fail("settings.validate raised an exception unexpectedly!") 36 | 37 | # Test for the old valid settings 38 | def test_old_valid_settings(self): 39 | test_settings = self.old_valid_settings 40 | try: 41 | settings.validate(test_settings) 42 | except: 43 | self.fail("settings.validate raised an exception unexpectedly!") 44 | 45 | # Test that validate passes when the cache exists 46 | @mock.patch('pistachio.cache.read', mock.Mock(return_value = {'pistachio': {'path': ['']}})) 47 | @mock.patch('pistachio.cache.exists', mock.Mock(return_value = True)) 48 | def test_cache_exists(self): 49 | test_settings = {'cache': {'path': 'exists'}} 50 | try: 51 | settings.validate(test_settings) 52 | except: 53 | self.fail("settings.validate raised an exception unexpectedly!") 54 | 55 | # Test that validate passes when the cache exists and expires is valid 56 | @mock.patch('pistachio.cache.read', mock.Mock(return_value = {'pistachio': {'path': ['']}})) 57 | @mock.patch('os.path.isfile', mock.Mock(return_value = True)) 58 | def test_cache_not_expired(self): 59 | test_settings = {'cache': {'path': 'exists', 'expires': 3}} 60 | try: 61 | settings.validate(test_settings) 62 | except: 63 | self.fail("settings.validate raised an exception unexpectedly!") 64 | 65 | # Test validate fails when the cache is expired and we don't have valid settings 66 | @mock.patch('os.path.isfile', mock.Mock(return_value = True)) 67 | def test_cache_expired(self): 68 | test_settings = {'cache': {'path': 'exists', 'expires': 1}} 69 | with self.assertRaises(ValueError): 70 | settings.validate(test_settings) 71 | 72 | # Test that validate fails when the cache doesn't exist, and we dont' have valid settings 73 | @mock.patch('os.path.isfile', mock.Mock(return_value = False)) 74 | def test_cache_does_not_exist(self): 75 | test_settings = {'cache': {'path': 'does not exist'}} 76 | with self.assertRaises(ValueError): 77 | settings.validate(test_settings) 78 | 79 | # Test that validate fails when the cache is disabled, and we dont' have valid settings 80 | @mock.patch('os.path.isfile', mock.Mock(return_value = False)) 81 | def test_cache_disabled(self): 82 | test_settings = {'cache': {'path': 'does not exist', 'enabled': False}} 83 | with self.assertRaises(ValueError): 84 | settings.validate(test_settings) 85 | 86 | # Test that validate() properly sets the default path value 87 | def test_path_default(self): 88 | # Validate 89 | test_settings = self.old_valid_settings 90 | settings.validate(test_settings) 91 | self.assertEqual(test_settings.get('path'), ['']) 92 | 93 | # Test that validate() converts the path to an array 94 | def test_path_type_conversion(self): 95 | test_settings = self.old_valid_settings 96 | test_settings['path'] = 'filepath' 97 | # Validate 98 | settings.validate(test_settings) 99 | self.assertEqual(test_settings.get('path'), ['filepath']) 100 | 101 | # Test that a defined path settings is not overwritten by set_defaults 102 | def test_path_defined(self): 103 | test_settings = self.old_valid_settings 104 | test_settings['path'] = ['filepath'] 105 | # Validate 106 | settings.validate(test_settings) 107 | self.assertEqual(test_settings.get('path'), ['filepath']) 108 | # Default should not affect 109 | settings.set_defaults(test_settings) 110 | self.assertEqual(test_settings.get('path'), ['filepath']) 111 | 112 | # Test that validate() properly sets the default cache value 113 | def test_cache_default(self): 114 | # Validate 115 | test_settings = self.old_valid_settings 116 | settings.validate(test_settings) 117 | self.assertEqual(test_settings.get('cache'), {}) 118 | 119 | # Test that a defined cache settings is not overwritten by set_defaults 120 | def test_cache_defined(self): 121 | test_settings = self.old_valid_settings 122 | test_settings['cache'] = {'a': 'b'} 123 | # Validate 124 | settings.validate(test_settings) 125 | self.assertEqual(test_settings.get('cache'), {'a': 'b', 'enabled': True}) 126 | 127 | # Test that validate() converts cache disabled to an array 128 | def test_cache_disabled_type_conversion(self): 129 | test_settings = self.old_valid_settings 130 | test_settings['cache'] = {'disable':'proddin'} 131 | # Validate 132 | settings.validate(test_settings) 133 | self.assertEqual(test_settings['cache']['disable'], ['proddin']) 134 | 135 | # Test that it does not require the 'key' key 136 | def test_no_key(self): 137 | test_settings = self.old_valid_settings 138 | del test_settings['key'] 139 | try: 140 | settings.validate(test_settings) 141 | except Exception as e: 142 | self.fail(e) 143 | 144 | # Test that it does not require the 'secret' key 145 | def test_no_secret(self): 146 | test_settings = self.old_valid_settings 147 | del test_settings['secret'] 148 | try: 149 | settings.validate(test_settings) 150 | except Exception as e: 151 | self.fail(e) 152 | 153 | # Test that it requires the 'bucket' key 154 | def test_bucket_required(self): 155 | test_settings = self.old_valid_settings 156 | del test_settings['bucket'] 157 | with self.assertRaises(ValueError): 158 | settings.validate(test_settings) 159 | 160 | # Test that validate passes when no key/secret is given 161 | def test_no_key_or_secret(self): 162 | test_settings = self.old_valid_settings 163 | del test_settings['key'] 164 | del test_settings['secret'] 165 | try: 166 | settings.validate(test_settings) 167 | except Exception as e: 168 | self.fail(e) 169 | 170 | # Test that validate() properly sets the default parallel value 171 | def test_parallel_default(self): 172 | test_settings = self.old_valid_settings 173 | # Validate 174 | settings.validate(test_settings) 175 | self.assertEqual(test_settings.get('parallel'), False) 176 | 177 | # Test that a defined parallel settings is not overwritten by set_defaults 178 | def test_parallel_defined(self): 179 | # Validate 180 | test_settings = self.old_valid_settings 181 | test_settings['parallel'] = True 182 | settings.validate(test_settings) 183 | self.assertEqual(test_settings.get('parallel'), True) 184 | 185 | # Test that validate() properly sets parllel to True when 'true' is passed in as a string 186 | def test_parallel_true_string(self): 187 | test_settings = self.old_valid_settings 188 | test_settings['parallel'] = 'true' 189 | settings.validate(test_settings) 190 | self.assertTrue(test_settings.get('parallel')) 191 | 192 | if __name__ == '__main__': 193 | unittest.main() 194 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------