├── berry
    ├── __init__.py
    ├── __main__.py
    └── cli.py
├── tox.ini
├── requirements.txt
├── MANIFEST.in
├── MAINTAINERS
├── .gitignore
├── .coveragerc
├── .travis.yml
├── LICENSE
├── release.sh
├── README.rst
├── setup.py
└── tests
    └── test_cli.py


/berry/__init__.py:
--------------------------------------------------------------------------------
1 | __version__ = '0.15'
2 | 


--------------------------------------------------------------------------------
/tox.ini:
--------------------------------------------------------------------------------
1 | [flake8]
2 | max-line-length=120
3 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | boto3>=1.2.3
2 | PyYAML
3 | dnspython>=1.15.0
4 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include *.txt
2 | include *.rst
3 | recursive-include berry *.py
4 | 


--------------------------------------------------------------------------------
/MAINTAINERS:
--------------------------------------------------------------------------------
1 | Henning Jacobs <henning.jacobs@zalando.de>
2 | team-foundation@zalando.de
3 | 


--------------------------------------------------------------------------------
/berry/__main__.py:
--------------------------------------------------------------------------------
1 | import berry.cli
2 | 
3 | if __name__ == '__main__':
4 |     exit(berry.cli.main())
5 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # vim
 2 | *.swp
 3 | 
 4 | # Python
 5 | *.egg*
 6 | __pycache__
 7 | .coverage
 8 | junit.xml
 9 | coverage.xml
10 | dist/
11 | *.pyc
12 | htmlcov
13 | .cache/
14 | 


--------------------------------------------------------------------------------
/.coveragerc:
--------------------------------------------------------------------------------
1 | [report]
2 | # Regexes for lines to exclude from consideration
3 | exclude_lines =
4 |     # Have to re-enable the standard pragma
5 |     pragma: no cover
6 | 
7 |     if __name__ == .__main__.:
8 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: python
 2 | python:
 3 |   - "3.4"
 4 |   - "2.7"
 5 | install:
 6 |   - pip install --upgrade setuptools -r requirements.txt
 7 |   - pip install coveralls
 8 | script:
 9 |   - python setup.py test
10 |   - python setup.py flake8
11 | after_success:
12 |   - coveralls
13 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright 2015 Zalando SE
 2 | 
 3 | Licensed under the Apache License, Version 2.0 (the "License");
 4 | you may not use this file except in compliance with the License.
 5 | You may obtain a copy of the License at
 6 | 
 7 |   http://www.apache.org/licenses/LICENSE-2.0
 8 | 
 9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 | 


--------------------------------------------------------------------------------
/release.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | 
 3 | if [ $# -ne 1 ]; then
 4 |     >&2 echo "usage: $0 <version>"
 5 |     exit 1
 6 | fi
 7 | 
 8 | set -xe
 9 | 
10 | python3 --version
11 | git --version
12 | 
13 | version=$1
14 | 
15 | sed -i "s/__version__ = .*/__version__ = '${version}'/" */__init__.py
16 | 
17 | # Do not tag/push on Go CD
18 | if [ -z "$GO_PIPELINE_LABEL" ]; then
19 |     python3 setup.py clean
20 |     python3 setup.py test
21 |     python3 setup.py flake8
22 | 
23 |     git add */__init__.py
24 | 
25 |     git commit -m "Bumped version to $version"
26 |     git push
27 | fi
28 | 
29 | python3 setup.py sdist bdist_wheel upload
30 | 
31 | if [ -z "$GO_PIPELINE_LABEL" ]; then
32 |     git tag ${version}
33 |     git push --tags
34 | fi
35 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
 1 | =====
 2 | berry
 3 | =====
 4 | 
 5 | .. image:: https://travis-ci.org/zalando-stups/berry.svg?branch=master
 6 |    :target: https://travis-ci.org/zalando-stups/berry
 7 |    :alt: Build Status
 8 | 
 9 | .. image:: https://coveralls.io/repos/zalando-stups/berry/badge.svg
10 |    :target: https://coveralls.io/r/zalando-stups/berry
11 |    :alt: Code Coverage
12 | 
13 | .. image:: https://img.shields.io/pypi/dw/stups-berry.svg
14 |    :target: https://pypi.python.org/pypi/stups-berry/
15 |    :alt: PyPI Downloads
16 | 
17 | .. image:: https://img.shields.io/pypi/v/stups-berry.svg
18 |    :target: https://pypi.python.org/pypi/stups-berry/
19 |    :alt: Latest PyPI version
20 | 
21 | .. image:: https://img.shields.io/pypi/l/stups-berry.svg
22 |    :target: https://pypi.python.org/pypi/stups-berry/
23 |    :alt: License
24 | 
25 | Berry is the partner component for `mint`_. Berry is a tiny agent, that
26 | constantly updates the local credentials file, so that applications can read their most recent passwords easily.
27 | 
28 | Installation
29 | ============
30 | 
31 | Python 2.7+ is required.
32 | 
33 | .. code-block:: bash
34 | 
35 |     $ sudo pip3 install --upgrade stups-berry
36 | 
37 | Usage
38 | =====
39 | 
40 | See the help for configuration options:
41 | 
42 | .. code-block:: bash
43 | 
44 |     $ berry --help
45 | 
46 | In addition, berry takes all the `standard AWS SDK inputs`_
47 | (local credentials file, environment variables and instance profiles).
48 | 
49 | License
50 | =======
51 | 
52 | Copyright © 2015 Zalando SE
53 | 
54 | Licensed under the Apache License, Version 2.0 (the "License");
55 | you may not use this file except in compliance with the License.
56 | You may obtain a copy of the License at
57 | 
58 |     http://www.apache.org/licenses/LICENSE-2.0
59 | 
60 | Unless required by applicable law or agreed to in writing, software
61 | distributed under the License is distributed on an "AS IS" BASIS,
62 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
63 | See the License for the specific language governing permissions and
64 | limitations under the License.
65 | 
66 | .. _mint: http://docs.stups.io/en/latest/components/mint.html
67 | .. _standard AWS SDK inputs: http://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs
68 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | import sys
  5 | import os
  6 | import inspect
  7 | 
  8 | import setuptools
  9 | from setuptools.command.test import test as TestCommand
 10 | from setuptools import setup
 11 | from pip.req import parse_requirements
 12 | 
 13 | if sys.version_info < (2, 7, 0):
 14 |     sys.stderr.write('FATAL: STUPS berry needs to be run with Python 2.7+\n')
 15 |     sys.exit(1)
 16 | 
 17 | __location__ = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())))
 18 | 
 19 | 
 20 | def read_version(package):
 21 |     data = {}
 22 |     with open(os.path.join(package, '__init__.py'), 'r') as fd:
 23 |         exec(fd.read(), data)
 24 |     return data['__version__']
 25 | 
 26 | 
 27 | NAME = 'stups-berry'
 28 | MAIN_PACKAGE = 'berry'
 29 | VERSION = read_version(MAIN_PACKAGE)
 30 | DESCRIPTION = 'Credentials distribution agent'
 31 | LICENSE = 'Apache License 2.0'
 32 | URL = 'https://github.com/zalando-stups/berry'
 33 | AUTHOR = 'Henning Jacobs'
 34 | EMAIL = 'henning.jacobs@zalando.de'
 35 | KEYWORDS = 'credentials distribution aws s3 stups'
 36 | 
 37 | COVERAGE_XML = True
 38 | COVERAGE_HTML = False
 39 | JUNIT_XML = True
 40 | 
 41 | # Add here all kinds of additional classifiers as defined under
 42 | # https://pypi.python.org/pypi?%3Aaction=list_classifiers
 43 | CLASSIFIERS = [
 44 |     'Development Status :: 4 - Beta',
 45 |     'Environment :: Console',
 46 |     'Intended Audience :: System Administrators',
 47 |     'License :: OSI Approved :: Apache Software License',
 48 |     'Operating System :: POSIX :: Linux',
 49 |     'Programming Language :: Python',
 50 |     'Programming Language :: Python :: 2.7',
 51 |     'Programming Language :: Python :: 3.4',
 52 |     'Programming Language :: Python :: Implementation :: CPython',
 53 | ]
 54 | 
 55 | CONSOLE_SCRIPTS = ['berry = berry.cli:main']
 56 | 
 57 | 
 58 | class PyTest(TestCommand):
 59 | 
 60 |     user_options = [('cov=', None, 'Run coverage'), ('cov-xml=', None, 'Generate junit xml report'), ('cov-html=',
 61 |                     None, 'Generate junit html report'), ('junitxml=', None, 'Generate xml of test results')]
 62 | 
 63 |     def initialize_options(self):
 64 |         TestCommand.initialize_options(self)
 65 |         self.cov = None
 66 |         self.cov_xml = False
 67 |         self.cov_html = False
 68 |         self.junitxml = None
 69 | 
 70 |     def finalize_options(self):
 71 |         TestCommand.finalize_options(self)
 72 |         if self.cov is not None:
 73 |             self.cov = ['--cov', self.cov, '--cov-report', 'term-missing']
 74 |             if self.cov_xml:
 75 |                 self.cov.extend(['--cov-report', 'xml'])
 76 |             if self.cov_html:
 77 |                 self.cov.extend(['--cov-report', 'html'])
 78 |         if self.junitxml is not None:
 79 |             self.junitxml = ['--junitxml', self.junitxml]
 80 | 
 81 |     def run_tests(self):
 82 |         try:
 83 |             import pytest
 84 |         except:
 85 |             raise RuntimeError('py.test is not installed, run: pip install pytest')
 86 |         params = {'args': self.test_args}
 87 |         if self.cov:
 88 |             params['args'] += self.cov
 89 |         if self.junitxml:
 90 |             params['args'] += self.junitxml
 91 |         params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-s']
 92 |         errno = pytest.main(**params)
 93 |         sys.exit(errno)
 94 | 
 95 | 
 96 | def get_install_requirements(path):
 97 |     # parse_requirements() returns generator of pip.req.InstallRequirement objects
 98 |     install_reqs = parse_requirements(path, session=False)
 99 |     return [str(ir.req) for ir in install_reqs if ir.match_markers()]
100 | 
101 | 
102 | def read(fname):
103 |     return open(os.path.join(__location__, fname)).read()
104 | 
105 | 
106 | def setup_package():
107 |     # Assemble additional setup commands
108 |     cmdclass = {}
109 |     cmdclass['test'] = PyTest
110 | 
111 |     # Some helper variables
112 |     version = VERSION
113 | 
114 |     install_reqs = get_install_requirements('requirements.txt')
115 | 
116 |     command_options = {'test': {'test_suite': ('setup.py', 'tests'), 'cov': ('setup.py', MAIN_PACKAGE)}}
117 |     if JUNIT_XML:
118 |         command_options['test']['junitxml'] = 'setup.py', 'junit.xml'
119 |     if COVERAGE_XML:
120 |         command_options['test']['cov_xml'] = 'setup.py', True
121 |     if COVERAGE_HTML:
122 |         command_options['test']['cov_html'] = 'setup.py', True
123 | 
124 |     setup(
125 |         name=NAME,
126 |         version=version,
127 |         url=URL,
128 |         description=DESCRIPTION,
129 |         author=AUTHOR,
130 |         author_email=EMAIL,
131 |         license=LICENSE,
132 |         keywords=KEYWORDS,
133 |         long_description=read('README.rst'),
134 |         classifiers=CLASSIFIERS,
135 |         test_suite='tests',
136 |         packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
137 |         install_requires=install_reqs,
138 |         setup_requires=['six', 'flake8'],
139 |         cmdclass=cmdclass,
140 |         tests_require=['pytest-cov', 'pytest', 'mock'],
141 |         command_options=command_options,
142 |         entry_points={'console_scripts': CONSOLE_SCRIPTS},
143 |     )
144 | 
145 | 
146 | if __name__ == '__main__':
147 |     setup_package()
148 | 


--------------------------------------------------------------------------------
/berry/cli.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | 
  3 | import argparse
  4 | import boto3.session
  5 | import botocore.exceptions
  6 | import json
  7 | import logging
  8 | import os
  9 | import yaml
 10 | import time
 11 | import dns.resolver
 12 | from botocore.client import Config
 13 | 
 14 | 
 15 | class UsageError(Exception):
 16 |     def __init__(self, msg):
 17 |         self.msg = msg
 18 | 
 19 |     def __str__(self):
 20 |         return 'Usage Error: {}'.format(self.msg)
 21 | 
 22 | 
 23 | def get_bucket_region(client, bucket_name, endpoint):
 24 |     try:
 25 |         return client.get_bucket_location(Bucket=bucket_name).get('LocationConstraint')
 26 |     except botocore.exceptions.ClientError as e:
 27 |         if e.response['Error'].get('Code') != 'AccessDenied':
 28 |             logging.error('Unkown Error on get_bucket_location({})! (S3 error message: {})'.format(bucket_name, e))
 29 |     if endpoint.endswith('.amazonaws.com'):
 30 |         endpoint_parts = endpoint.split('.')
 31 |         if endpoint_parts[-3].startswith('s3-'):
 32 |             return endpoint_parts[-3].replace('s3-', '')
 33 |     bucket_dns = '{}.s3.amazonaws.com'.format(bucket_name)
 34 |     try:
 35 |         answers = dns.resolver.query(bucket_dns, 'CNAME')
 36 |         if len(answers) == 1:
 37 |             answer = answers[0]
 38 |             if (len(answer.target) == 5 and
 39 |                     str(answer.target).endswith('.amazonaws.com.') and
 40 |                     str(answer.target).startswith('s3')):
 41 |                 return answer.target.labels[1].decode()
 42 |             if (len(answer.target) == 4 and
 43 |                     str(answer.target).endswith('.amazonaws.com.') and
 44 |                     str(answer.target).startswith('s3-')):
 45 |                 return answer.target.labels[0].decode().replace('s3-', '')
 46 |             logging.error('Unsupportet DNS response for {}: {}'.format(bucket_dns, answer))
 47 |         logging.error('Two many entrys ({}) for DNS Name {}'.format(len(answers), bucket_dns))
 48 |     except Exception as e:
 49 |         logging.exception('Unsupportet Exception: {}'.format(e))
 50 |     return None
 51 | 
 52 | 
 53 | def lookup_aws_credentials(application_id, path):
 54 |     with open(path) as fd:
 55 |         for line in fd:
 56 |             line = line.strip()
 57 |             if not line.startswith('#'):
 58 |                 parts = line.split(':')
 59 |                 if parts[0] == application_id:
 60 |                     return parts[1], parts[2]
 61 |     return None, None
 62 | 
 63 | 
 64 | def use_aws_credentials(application_id, path):
 65 |     access_key_id, secret_access_key = lookup_aws_credentials(application_id, path)
 66 |     if not access_key_id:
 67 |         raise UsageError('No AWS credentials found for application "{}" in {}'.format(application_id, path))
 68 |     return {'aws_access_key_id': access_key_id, 'aws_secret_access_key': secret_access_key}
 69 | 
 70 | 
 71 | def run_berry(args):
 72 |     try:
 73 |         with open(args.config_file) as fd:
 74 |             config = yaml.load(fd)
 75 |     except Exception as e:
 76 |         logging.warn('Could not load configuration from {}: {}'.format(args.config_file, e))
 77 |         config = {}
 78 | 
 79 |     application_id = args.application_id or config.get('application_id')
 80 |     mint_bucket = args.mint_bucket or config.get('mint_bucket')
 81 |     local_directory = args.local_directory
 82 | 
 83 |     if not application_id:
 84 |         raise UsageError('Application ID missing, please set "application_id" in your configuration YAML')
 85 | 
 86 |     if not mint_bucket:
 87 |         raise UsageError('Mint Bucket is not configured, please set "mint_bucket" in your configuration YAML')
 88 | 
 89 |     while True:
 90 |         if args.aws_credentials_file:
 91 |             aws_credentials = use_aws_credentials(application_id, args.aws_credentials_file)
 92 |         else:
 93 |             aws_credentials = {}
 94 | 
 95 |         session = boto3.session.Session(**aws_credentials)
 96 |         s3 = session.client('s3')
 97 |         err_count = 0
 98 |         for fn in ['user', 'client']:
 99 |             key_name = '{}/{}.json'.format(application_id, fn)
100 |             try:
101 |                 local_file = os.path.join(local_directory, '{}.json'.format(fn))
102 |                 tmp_file = local_file + '.tmp'
103 |                 response = None
104 |                 retry = 3
105 |                 while retry:
106 |                     try:
107 |                         response = s3.get_object(Bucket=mint_bucket, Key=key_name)
108 |                         retry = False
109 |                     except botocore.exceptions.ClientError as e:
110 |                         # more friendly error messages
111 |                         # https://github.com/zalando-stups/berry/issues/2
112 |                         status_code = e.response.get('ResponseMetadata', {}).get('HTTPStatusCode')
113 |                         msg = e.response['Error'].get('Message')
114 |                         error_code = e.response['Error'].get('Code')
115 |                         endpoint = e.response['Error'].get('Endpoint', '')
116 |                         retry -= 1
117 |                         if error_code == 'InvalidRequest' and 'Please use AWS4-HMAC-SHA256.' in msg:
118 |                             logging.debug(('Invalid Request while trying to read "{}" from mint S3 bucket "{}". ' +
119 |                                            'Retrying with signature version v4! ' +
120 |                                            '(S3 error message: {})').format(
121 |                                          key_name, mint_bucket, msg))
122 |                             s3 = session.client('s3', config=Config(signature_version='s3v4'))
123 |                         elif error_code == 'PermanentRedirect' and endpoint.endswith('.amazonaws.com'):
124 |                             region = get_bucket_region(s3, mint_bucket, endpoint)
125 |                             logging.debug(('Got Redirect while trying to read "{}" from mint S3 bucket "{}". ' +
126 |                                            'Retrying with region {}, endpoint {}! ' +
127 |                                            '(S3 error message: {})').format(
128 |                                          key_name, mint_bucket, region, endpoint, msg))
129 |                             s3 = session.client('s3', region)
130 |                         elif status_code == 403:
131 |                             logging.error(('Access denied while trying to read "{}" from mint S3 bucket "{}". ' +
132 |                                            'Check your IAM role/user policy to allow read access! ' +
133 |                                            '(S3 error message: {})').format(
134 |                                           key_name, mint_bucket, msg))
135 |                             retry = False
136 |                             err_count += 1
137 |                         elif status_code == 404:
138 |                             logging.error(('Credentials file "{}" not found in mint S3 bucket "{}". ' +
139 |                                            'Mint either did not sync them yet or the mint configuration is wrong. ' +
140 |                                            '(S3 error message: {})').format(
141 |                                           key_name, mint_bucket, msg))
142 |                             retry = False
143 |                             err_count += 1
144 |                         else:
145 |                             logging.error('Could not read from mint S3 bucket "{}": {}'.format(
146 |                                           mint_bucket, e))
147 |                             retry = False
148 |                             err_count += 1
149 | 
150 |                 if response:
151 |                     body = response['Body']
152 |                     json_data = body.read()
153 | 
154 |                     # check that the file contains valid JSON
155 |                     new_data = json.loads(json_data.decode('utf-8'))
156 | 
157 |                     try:
158 |                         with open(local_file, 'r') as fd:
159 |                             old_data = json.load(fd)
160 |                     except:
161 |                         old_data = None
162 |                     # check whether the file contents changed
163 |                     if new_data != old_data:
164 |                         with open(tmp_file, 'wb') as fd:
165 |                             fd.write(json_data)
166 |                         os.rename(tmp_file, local_file)
167 |                         logging.info('Rotated {} credentials for {}'.format(fn, application_id))
168 |             except:
169 |                 logging.exception('Failed to download {} credentials'.format(fn))
170 |                 err_count += 1
171 | 
172 |         if args.once:
173 |             return err_count == 0
174 | 
175 |         time.sleep(args.interval)  # pragma: no cover
176 | 
177 | 
178 | def configure():
179 |     parser = argparse.ArgumentParser()
180 |     parser.add_argument('local_directory', help='Local directory to write credentials to')
181 |     parser.add_argument('-f', '--config-file', help='Read berry settings from given YAML file',
182 |                         default='/etc/taupage.yaml')
183 |     parser.add_argument('-a', '--application-id', help='Application ID as registered in Kio')
184 |     parser.add_argument('-m', '--mint-bucket', help='Mint S3 bucket name')
185 |     parser.add_argument('-c', '--aws-credentials-file',
186 |                         help='Lookup AWS credentials by application ID in the given file')
187 |     parser.add_argument('-i', '--interval', help='Interval in seconds', type=int, default=120)
188 |     parser.add_argument('--once', help='Download credentials once and exit', action='store_true')
189 |     parser.add_argument('-s', '--silent', action='store_true',
190 |                         help='silent output - only errors will be displayed')
191 |     args = parser.parse_args()
192 |     log_level = logging.ERROR if args.silent else logging.INFO
193 |     logging.basicConfig(level=log_level, format='%(levelname)s: %(message)s')
194 |     # do not log new HTTPS connections (INFO level):
195 |     logging.getLogger('botocore.vendored.requests').setLevel(logging.WARN)
196 |     return args
197 | 
198 | 
199 | def main():
200 |     args = configure()
201 |     try:
202 |         if run_berry(args):
203 |             return 0
204 |         else:
205 |             return 1
206 |     except UsageError as e:
207 |         logging.error(str(e))
208 |         return 1
209 | 
210 | 
211 | if __name__ == '__main__':
212 |     exit(main())
213 | 


--------------------------------------------------------------------------------
/tests/test_cli.py:
--------------------------------------------------------------------------------
  1 | 
  2 | import botocore.exceptions
  3 | import logging
  4 | import os
  5 | import pytest
  6 | import yaml
  7 | import dns
  8 | 
  9 | from berry.cli import use_aws_credentials, run_berry, main, UsageError
 10 | import berry.cli
 11 | from mock import MagicMock
 12 | 
 13 | 
 14 | def test_use_aws_credentials(tmpdir):
 15 |     p = tmpdir.join('aws-creds')
 16 |     p.write('# my comment\nsomeapp:foo:bar\nmyapp:abc123:456789\nblub:a:b')
 17 |     creds = use_aws_credentials('myapp', str(p))
 18 | 
 19 |     assert creds == {'aws_access_key_id': 'abc123', 'aws_secret_access_key': '456789'}
 20 | 
 21 | 
 22 | def mock_session(client):
 23 |     session = MagicMock()
 24 |     session.client.return_value = client
 25 |     return lambda **kwargs: session
 26 | 
 27 | 
 28 | def test_rotate_credentials(monkeypatch, tmpdir, capsys):
 29 |     response = MagicMock()
 30 |     response['Body'].read.return_value = b'{"application_username": "myteam_myapp", "application_password": "secret"}'
 31 | 
 32 |     s3 = MagicMock()
 33 |     s3.get_object.return_value = response
 34 |     monkeypatch.setattr('boto3.session.Session', mock_session(s3))
 35 |     monkeypatch.setattr('time.sleep', lambda x: 0)
 36 | 
 37 |     args = MagicMock()
 38 |     args.application_id = 'myapp'
 39 |     args.config_file = str(tmpdir.join('taupage.yaml'))
 40 |     args.once = True
 41 |     args.aws_credentials_file = None
 42 |     args.local_directory = str(tmpdir.join('credentials'))
 43 | 
 44 |     os.makedirs(args.local_directory)
 45 | 
 46 |     logging.basicConfig(level=logging.INFO)
 47 |     assert run_berry(args) is True
 48 |     out, err = capsys.readouterr()
 49 |     assert 'Rotated user credentials for myapp' in err
 50 |     assert 'Rotated client credentials for myapp' in err
 51 | 
 52 |     # https://github.com/zalando-stups/berry/issues/4
 53 |     # check that we don't rotate/write the file if it wasn't changed
 54 |     assert run_berry(args) is True
 55 |     out, err = capsys.readouterr()
 56 |     assert 'Rotated' not in err
 57 | 
 58 | 
 59 | def test_rotate_credentials_with_file_config(monkeypatch, tmpdir):
 60 |     response = MagicMock()
 61 |     response['Body'].read.return_value = b'{"application_username": "myteam_myapp", "application_password": "secret"}'
 62 | 
 63 |     s3 = MagicMock()
 64 |     s3.get_object.return_value = response
 65 |     monkeypatch.setattr('boto3.session.Session', mock_session(s3))
 66 |     monkeypatch.setattr('time.sleep', lambda x: 0)
 67 | 
 68 |     log_info = MagicMock()
 69 |     monkeypatch.setattr('logging.warn', MagicMock())
 70 |     monkeypatch.setattr('logging.info', log_info)
 71 | 
 72 |     config_path = str(tmpdir.join('taupage.yaml'))
 73 |     with open(config_path, 'w') as fd:
 74 |         yaml.safe_dump({'application_id': 'someapp'}, fd)
 75 | 
 76 |     credentials_path = str(tmpdir.join('credentials'))
 77 |     with open(credentials_path, 'w') as fd:
 78 |         fd.write('someapp:foo:bar')
 79 | 
 80 |     args = MagicMock()
 81 |     args.application_id = None
 82 |     args.config_file = config_path
 83 |     args.once = True
 84 |     args.aws_credentials_file = credentials_path
 85 |     args.local_directory = str(tmpdir.join('out'))
 86 | 
 87 |     os.makedirs(args.local_directory)
 88 | 
 89 |     assert run_berry(args) is True
 90 |     log_info.assert_called_with('Rotated client credentials for someapp')
 91 | 
 92 |     args.application_id = 'wrongapp'
 93 |     with pytest.raises(UsageError) as excinfo:
 94 |         assert run_berry(args) is False
 95 |     assert 'No AWS credentials found for application "wrongapp" in' in excinfo.value.msg
 96 | 
 97 | 
 98 | def test_main_noargs(monkeypatch):
 99 |     monkeypatch.setattr('sys.argv', ['berry'])
100 |     try:
101 |         main()
102 |         assert False
103 |     except SystemExit:
104 |         pass
105 | 
106 | 
107 | def test_run_berry_status(monkeypatch):
108 |     orig_run_berry = berry.cli.run_berry
109 |     orig_configure = berry.cli.configure
110 |     try:
111 |         args = {11: 22}
112 |         berry.cli.run_berry = MagicMock()
113 |         berry.cli.configure = lambda: args
114 | 
115 |         berry.cli.run_berry.return_value = False
116 |         assert main() == 1
117 | 
118 |         berry.cli.run_berry.assert_called_with(args)
119 | 
120 |         berry.cli.run_berry.return_value = True
121 |         assert main() == 0
122 |     finally:
123 |         berry.cli.run_berry = orig_run_berry
124 |         berry.cli.configure = orig_configure
125 | 
126 | 
127 | def test_main_missingargs(monkeypatch):
128 |     monkeypatch.setattr('sys.argv', ['berry', './'])
129 |     log_error = MagicMock()
130 |     monkeypatch.setattr('logging.warn', MagicMock())
131 |     monkeypatch.setattr('logging.error', log_error)
132 |     main()
133 |     log_error.assert_called_with(
134 |         ('Usage Error: Application ID missing, please set "application_id" in your '
135 |          'configuration YAML')
136 |     )
137 |     args = MagicMock()
138 |     args.config_file = None
139 |     args.application_id = 'myapp'
140 |     args.aws_credentials_file = None
141 |     args.mint_bucket = None
142 | 
143 |     with pytest.raises(UsageError) as excinfo:
144 |         assert run_berry(args) is False
145 |         assert ('Usage Error: Mint Bucket is not configured, please set "mint_bucket" in '
146 |                 'your configuration YAML') in str(excinfo.value)
147 | 
148 | 
149 | def test_s3_error_message(monkeypatch, tmpdir):
150 |     log_error = MagicMock()
151 |     log_info = MagicMock()
152 |     log_debug = MagicMock()
153 |     monkeypatch.setattr('logging.warn', MagicMock())
154 |     monkeypatch.setattr('logging.error', log_error)
155 |     monkeypatch.setattr('logging.info', log_info)
156 |     monkeypatch.setattr('logging.debug', log_debug)
157 | 
158 |     s3 = MagicMock()
159 |     s3.get_object.side_effect = botocore.exceptions.ClientError(
160 |         {'ResponseMetadata': {'HTTPStatusCode': 403},
161 |          'Error': {'Message': 'Access Denied'}}, 'get_object')
162 |     monkeypatch.setattr('boto3.session.Session', mock_session(s3))
163 |     monkeypatch.setattr('time.sleep', lambda x: 0)
164 | 
165 |     dns_resolver = MagicMock()
166 |     monkeypatch.setattr('dns.resolver.query', dns_resolver)
167 | 
168 |     args = MagicMock()
169 |     args.application_id = 'myapp'
170 |     args.config_file = str(tmpdir.join('taupage.yaml'))
171 |     args.once = True
172 |     args.aws_credentials_file = None
173 |     args.mint_bucket = 'my-mint-bucket'
174 |     args.local_directory = str(tmpdir.join('credentials'))
175 | 
176 |     os.makedirs(args.local_directory)
177 | 
178 |     logging.basicConfig(level=logging.INFO)
179 | 
180 |     assert run_berry(args) is False
181 |     log_error.assert_called_with(
182 |         ('Access denied while trying to read "myapp/client.json" from mint S3 bucket '
183 |          '"my-mint-bucket". Check your IAM role/user policy to allow read access! '
184 |          '(S3 error message: Access Denied)'))
185 | 
186 |     s3.get_object.side_effect = botocore.exceptions.ClientError(
187 |         {'ResponseMetadata': {'HTTPStatusCode': 404},
188 |          'Error': {}}, 'get_object')
189 |     assert run_berry(args) is False
190 |     log_error.assert_called_with(
191 |         'Credentials file "myapp/client.json" not found in mint S3 bucket "my-mint-bucket". '
192 |         'Mint either did not sync them yet or the mint configuration is wrong. (S3 error message: None)')
193 | 
194 |     s3.get_object.side_effect = botocore.exceptions.ClientError(
195 |         {'Error': {'Bucket': 'my-mint-bucket',
196 |                    'Code': 'PermanentRedirect',
197 |                    'Endpoint': 'my-mint-bucket.s3-eu-foobar-1.amazonaws.com',
198 |                    'Message': 'The bucket you are attempting to access must be '
199 |                               'addressed using the specified endpoint. Please send '
200 |                               'all future requests to this endpoint.'},
201 |          'ResponseMetadata': {'HTTPStatusCode': 301,
202 |                               'HostId': '',
203 |                               'RequestId': ''}}, 'get_object')
204 |     s3.get_bucket_location.return_value = {'LocationConstraint': 'eu-foobar-1'}
205 |     assert run_berry(args) is True
206 |     log_debug.assert_called_with(
207 |         ('Got Redirect while trying to read "myapp/client.json" from mint S3 bucket '
208 |          '"my-mint-bucket". Retrying with region eu-foobar-1, endpoint '
209 |          'my-mint-bucket.s3-eu-foobar-1.amazonaws.com! (S3 error message: The bucket '
210 |          'you are attempting to access must be addressed using the specified '
211 |          'endpoint. Please send all future requests to this endpoint.)'))
212 | 
213 |     s3.get_object.side_effect = botocore.exceptions.ClientError(
214 |         {'Error': {'Bucket': 'my-mint-bucket',
215 |                    'Code': 'PermanentRedirect',
216 |                    'Endpoint': 'my-mint-bucket.s3-eu-foobar-1.amazonaws.com',
217 |                    'Message': 'The bucket you are attempting to access must be '
218 |                               'addressed using the specified endpoint. Please send '
219 |                               'all future requests to this endpoint.'},
220 |          'ResponseMetadata': {'HTTPStatusCode': 301,
221 |                               'HostId': '',
222 |                               'RequestId': ''}}, 'get_object')
223 |     s3.get_bucket_location.side_effect = botocore.exceptions.ClientError(
224 |         {'Error': {'Code': 'UnknownError',
225 |                    'Message': 'Unknown Error, only for Test'},
226 |          'ResponseMetadata': {'HTTPStatusCode': 403,
227 |                               'HostId': '',
228 |                               'RequestId': ''}}, 'get_bucket_location')
229 |     assert run_berry(args) is True
230 |     log_debug.assert_called_with(
231 |         ('Got Redirect while trying to read "myapp/client.json" from mint S3 bucket '
232 |          '"my-mint-bucket". Retrying with region eu-foobar-1, endpoint '
233 |          'my-mint-bucket.s3-eu-foobar-1.amazonaws.com! (S3 error message: The bucket '
234 |          'you are attempting to access must be addressed using the specified '
235 |          'endpoint. Please send all future requests to this endpoint.)'))
236 | 
237 |     s3.get_object.side_effect = botocore.exceptions.ClientError(
238 |         {'Error': {'Bucket': 'my-mint-bucket',
239 |                    'Code': 'PermanentRedirect',
240 |                    'Endpoint': 'my-mint-bucket.s3.amazonaws.com',
241 |                    'Message': 'The bucket you are attempting to access must be '
242 |                               'addressed using the specified endpoint. Please send '
243 |                               'all future requests to this endpoint.'},
244 |          'ResponseMetadata': {'HTTPStatusCode': 301,
245 |                               'HostId': '',
246 |                               'RequestId': ''}}, 'get_object')
247 |     s3.get_bucket_location.side_effect = botocore.exceptions.ClientError(
248 |         {'Error': {'Code': 'UnknownError',
249 |                    'Message': 'Unknown Error, only for Test'},
250 |          'ResponseMetadata': {'HTTPStatusCode': 403,
251 |                               'HostId': '',
252 |                               'RequestId': ''}}, 'get_bucket_location')
253 |     message_text = """id 1234
254 | opcode QUERY
255 | rcode NOERROR
256 | flags QR AA RD
257 | ;QUESTION
258 | my-mint-bucket.s3.amazonaws.com. IN CNAME
259 | ;ANSWER
260 | my-mint-bucket.s3.amazonaws.com. 1 IN CNAME s3.eu-foobar-1.amazonaws.com.
261 | ;AUTHORITY
262 | ;ADDITIONAL
263 | """
264 |     dns_resolver.return_value = (dns.resolver.Answer(
265 |         dns.name.from_text('my-mint-bucket.s3.amazonaws.com.'),
266 |         dns.rdatatype.CNAME,
267 |         dns.rdataclass.IN,
268 |         dns.message.from_text(message_text)))
269 |     assert run_berry(args) is True
270 |     log_debug.assert_called_with(
271 |         ('Got Redirect while trying to read "myapp/client.json" from mint S3 bucket '
272 |          '"my-mint-bucket". Retrying with region eu-foobar-1, endpoint '
273 |          'my-mint-bucket.s3.amazonaws.com! (S3 error message: The bucket you are '
274 |          'attempting to access must be addressed using the specified endpoint. '
275 |          'Please send all future requests to this endpoint.)'))
276 | 
277 |     message_text = """id 1234
278 | opcode QUERY
279 | rcode NOERROR
280 | flags QR AA RD
281 | ;QUESTION
282 | my-mint-bucket.s3.amazonaws.com. IN CNAME
283 | ;ANSWER
284 | my-mint-bucket.s3.amazonaws.com. 1 IN CNAME s3-eu-foobar-1.amazonaws.com.
285 | ;AUTHORITY
286 | ;ADDITIONAL
287 | """
288 |     dns_resolver.return_value = (dns.resolver.Answer(
289 |         dns.name.from_text('my-mint-bucket.s3.amazonaws.com.'),
290 |         dns.rdatatype.CNAME,
291 |         dns.rdataclass.IN,
292 |         dns.message.from_text(message_text)))
293 |     assert run_berry(args) is True
294 |     log_debug.assert_called_with(
295 |         ('Got Redirect while trying to read "myapp/client.json" from mint S3 bucket '
296 |          '"my-mint-bucket". Retrying with region eu-foobar-1, endpoint '
297 |          'my-mint-bucket.s3.amazonaws.com! (S3 error message: The bucket you are '
298 |          'attempting to access must be addressed using the specified endpoint. '
299 |          'Please send all future requests to this endpoint.)'))
300 | 
301 |     message_text = """id 1234
302 | opcode QUERY
303 | rcode NOERROR
304 | flags QR AA RD
305 | ;QUESTION
306 | my-mint-bucket.s3.amazonaws.com. IN CNAME
307 | ;ANSWER
308 | my-mint-bucket.s3.amazonaws.com. 1 IN CNAME s3.amazonaws.com.
309 | ;AUTHORITY
310 | ;ADDITIONAL
311 | """
312 |     dns_resolver.return_value = (dns.resolver.Answer(
313 |         dns.name.from_text('my-mint-bucket.s3.amazonaws.com.'),
314 |         dns.rdatatype.CNAME,
315 |         dns.rdataclass.IN,
316 |         dns.message.from_text(message_text)))
317 |     assert run_berry(args) is True
318 |     log_debug.assert_called_with(
319 |         ('Got Redirect while trying to read "myapp/client.json" from mint S3 bucket '
320 |          '"my-mint-bucket". Retrying with region None, endpoint '
321 |          'my-mint-bucket.s3.amazonaws.com! (S3 error message: The bucket you are '
322 |          'attempting to access must be addressed using the specified endpoint. '
323 |          'Please send all future requests to this endpoint.)'))
324 | 
325 |     dns_resolver.side_effect = dns.resolver.NXDOMAIN
326 |     assert run_berry(args) is True
327 |     log_debug.assert_called_with(
328 |         ('Got Redirect while trying to read "myapp/client.json" from mint S3 bucket '
329 |          '"my-mint-bucket". Retrying with region None, endpoint '
330 |          'my-mint-bucket.s3.amazonaws.com! (S3 error message: The bucket you are '
331 |          'attempting to access must be addressed using the specified endpoint. '
332 |          'Please send all future requests to this endpoint.)'))
333 | 
334 |     s3.get_object.side_effect = botocore.exceptions.ClientError(
335 |         {'Error': {'Code': 'InvalidRequest',
336 |                    'Message': 'The authorization mechanism you have provided is not '
337 |                               'supported. Please use AWS4-HMAC-SHA256.'},
338 |          'ResponseMetadata': {'HTTPStatusCode': 400,
339 |                               'HostId': '',
340 |                               'RequestId': ''}}, 'get_object')
341 |     assert run_berry(args) is True
342 |     log_debug.assert_called_with(
343 |         ('Invalid Request while trying to read "myapp/client.json" from mint S3 '
344 |          'bucket "my-mint-bucket". Retrying with signature version v4! (S3 error '
345 |          'message: The authorization mechanism you have provided is not supported. '
346 |          'Please use AWS4-HMAC-SHA256.)'))
347 | 
348 |     # generic ClientError
349 |     s3.get_object.side_effect = botocore.exceptions.ClientError(
350 |         {'ResponseMetadata': {'HTTPStatusCode': 999}, 'Error': {}}, 'get_object')
351 |     assert run_berry(args) is False
352 |     log_error.assert_called_with(
353 |         ('Could not read from mint S3 bucket "my-mint-bucket": An error occurred '
354 |          '(Unknown) when calling the get_object operation: Unknown'))
355 | 
356 |     # generic Exception
357 |     s3.get_object.side_effect = Exception('foobar')
358 |     assert run_berry(args) is False
359 |     log_error.assert_called_with('Failed to download client credentials', exc_info=True)
360 | 


--------------------------------------------------------------------------------