├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── bad.py ├── py_look_for_timeouts ├── __init__.py └── main.py ├── requirements-tests.txt ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── samples ├── bad_hardcoded.py ├── bad_httpconnection.py ├── bad_requests.py ├── bad_script.py ├── bad_script_2.py ├── bad_twilio_connection.py └── good.py └── test_basic.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | dist/ 3 | env/ 4 | sdist/ 5 | build/ 6 | *.egg-info/ 7 | *.egg_info/ 8 | 9 | .DS_Store 10 | .coverage 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | install: 5 | - "pip install -r requirements-tests.txt" 6 | script: 7 | - "flake8 setup.py py_look_for_timeouts tests" 8 | - "nosetests -s -v --with-cover --cover-package=py_look_for_timeouts tests" 9 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | py-find-injection 2 | Copyright (c) 2013 Uber Technologies, Inc. 3 | The MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to 7 | deal in the Software without restriction, including without limitation the 8 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | sell copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | IN THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | include *.md 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/uber/py-look-for-timeouts.png)](https://travis-ci.org/uber/py-look-for-timeouts) 2 | 3 | `py_look_for_timeouts` looks for network calls without timeouts on them, using various heuristics. It knows and cares about urllib, httplib, twilio, and requests calls. 4 | 5 | For a similar program which looks for SQL injection vulnerabilities, look at [py-find-injection](https://github.com/uber/py-find-injection) 6 | -------------------------------------------------------------------------------- /bad.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | 3 | urllib2.urlopen('foo') 4 | -------------------------------------------------------------------------------- /py_look_for_timeouts/__init__.py: -------------------------------------------------------------------------------- 1 | version_info = (0, 4) 2 | __version__ = '.'.join(map(str, version_info)) 3 | -------------------------------------------------------------------------------- /py_look_for_timeouts/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import ast 5 | 6 | from . import __version__ 7 | 8 | 9 | class IllegalLine(object): 10 | def __init__(self, reason, node, filename): 11 | self.reason = reason 12 | self.lineno = node.lineno 13 | self.filename = filename 14 | self.node = node 15 | 16 | def __str__(self): 17 | return "%s:%d\t%s" % (self.filename, self.lineno, self.reason) 18 | 19 | def __repr__(self): 20 | return "IllegalLine<%s, %s:%s>" % ( 21 | self.reason, 22 | self.filename, 23 | self.lineno 24 | ) 25 | 26 | 27 | def _intify(something): 28 | if isinstance(something, ast.Num): 29 | return something.n 30 | else: 31 | # we aren't going to evaluate anything else, so, uh 32 | # assume it was okay 33 | return None 34 | 35 | 36 | def _stringify(node): 37 | if isinstance(node, ast.Name): 38 | return node.id 39 | elif isinstance(node, ast.Attribute): 40 | return '%s.%s' % (_stringify(node.value), node.attr) 41 | elif isinstance(node, ast.Subscript): 42 | return '%s[%s]' % (_stringify(node.value), _stringify(node.slice)) 43 | elif isinstance(node, ast.Index): 44 | return _stringify(node.value) 45 | elif isinstance(node, ast.Call): 46 | return '%s(%s, %s)' % ( 47 | _stringify(node.func), 48 | _stringify(node.args), 49 | _stringify(node.keywords) 50 | ) 51 | elif isinstance(node, list): 52 | return '[%s]' % (', '.join(_stringify(n) for n in node)) 53 | elif isinstance(node, ast.Str): 54 | return node.s 55 | else: 56 | return ast.dump(node) 57 | 58 | 59 | class Visitor(ast.NodeVisitor): 60 | def __init__(self, filename, checker, *args, **kwargs): 61 | self.filename = filename 62 | self.checker = checker 63 | self.errors = [] 64 | super(Visitor, self).__init__(*args, **kwargs) 65 | 66 | @staticmethod 67 | def _is_urlopen_call(function_name): 68 | if '.' in function_name: 69 | if function_name in ('urllib.urlopen', 'urllib2.urlopen'): 70 | return True 71 | else: 72 | if function_name == 'urlopen': 73 | return True 74 | return False 75 | 76 | @staticmethod 77 | def _is_httplib_call(function_name): 78 | if '.' in function_name: 79 | if function_name in ( 80 | 'httplib.HTTPConnection', 81 | 'httplib.HTTPSConnection' 82 | ): 83 | return True 84 | else: 85 | if function_name in ('HTTPConnection', 'HTTPSConnection'): 86 | return True 87 | return False 88 | 89 | @staticmethod 90 | def _is_twilio_call(function_name): 91 | if '.' in function_name: 92 | if function_name.endswith('rest.TwilioRestClient'): 93 | return True 94 | elif function_name == 'TwilioRestClient': 95 | return True 96 | return False 97 | 98 | @staticmethod 99 | def _is_requests_call(function_name): 100 | if function_name in ( 101 | 'requests.get', 102 | 'requests.post', 103 | 'requests.put', 104 | 'requests.head', 105 | 'requests.request', 106 | ): 107 | return True 108 | return False 109 | 110 | def _check_timeout_call(self, node, arg_offset, kwarg_name, desc): 111 | # Grab the timeout node inside the function call 112 | timeout = None 113 | is_kwarg = False 114 | if arg_offset is not None and len(node.args) > arg_offset: 115 | timeout = node.args[arg_offset] 116 | elif node.keywords: 117 | keywords = [k for k in node.keywords if k.arg == kwarg_name] 118 | if keywords: 119 | is_kwarg = True 120 | timeout = keywords[0].value 121 | errors = self.checker(timeout, desc, node, self.filename, is_kwarg) 122 | if errors: 123 | self.errors.extend(errors) 124 | 125 | def visit_Call(self, node): 126 | function_name = _stringify(node.func) 127 | if self._is_urlopen_call(function_name): 128 | self._check_timeout_call( 129 | node, 130 | arg_offset=2, 131 | kwarg_name='timeout', 132 | desc='urlopen call' 133 | ) 134 | elif self._is_httplib_call(function_name): 135 | self._check_timeout_call( 136 | node, 137 | arg_offset=5, 138 | kwarg_name='timeout', 139 | desc='httplib connection' 140 | ) 141 | elif self._is_twilio_call(function_name): 142 | self._check_timeout_call( 143 | node, 144 | arg_offset=5, 145 | kwarg_name='timeout', 146 | desc='twilio rest connection' 147 | ) 148 | elif self._is_requests_call(function_name): 149 | self._check_timeout_call( 150 | node, 151 | arg_offset=None, 152 | kwarg_name='timeout', 153 | desc='requests call' 154 | ) 155 | 156 | 157 | class Checker(object): 158 | 159 | def __init__(self, allow_hardcoded=True): 160 | self.allow_hardcoded = allow_hardcoded 161 | 162 | def __call__(self, timeout_node, desc, node, filename, is_kwarg): 163 | """Return a list of IllegalLine on misconfigured timeout. 164 | 165 | :param timeout_node: 166 | :param desc: 167 | :param node: 168 | :param str filename: 169 | """ 170 | msg = None 171 | if not timeout_node: 172 | msg = '%s without a timeout arg or kwarg' % desc 173 | return [IllegalLine(msg, node, filename)] 174 | 175 | value = _intify(timeout_node) 176 | if value == 0: 177 | msg = '%s with a timeout %sarg of 0' % ( 178 | desc, 'kw' if is_kwarg else '') 179 | elif isinstance(value, int) and not self.allow_hardcoded: 180 | msg = '%s with an hardcoded timeout arg of %d' % (desc, value) 181 | 182 | if msg: 183 | return [IllegalLine(msg, node, filename)] 184 | 185 | 186 | def check(filename, checker=None): 187 | """Check a file for missing/misconfigure timeouts.""" 188 | if not checker: 189 | checker = Checker() 190 | 191 | v = Visitor(filename, checker=checker) 192 | with open(filename, 'r') as fobj: 193 | try: 194 | parsed = ast.parse(fobj.read(), filename) 195 | v.visit(parsed) 196 | except Exception: # noqa 197 | raise # noqa 198 | return v.errors 199 | 200 | 201 | def main(): 202 | parser = argparse.ArgumentParser( 203 | description='Look for python source files missing timeouts', 204 | epilog=('Exit status is 0 if all files are okay, 1 if any files ' 205 | 'have an error. Errors are printed to stdout') 206 | ) 207 | parser.add_argument( 208 | '--version', 209 | action='version', 210 | version='%(prog)s ' + __version__ 211 | ) 212 | parser.add_argument( 213 | '--no-hardcoded', 214 | action='store_true', 215 | help="Do not allow hardcoded constant" 216 | ) 217 | parser.add_argument('files', nargs='+', help='Files to check') 218 | args = parser.parse_args() 219 | 220 | errors = [] 221 | checker = Checker(allow_hardcoded=not args.no_hardcoded) 222 | for fname in args.files: 223 | these_errors = check(fname, checker=checker) 224 | if these_errors: 225 | print '\n'.join(str(e) for e in these_errors) 226 | errors.extend(these_errors) 227 | if errors: 228 | print '%d total errors' % len(errors) 229 | return 1 230 | else: 231 | return 0 232 | 233 | 234 | if __name__ == '__main__': 235 | main() 236 | -------------------------------------------------------------------------------- /requirements-tests.txt: -------------------------------------------------------------------------------- 1 | mock==1.0.1 2 | nose==1.3.3 3 | nose-cov==1.6 4 | flake8==2.0.0 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=120 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | from distutils.version import StrictVersion 4 | from setuptools import setup, find_packages 5 | from pip.req import parse_requirements 6 | import pip 7 | 8 | with open('README.md', 'r') as readme_fd: 9 | LONG_DESCRIPTION = readme_fd.read() 10 | 11 | 12 | def get_install_requirements(fname): 13 | 14 | ReqOpts = collections.namedtuple('ReqOpts', [ 15 | 'skip_requirements_regex', 16 | 'default_vcs', 17 | 'isolated_mode', 18 | ]) 19 | 20 | opts = ReqOpts(None, 'git', False) 21 | params = {'options': opts} 22 | 23 | requires = [] 24 | 25 | pip_version = StrictVersion(pip.__version__) 26 | session_support_since = StrictVersion('1.5.0') 27 | if pip_version >= session_support_since: 28 | from pip.download import PipSession 29 | session = PipSession() 30 | params.update({'session': session}) 31 | 32 | for ir in parse_requirements(fname, **params): 33 | if ir is not None and ir.req is not None: 34 | requires.append(str(ir.req)) 35 | return requires 36 | 37 | 38 | tests_require = get_install_requirements('requirements-tests.txt') 39 | 40 | setup( 41 | name="py-look-for-timeouts", 42 | version="0.4", 43 | author="James Brown", 44 | author_email="jbrown@uber.com", 45 | url="https://github.com/uber/py-look-for-timeouts", 46 | description="ple python ast consumer which searches for missing timeouts", 47 | license='MIT (Expat)', 48 | classifiers=[ 49 | "Programming Language :: Python", 50 | "Operating System :: OS Independent", 51 | "Topic :: Security", 52 | "Topic :: Security", 53 | "Intended Audience :: Developers", 54 | "Development Status :: 4 - Alpha", 55 | "Programming Language :: Python :: 2.7", 56 | "License :: OSI Approved :: MIT License", 57 | ], 58 | packages=find_packages(exclude=["tests"]), 59 | entry_points={ 60 | "console_scripts": [ 61 | "py-look-for-timeouts = py_look_for_timeouts.main:main", 62 | ] 63 | }, 64 | tests_require=tests_require, 65 | test_suite="nose.collector", 66 | long_description=LONG_DESCRIPTION 67 | ) 68 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/py-look-for-timeouts/e7f47a81b57a14dd38bb45881e7a1db7cb6e3b5b/tests/__init__.py -------------------------------------------------------------------------------- /tests/samples/bad_hardcoded.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | 3 | urllib2.urlopen('foo', timeout=2) 4 | -------------------------------------------------------------------------------- /tests/samples/bad_httpconnection.py: -------------------------------------------------------------------------------- 1 | import httplib 2 | from httplib import HTTPConnection, HTTPSConnection 3 | 4 | c = httplib.HTTPConnection('foo') 5 | c = httplib.HTTPSConnection('foo') 6 | c = httplib.HTTPConnection('foo', timeout=0) 7 | c = HTTPConnection('foo', timeout=0) 8 | c = HTTPSConnection('foo', timeout=0) 9 | c = httplib.HTTPConnection('foo', 80, 'bar', 'baz', False, 0) 10 | -------------------------------------------------------------------------------- /tests/samples/bad_requests.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | requests.get('foo') 4 | requests.put('bar', timeout=0) 5 | requests.post('baz') 6 | requests.head('bing') 7 | requests.request('bing', method='GET', timeout=0) 8 | -------------------------------------------------------------------------------- /tests/samples/bad_script.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | 3 | urllib2.urlopen('foo') 4 | urllib2.urlopen('foo', timeout=0) 5 | 6 | with urllib2.urlopen('baz', None, 0) as f: 7 | pass 8 | -------------------------------------------------------------------------------- /tests/samples/bad_script_2.py: -------------------------------------------------------------------------------- 1 | from urllib2 import urlopen 2 | 3 | urlopen('foo') 4 | urlopen('foo', timeout=0) 5 | 6 | with urlopen('baz', None, 0) as f: 7 | pass 8 | -------------------------------------------------------------------------------- /tests/samples/bad_twilio_connection.py: -------------------------------------------------------------------------------- 1 | import twilio.rest 2 | from twilio.rest import TwilioRestClient 3 | 4 | c = twilio.rest.TwilioRestClient('account', 'token', 'base', 'version', 'client') 5 | c = twilio.rest.TwilioRestClient('account') 6 | c = twilio.rest.TwilioRestClient('account', timeout=0) 7 | c = TwilioRestClient('account') 8 | c = TwilioRestClient('account') 9 | c = twilio.rest.TwilioRestClient('account', 'token', 'base', 'version', 'client', 0) 10 | -------------------------------------------------------------------------------- /tests/samples/good.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | import httplib 3 | import requests 4 | import foobar 5 | 6 | TIMEOUT = 0 7 | 8 | urllib2.urlopen('foo', timeout=2) 9 | urllib2.urlopen('foo', timeout=TIMEOUT) 10 | foobar.urlopen('baz') 11 | 12 | urllib2.urlopen('foo', None, 2) 13 | urllib2.urlopen('foo', None, TIMEOUT) 14 | 15 | c = httplib.HTTPConnection('foo', timeout=2) 16 | c = httplib.HTTPConnection('foo', 80, 'bar', 'baz', False, 2) 17 | 18 | requests.get('foo', timeout=2) 19 | requests.put('foo', timeout=TIMEOUT) 20 | 21 | # junk to make sure we parse stuff correctly 22 | print [1, 2, 3][1] 23 | print {"foo", urllib2.urlopen}["foo"]() 24 | -------------------------------------------------------------------------------- /tests/test_basic.py: -------------------------------------------------------------------------------- 1 | from os.path import join, realpath, dirname 2 | from unittest import TestCase 3 | 4 | from py_look_for_timeouts.main import check, Checker 5 | 6 | 7 | SAMPLE_PATH = realpath(join(dirname(__file__), 'samples')) 8 | 9 | 10 | class TestSimple(TestCase): 11 | def test_good_file(self): 12 | errors = check(join(SAMPLE_PATH, 'good.py')) 13 | self.assertEqual(errors, []) 14 | 15 | def test_bad_urllib_urlopen(self): 16 | errors = check(join(SAMPLE_PATH, 'bad_script.py')) 17 | self.assertEqual(len(errors), 3) 18 | self.assertEqual(errors[0].lineno, 3) 19 | self.assertEqual(errors[0].reason, 'urlopen call without a timeout arg or kwarg') 20 | self.assertEqual(errors[1].lineno, 4) 21 | self.assertEqual(errors[1].reason, 'urlopen call with a timeout kwarg of 0') 22 | self.assertEqual(errors[2].lineno, 6) 23 | self.assertEqual(errors[2].reason, 'urlopen call with a timeout arg of 0') 24 | 25 | def test_bad_bare_urlopen(self): 26 | errors = check(join(SAMPLE_PATH, 'bad_script_2.py')) 27 | self.assertEqual(len(errors), 3) 28 | self.assertEqual(errors[0].lineno, 3) 29 | self.assertEqual(errors[0].reason, 'urlopen call without a timeout arg or kwarg') 30 | self.assertEqual(errors[1].lineno, 4) 31 | self.assertEqual(errors[1].reason, 'urlopen call with a timeout kwarg of 0') 32 | self.assertEqual(errors[2].lineno, 6) 33 | self.assertEqual(errors[2].reason, 'urlopen call with a timeout arg of 0') 34 | 35 | def test_bad_httpconnection(self): 36 | errors = check(join(SAMPLE_PATH, 'bad_httpconnection.py')) 37 | self.assertEqual(len(errors), 6) 38 | self.assertEqual(errors[0].reason, 'httplib connection without a timeout arg or kwarg') 39 | self.assertEqual(errors[2].reason, 'httplib connection with a timeout kwarg of 0') 40 | self.assertEqual(errors[5].reason, 'httplib connection with a timeout arg of 0') 41 | 42 | def test_bad_twilio_connection(self): 43 | errors = check(join(SAMPLE_PATH, 'bad_twilio_connection.py')) 44 | self.assertEqual(len(errors), 6) 45 | self.assertEqual(errors[0].reason, 'twilio rest connection without a timeout arg or kwarg') 46 | self.assertEqual(errors[2].reason, 'twilio rest connection with a timeout kwarg of 0') 47 | self.assertEqual(errors[5].reason, 'twilio rest connection with a timeout arg of 0') 48 | 49 | def test_bad_requests_call(self): 50 | errors = check(join(SAMPLE_PATH, 'bad_requests.py')) 51 | self.assertEqual(len(errors), 5) 52 | 53 | def test_hardcoded_timeout(self): 54 | """Verify that hardcoded timeouts are identified.""" 55 | checker = Checker(allow_hardcoded=False) 56 | errors = check(join(SAMPLE_PATH, 'bad_hardcoded.py'), checker=checker) 57 | self.assertEqual(len(errors), 1) 58 | error = errors[0] 59 | self.assertEqual(error.lineno, 3) 60 | self.assertEqual(error.reason, 'urlopen call with an hardcoded timeout arg of 2') 61 | --------------------------------------------------------------------------------