├── tests ├── __init__.py ├── test_cli.py ├── test_plugin_load.py ├── test_expand.py └── test_hostlists.py ├── docs └── source │ ├── global.rst │ ├── index.rst │ ├── topics │ └── hostlists.rst │ └── conf.py ├── hostlists ├── plugins │ ├── __init__.py │ ├── dnsip.py │ ├── get_haproxy_phys │ ├── plugintype.py │ └── haproxy.py ├── examples │ ├── foodb~ │ ├── foodb │ ├── testlist~ │ ├── foo │ └── foo~ ├── plugin_base.py ├── exceptions.py ├── __init__.py ├── decorators.py ├── cli.py ├── plugin_manager.py └── range.py ├── hostlists_plugins_default ├── __init__.py ├── hostlists_plugin_file.py ├── hostlists_plugin_dns.py └── range.py ├── changelog.d ├── HEADER.md ├── 1.removal.md └── README.md ├── LICENSE.txt ├── .gitignore ├── setup.py ├── tox.ini ├── screwdriver.yaml ├── README.md ├── setup.cfg └── .pylintrc /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/source/global.rst: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /hostlists/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /hostlists_plugins_default/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /changelog.d/HEADER.md: -------------------------------------------------------------------------------- 1 | # Hostlists changes 2 | -------------------------------------------------------------------------------- /hostlists/examples/foodb~: -------------------------------------------------------------------------------- 1 | file2testhost1 2 | -------------------------------------------------------------------------------- /changelog.d/1.removal.md: -------------------------------------------------------------------------------- 1 | Removed Python 2.x specific code from the haproxy plugin. -------------------------------------------------------------------------------- /hostlists/examples/foodb: -------------------------------------------------------------------------------- 1 | # Foo domain database backends 2 | range:db[01-04].foo.com 3 | 4 | -------------------------------------------------------------------------------- /hostlists/plugin_base.py: -------------------------------------------------------------------------------- 1 | """hostlists plugins class""" 2 | 3 | 4 | class HostlistsPlugin(object): 5 | 6 | names = [] 7 | 8 | def expand(self, value, name=None): 9 | return value 10 | -------------------------------------------------------------------------------- /hostlists/exceptions.py: -------------------------------------------------------------------------------- 1 | """Hostlists exceptions""" 2 | 3 | 4 | class HostListsError(Exception): # pragma: no cover 5 | pass 6 | 7 | 8 | class MethodTimeoutError(Exception): # pragma: no cover 9 | """ 10 | Timeout exception class 11 | """ 12 | pass 13 | -------------------------------------------------------------------------------- /hostlists/examples/testlist~: -------------------------------------------------------------------------------- 1 | # This is an example hostlist file for the file: plugin 2 | # Each line not beginning with a # (hash mark) may be 3 | # a host or another plugin reference. 4 | web1.foo.com 5 | 6 | # A plugin can reference another plugin, such as a range of hosts 7 | range:db[1-7].foo.com 8 | 9 | # Host IP addresses from dns round robin 10 | dns:cnn.com 11 | 12 | # or even another instance of thw same plugin itself 13 | file:sshmap/examples/hostlists/testlist2 14 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. hostlists documentation master file, created by 2 | sphinx-quickstart on Thu Mar 5 15:56:28 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. include global.rst 7 | 8 | Welcome to hostlists's documentation! 9 | ===================================== 10 | 11 | Contents: 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | 16 | topics/hostlists.rst 17 | 18 | Indices and tables 19 | ================== 20 | 21 | * :ref:`genindex` 22 | * :ref:`modindex` 23 | * :ref:`search` 24 | 25 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010-2013 Yahoo! Inc. All rights reserved. 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. See accompanying LICENSE file. 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | /.tox 27 | nosetests.xml 28 | 29 | # Translations 30 | *.mo 31 | 32 | # Mr Developer 33 | .mr.developer.cfg 34 | .project 35 | .pydevproject 36 | 37 | # Backup files 38 | *~ 39 | 40 | /.idea 41 | 42 | # Package metadata 43 | hostlists/package_metadata.json 44 | 45 | # unittest output files 46 | /nosetests.xml 47 | /pytest_py*.xml 48 | 49 | # Coverage report 50 | /cobertura.xml 51 | /coverage.xml 52 | -------------------------------------------------------------------------------- /changelog.d/README.md: -------------------------------------------------------------------------------- 1 | **Changelog Messages** 2 | 3 | This directory contains changelog messages. 4 | 5 | # Adding a new changelog message 6 | 7 | Create a file in this directory named in the following format: 8 | 9 | {issuenum}.{changetype}.md 10 | 11 | issuenum - Is the issue number for the change. 12 | 13 | changetype - Is the type of change, it can be one of the following: 14 | 15 | - feature - A new feature 16 | - bugfix - The change fixes a bug 17 | - doc - The change is an improvement to the documentation 18 | - removal - The changed involved removing code or features 19 | - misc - Other kinds of changes 20 | 21 | The changes are automatically added to the changelog of the release that contains 22 | the new change file. -------------------------------------------------------------------------------- /hostlists/examples/foo: -------------------------------------------------------------------------------- 1 | # This is an example hostlist file for the file: plugin 2 | # Each line not beginning with a # (hash mark) may be 3 | # a host or another plugin reference. 4 | www.foo.com 5 | 6 | # A plugin can reference another plugin, such as a range of hosts 7 | range:frontend[01-11].foo.com 8 | 9 | # Get a list of hosts or ip addresses from dns 10 | dns:foo.com 11 | 12 | # HAProxy backend nodes for frontend web 13 | haproxy:myhaproxy.foo.com:web 14 | 15 | # HAProxy backend nodes in up state for frontend web 16 | haproxy_up:myhaproxy.foo.com:web 17 | 18 | # HAProxy backend nodes in down state for frontend web 19 | haproxy_down:myhaproxy.foo.com:web 20 | 21 | # or even another instance of thw same plugin itself 22 | file:hostlists/examples/foodb 23 | 24 | -------------------------------------------------------------------------------- /hostlists/examples/foo~: -------------------------------------------------------------------------------- 1 | # This is an example hostlist file for the file: plugin 2 | # Each line not beginning with a # (hash mark) may be 3 | # a host or another plugin reference. 4 | www.foo.com 5 | 6 | # A plugin can reference another plugin, such as a range of hosts 7 | range:frontend[01-11].foo.com 8 | 9 | # Host IP addresses from dns round robin 10 | dns:foo.com 11 | 12 | # HAProxy backend nodes for frontend web 13 | haproxy:myhaproxy.foo.com:web 14 | 15 | # HAProxy backend nodes in up state for frontend web 16 | haproxy_up:myhaproxy.foo.com:web 17 | 18 | # HAProxy backend nodes in down state for frontend web 19 | haproxy_down:myhaproxy.foo.com:web 20 | 21 | # or even another instance of thw same plugin itself 22 | file:hostlists/examples/foodb - db03.foo.com 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Setup configuration for hostlists 4 | """ 5 | 6 | # Copyright (c) 2010-2015 Yahoo! Inc. All rights reserved. 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. See accompanying LICENSE file. 18 | from setuptools import setup 19 | 20 | 21 | setup() 22 | -------------------------------------------------------------------------------- /docs/source/topics/hostlists.rst: -------------------------------------------------------------------------------- 1 | .. include global.rst 2 | 3 | Documentation for the hostlists module 4 | ************************************** 5 | 6 | .. automodule:: hostlists.hostlists 7 | :members: 8 | 9 | Hostlists Plugin Modules 10 | ======================== 11 | 12 | Hostlists supports expansion using plugins. Programs by default are run 13 | through each other recursively. 14 | 15 | Host Range Plugin 16 | ----------------- 17 | 18 | .. automodule:: hostlists_plugins_default.range 19 | 20 | File Plugin 21 | ----------- 22 | 23 | .. automodule:: hostlists.plugins.file 24 | 25 | DNS Plugin 26 | ---------- 27 | 28 | .. automodule:: hostlists.plugins.dns 29 | 30 | DNS IP Plugin 31 | ------------- 32 | 33 | .. automodule:: hostlists.plugins.dnsip 34 | 35 | HAProxy Plugin 36 | -------------- 37 | 38 | .. automodule:: hostlists.plugins.haproxy 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /hostlists/plugins/dnsip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import 3 | 4 | """ hostlists plugin to get hosts from dns """ 5 | 6 | # Copyright (c) 2012-2015 Yahoo! Inc. All rights reserved. 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. See accompanying LICENSE file. 18 | 19 | 20 | import dns.resolver 21 | 22 | 23 | def name(): 24 | return 'dnsip' 25 | 26 | 27 | def expand(value, name=None): 28 | tmplist = [] 29 | try: 30 | answers = list(dns.resolver.query(value)) 31 | except dns.resolver.NoAnswer: 32 | answers = [] 33 | for rdata in answers: 34 | tmplist.append(rdata.address) 35 | return tmplist 36 | -------------------------------------------------------------------------------- /hostlists/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2015 Yahoo! Inc. All rights reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. See accompanying LICENSE file. 13 | 14 | __all__ = ['cli', 'decorators', 'exceptions', 'plugin_base', 'plugin_manager', 'range'] 15 | __version__ = '0.0.0dev0' 16 | 17 | 18 | import json 19 | import os 20 | from .plugin_manager import get_plugins, installed_plugins, multiple_names, run_plugin_expand 21 | from .range import compress, expand 22 | from .range import range_split 23 | from .exceptions import HostListsError 24 | 25 | 26 | # Config file 27 | CONF_FILE = os.path.expanduser('~/.hostlists.conf') 28 | 29 | 30 | def get_setting(key): 31 | """ 32 | Get setting values from CONF_FILE 33 | :param key: 34 | :return: 35 | """ 36 | try: 37 | with open(CONF_FILE) as cf: 38 | settings = json.load(cf) 39 | except IOError: 40 | return None 41 | if key in settings.keys(): 42 | return settings[key] 43 | return None # pragma: no cover 44 | 45 | 46 | try: 47 | import pkg_resources 48 | __version__ = pkg_resources.get_distribution("hostlists").version 49 | except (ImportError, pkg_resources.DistributionNotFound): # pragma: no cover 50 | pass 51 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [config] 2 | package_dir = hostlists 3 | package_name = hostlists 4 | 5 | [tox] 6 | skip_missing_interpreters=True 7 | envlist = py39,py310,py311 8 | 9 | [testenv] 10 | changedir = {toxinidir} 11 | commands = 12 | pytest -x --junitxml=pytest_{envname}.xml -o junit_suite_name={envname} --cov=hostlists --cov-report=xml:coverage.xml --cov-report term-missing tests/ 13 | 14 | deps = 15 | pytest 16 | pytest-cov 17 | passenv = 18 | SSH_AUTH_SOCK 19 | BUILD_NUMBER 20 | extras = 21 | test 22 | 23 | [testenv:build_docs] 24 | deps= 25 | sphinx 26 | sphinx-pypi-upload 27 | sphinx_rtd_theme 28 | 29 | commands= 30 | python setup.py build_sphinx 31 | 32 | [pycodestyle] 33 | filename= *.py 34 | show-source = False 35 | 36 | # H104 File contains nothing but comments 37 | # H405 multi line docstring summary not separated with an empty line 38 | # H803 Commit message should not end with a period (do not remove per list discussion) 39 | # H904 Wrap long lines in parentheses instead of a backslash 40 | ignore = H104,H405,H803,H904 41 | 42 | builtins = _ 43 | exclude=.venv,.git,.tox,build,dist,docs,*lib/python*,*egg,tools,vendor,.update-venv,*.ini,*.po,*.pot 44 | max-line-length = 160 45 | 46 | [testenv:pycodestyle] 47 | deps= 48 | pycodestyle 49 | commands = 50 | pycodestyle hostlists 51 | 52 | [testenv:lint_pylint] 53 | deps = 54 | isort<=4.2.15 55 | six 56 | pylint 57 | commands = {envpython} {envbindir}/pylint --output-format=parseable {[config]package_dir} 58 | passenv = SSH_AUTH_SOCK BUILD_NUMBER 59 | extras = 60 | pylint 61 | 62 | [testenv:lint_mypy] 63 | deps = 64 | mypy 65 | lxml 66 | commands = 67 | mypy --ignore-missing-imports --txt-report artifacts/mypy src/screwdrivercd 68 | passenv = 69 | SSH_AUTH_SOCK 70 | BUILD_NUMBER 71 | extras = 72 | mypy 73 | -------------------------------------------------------------------------------- /screwdriver.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019, Oath Inc. 2 | # Licensed under the terms of the Apache 2.0 license. See the LICENSE file in the project root for terms 3 | 4 | version: 4 5 | shared: 6 | environment: 7 | CHANGELOG_FILENAME: docs/changelog.md 8 | 9 | jobs: 10 | validate_codestyle: 11 | template: python/validate_codestyle 12 | requires: [~commit, ~pr] 13 | 14 | validate_lint: 15 | template: python/validate_lint 16 | requires: [~commit, ~pr] 17 | 18 | validate_security: 19 | template: python/validate_security 20 | requires: [~commit, ~pr] 21 | 22 | validate_test: 23 | template: python/validate_unittest 24 | environment: 25 | TOX_ENVLIST: py39,py310,py311 26 | requires: [~commit, ~pr] 27 | 28 | validate_documentation: 29 | template: python/documentation 30 | environment: 31 | DOCUMENTATION_PUBLISH: False 32 | requires: [~pr] 33 | 34 | generate_version: 35 | template: python/generate_version 36 | requires: [validate_test, validate_lint, validate_codestyle, validate_security] 37 | 38 | publish_test_pypi: 39 | template: python/package_python 40 | environment: 41 | PUBLISH: True 42 | TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ 43 | requires: [generate_version] 44 | 45 | verify_test_package: 46 | template: python/validate_pypi_package 47 | environment: 48 | PYPI_INDEX_URL: https://test.pypi.org/simple 49 | requires: [publish_test_pypi] 50 | 51 | publish_pypi: 52 | template: python/package_python 53 | environment: 54 | PUBLISH: True 55 | requires: [verify_test_package] 56 | 57 | publish_documentation: 58 | template: python/documentation 59 | requires: [publish_pypi] 60 | -------------------------------------------------------------------------------- /hostlists_plugins_default/hostlists_plugin_file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ hostlists plugin to get hosts from a filei """ 3 | 4 | # Copyright (c) 2010-2013 Yahoo! Inc. All rights reserved. 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. See accompanying LICENSE file. 16 | import os 17 | from hostlists.plugin_base import HostlistsPlugin 18 | 19 | 20 | class HostlistsPluginFile(HostlistsPlugin): 21 | names = ['file'] 22 | 23 | def _find_file(self, filename): 24 | filename = os.path.expanduser(filename) 25 | if os.path.exists(filename): 26 | return filename 27 | 28 | search_path = os.environ.get('HOSTLISTSPATH', '.;~/.hostlists/file').split(';') 29 | for directory in search_path: 30 | new_filename = os.path.expanduser(os.path.join(directory, filename)) 31 | if os.path.exists(new_filename): 32 | return new_filename 33 | 34 | def expand(self, value, name="file"): 35 | tmplist = [] 36 | filename = self._find_file(value) 37 | if not filename: 38 | return [] 39 | 40 | with open(filename, 'r') as file_handle: 41 | for host in file_handle: 42 | host = host.strip() 43 | if not host or host.startswith('#'): 44 | continue 45 | tmplist.append(host) 46 | 47 | return tmplist 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://cd.screwdriver.cd/pipelines/3816/badge?nocache=true)](https://cd.screwdriver.cd/pipelines/3816) 2 | [![Package](https://img.shields.io/badge/package-pypi-blue.svg)](https://pypi.org/project/screwdrivercd/) 3 | [![Downloads](https://img.shields.io/pypi/dm/hostlists.svg)](https://img.shields.io/pypi/dm/hostlists.svg) 4 | [![Codecov](https://codecov.io/gh/yahoo/hostlists/branch/master/graph/badge.svg?nocache=true)](https://codecov.io/gh/yahoo/hostlists) 5 | [![Codestyle](https://img.shields.io/badge/code%20style-pep8-blue.svg)](https://www.python.org/dev/peps/pep-0008/) 6 | [![Documentation](https://img.shields.io/badge/Documentation-latest-blue.svg)](https://yahoo.github.io/hostlists/) 7 | 8 | --- 9 | 10 | # hostlists 11 | 12 | Python module to generate lists of hosts from various sources that is extensible 13 | via plugins. 14 | 15 | 16 | ## Components 17 | 18 | hostlists has 2 components: 19 | 20 | - hostlists - This module handles hostlist expansion 21 | - hostlists_plugins - This module contains plugins that allow hostlists to get lists of hosts from various backend systems. 22 | 23 | 24 | ## Dependencies 25 | 26 | - dnspython (BSD License) - This python module is used for the dns plugins to perform host expansion based on dns. 27 | 28 | 29 | ## Usage 30 | 31 | The hostlists module provides a python module to do host expansion within python 32 | programs. It also provides a command line utility to allow usage from the 33 | command line. 34 | 35 | ## Command Line Examples 36 | 37 | Expand a list of hosts from round robin dns using the dns plugin 38 | 39 | ```bash 40 | $ hostlists dns:www.google.com 41 | pb-in-f99.1e100.net, pb-in-f[103-106].1e100.net, pb-in-f147.1e100.net 42 | ``` 43 | 44 | 45 | Multiple hosts, ranges and plugins can be passed for a single hostlists 46 | 47 | ```bash 48 | $ hostlists dns:www.google.com, poodle[10-20,23].dog.com 49 | pb-in-f99.1e100.net, pb-in-f[103-106].1e100.net, pb-in-f147.1e100.net, poodle[10-20].dog.com, poodle23.dog.com 50 | ``` 51 | -------------------------------------------------------------------------------- /hostlists/decorators.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | A plugin extendable hostlist infrastructure 4 | 5 | This module provides functions for getting a list of hosts 6 | from various systems as well as compressing the list into 7 | a simplified list. 8 | 9 | This module uses the hostlists_plugins python scripts 10 | to actually obtain the listings. 11 | """ 12 | 13 | # Copyright (c) 2010-2015 Yahoo! Inc. All rights reserved. 14 | # Licensed under the Apache License, Version 2.0 (the "License"); 15 | # you may not use this file except in compliance with the License. 16 | # You may obtain a copy of the License at 17 | # 18 | # http://www.apache.org/licenses/LICENSE-2.0 19 | # 20 | # Unless required by applicable law or agreed to in writing, software 21 | # distributed under the License is distributed on an "AS IS" BASIS, 22 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23 | # See the License for the specific language governing permissions and 24 | # limitations under the License. See accompanying LICENSE file. 25 | 26 | 27 | import errno 28 | import os 29 | import signal 30 | from functools import wraps 31 | from .exceptions import MethodTimeoutError 32 | 33 | 34 | def timeout(seconds=300, error_message=os.strerror(errno.ETIME)): 35 | """ 36 | Time out the function after a period of time 37 | :param seconds: 38 | :param error_message: 39 | :return: 40 | """ 41 | def timeout_decorator(func): 42 | """ 43 | Decorator function 44 | """ 45 | def _handle_timeout(signum, frame): 46 | raise MethodTimeoutError(error_message) 47 | 48 | def timeout_wrapper(*args, **kwargs): 49 | signal.signal(signal.SIGALRM, _handle_timeout) 50 | signal.alarm(seconds) 51 | try: 52 | result = func(*args, **kwargs) 53 | finally: 54 | signal.alarm(0) 55 | return result 56 | return wraps(func)(timeout_wrapper) 57 | return timeout_decorator 58 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import unittest 3 | from hostlists.cli import main, parse_arguments 4 | 5 | 6 | class TestCLI(unittest.TestCase): 7 | argv_orig = None 8 | 9 | def setUp(self): 10 | self.argv_orig = sys.argv 11 | 12 | def tearDown(self): 13 | if self.argv_orig: 14 | sys.argv = self.argv_orig 15 | self.argv_orig = None 16 | 17 | def test__parse_arguments__list_plugins(self): 18 | sys.argv = ['hostlists', '--list_plugins'] 19 | result = parse_arguments() 20 | self.assertTrue(result.list_plugins) 21 | 22 | def test__parse_arguments__list_plugins_short(self): 23 | sys.argv = ['hostlists', '-l'] 24 | result = parse_arguments() 25 | self.assertTrue(result.list_plugins) 26 | 27 | def test__parse_arguments__hostrange__single(self): 28 | sys.argv = ['hostlists', 'test[1-2].yahoo.com'] 29 | result = parse_arguments() 30 | self.assertIsInstance(result.host_range, list) 31 | self.assertEqual(len(result.host_range), 1) 32 | self.assertListEqual(result.host_range, ['test[1-2].yahoo.com']) 33 | 34 | def test__parse_arguments__expand__default(self): 35 | sys.argv = ['hostlists', '-e', 'test[1-2].yahoo.com'] 36 | result = parse_arguments() 37 | self.assertIsInstance(result.host_range, list) 38 | self.assertEqual(len(result.host_range), 1) 39 | self.assertListEqual(result.host_range, ['test[1-2].yahoo.com']) 40 | self.assertTrue(result.expand) 41 | self.assertEqual(result.sep, '\n') 42 | 43 | def test__parse_arguments__expand__sep(self): 44 | sys.argv = ['hostlists', '-e', '-s', ',', 'test[1-2].yahoo.com'] 45 | result = parse_arguments() 46 | self.assertIsInstance(result.host_range, list) 47 | self.assertEqual(len(result.host_range), 1) 48 | self.assertListEqual(result.host_range, ['test[1-2].yahoo.com']) 49 | self.assertTrue(result.expand) 50 | self.assertEqual(result.sep, ',') 51 | -------------------------------------------------------------------------------- /tests/test_plugin_load.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2012-2015 Yahoo! Inc. All rights reserved. 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. See accompanying LICENSE file. 14 | """ 15 | Unit tests of sshmap 16 | """ 17 | import hostlists 18 | import unittest 19 | 20 | 21 | class TestHostPluginLoad(unittest.TestCase): 22 | """ 23 | hostmap unit tests 24 | """ 25 | def setUp(self): 26 | self.hostlists_plugins = hostlists.installed_plugins() 27 | 28 | def testPluginHaproxyLoad(self): 29 | import hostlists.plugins.haproxy 30 | self.assertIn('haproxy', hostlists.plugins.haproxy.name()) 31 | self.assertIn('haproxy', self.hostlists_plugins) 32 | 33 | def testPluginDnsIPLoad(self): 34 | import hostlists.plugins.dnsip 35 | self.assertIn('dnsip', hostlists.plugins.dnsip.name()) 36 | self.assertIn('dnsip', self.hostlists_plugins) 37 | 38 | def testPluginDnsLoad(self): 39 | from hostlists_plugins_default.hostlists_plugin_dns import HostlistsPluginDns 40 | self.assertIn('dns', HostlistsPluginDns.names) 41 | self.assertIn('dns', self.hostlists_plugins) 42 | 43 | def testPluginFileLoad(self): 44 | from hostlists_plugins_default.hostlists_plugin_file import HostlistsPluginFile 45 | self.assertIn('file', HostlistsPluginFile.names) 46 | self.assertIn('file', self.hostlists_plugins) 47 | 48 | def testPluginRangeLoad(self): 49 | from hostlists_plugins_default.range import HostlistsPluginRange 50 | self.assertIn('range', HostlistsPluginRange.names) 51 | self.assertIn('range', self.hostlists_plugins) 52 | 53 | 54 | if __name__ == '__main__': 55 | unittest.main() 56 | -------------------------------------------------------------------------------- /hostlists/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from .range import compress, expand, range_split 3 | from .plugin_manager import installed_plugins 4 | 5 | 6 | def parse_arguments(): 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument( 9 | 'host_range', nargs='*', type=str, help='plugin:parameters' 10 | ) 11 | parser.add_argument( 12 | "-s", "--sep", 13 | dest="sep", 14 | type=str, 15 | default='', 16 | help="Separator character, default=\",\"" 17 | ) 18 | parser.add_argument( 19 | "--onepass", 20 | dest="onepass", 21 | default=False, 22 | action="store_true", 23 | help="Only perform a single expansion pass (no recursion)" 24 | ) 25 | parser.add_argument( 26 | "--expand", "-e", 27 | dest="expand", 28 | default=False, 29 | action="store_true", 30 | help="Expand the host list and display one host per line" 31 | ) 32 | parser.add_argument( 33 | "--list_plugins", "-l", 34 | dest="list_plugins", 35 | default=False, 36 | action="store_true", 37 | help="List the currently found hostlists plugins" 38 | ) 39 | result = parser.parse_args() 40 | if result.expand: 41 | if not result.sep: 42 | result.sep = '\n' 43 | else: 44 | if not result.sep: 45 | result.sep = ',' 46 | return result 47 | 48 | 49 | def main(): # pragma: no cover 50 | options = parse_arguments() 51 | if options.list_plugins: 52 | plugins = installed_plugins() 53 | plugins.sort() 54 | print( 55 | 'Hostlists plugins currently installed are:' 56 | ) 57 | print('\t' + '\n\t'.join(plugins)) 58 | return 59 | 60 | hostnames = range_split(','.join(options.host_range)) 61 | seperator = options.sep + ' ' 62 | if options.expand: 63 | print(options.sep.join(expand(hostnames, onepass=options.onepass))) 64 | else: 65 | print( 66 | seperator.join( 67 | compress( 68 | expand( 69 | hostnames, onepass=options.onepass 70 | ) 71 | ) 72 | ).strip().strip(',').strip() 73 | ) 74 | -------------------------------------------------------------------------------- /hostlists_plugins_default/hostlists_plugin_dns.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ hostlists plugin to get hosts from dns """ 3 | 4 | # Copyright (c) 2010-2013 Yahoo! Inc. All rights reserved. 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. See accompanying LICENSE file. 16 | import dns.resolver 17 | import dns.reversename 18 | from hostlists.plugin_base import HostlistsPlugin 19 | 20 | 21 | class HostlistsPluginDns(HostlistsPlugin): 22 | names = ['dns'] 23 | 24 | def expand(self, value, name=None): 25 | tmplist = [] 26 | addresses = [] 27 | try: 28 | answers = list(dns.resolver.query(value)) 29 | except dns.resolver.NoAnswer: 30 | answers = [] 31 | for rdata in answers: 32 | addresses.append(rdata.address) 33 | for address in addresses: 34 | result = dns.reversename.from_address(address) 35 | try: 36 | # See if we can reverse resolv the IP address and insert 37 | # it only if it's a unique hostname. 38 | revaddress = str(dns.resolver.query(result, 'PTR')[0]).strip('.') 39 | if revaddress not in tmplist: 40 | tmplist.append('type_vip:' + revaddress) 41 | except dns.resolver.NXDOMAIN: 42 | tmplist.append('type_vip:' + address) 43 | # if the tmplist with the reverse resolved hostnames 44 | # is not the same length as the list of addresses then 45 | # the site has the same hostname assigned to more than 46 | # one IP address dns reverse resolving 47 | # If this happens we return the IP addresses because 48 | # they are unique, otherwise we return the hostnames. 49 | if len(tmplist) < len(addresses): 50 | return addresses 51 | return tmplist -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | author = Dwight Hubbard 3 | author_email = dhubbard@yahoo-inc.com 4 | classifier = 5 | Development Status :: 5 - Production/Stable 6 | Environment :: Console 7 | Intended Audience :: Developers 8 | Intended Audience :: System Administrators 9 | License :: OSI Approved :: Apache Software License 10 | Natural Language :: English 11 | Operating System :: POSIX 12 | Operating System :: POSIX :: Linux 13 | Operating System :: POSIX :: SunOS/Solaris 14 | Operating System :: MacOS :: MacOS X 15 | Programming Language :: Python 16 | Programming Language :: Python :: 3 17 | Programming Language :: Python :: 3.9 18 | Programming Language :: Python :: 3.10 19 | Programming Language :: Python :: 3.11 20 | Programming Language :: Python :: 3.12 21 | Programming Language :: Python :: Implementation :: CPython 22 | Topic :: System :: Systems Administration 23 | Topic :: Utilities 24 | description = Python module to generate lists of hosts from various sources that is extensible via plugins. 25 | keywords = hostlist, utility 26 | license = Apache Software License 27 | long_description = file:README.md 28 | long_description_content_type = text/markdown 29 | name = hostlists 30 | project_urls = 31 | Documentation = https://yahoo.github.io/hostlists/ 32 | Source = https://github.com/yahoo/hostlists 33 | CI Pipeline = https://cd.screwdriver.cd/pipelines/3816 34 | url = https://github.com/yahoo/hostlists 35 | version = 0.10.0 36 | 37 | [options] 38 | install_requires = 39 | dnspython 40 | requests 41 | packages = 42 | hostlists 43 | hostlists.plugins 44 | hostlists_plugins_default 45 | python_requires = >=3.9 46 | zip_safe = True 47 | 48 | [options.entry_points] 49 | console_scripts = 50 | hostlists=hostlists.cli:main 51 | hostlists_plugins = 52 | dns=hostlists_plugins_default.hostlists_plugin_dns:HostlistsPluginDns 53 | file=hostlists_plugins_default.hostlists_plugin_file:HostlistsPluginFile 54 | range=hostlists_plugins_default.range:HostlistsPluginRange 55 | 56 | [build_sphinx] 57 | source-dir = docs/source 58 | build-dir = docs/build 59 | all_files = 1 60 | 61 | [pycodestyle] 62 | count = False 63 | ignore = E1,E2,E3,E4,E5,W293,E226,E302,E41 64 | max-line-length = 160 65 | statistics = True 66 | 67 | [upload_sphinx] 68 | upload-dir = docs/build/html 69 | 70 | [screwdrivercd.version] 71 | version_type = sdv4_SD_BUILD 72 | -------------------------------------------------------------------------------- /hostlists/plugins/get_haproxy_phys: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Copyright (c) 2012-2014 Yahoo! Inc. All rights reserved. 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. See accompanying LICENSE file. 16 | """ 17 | 18 | import sys 19 | import socket 20 | import json 21 | 22 | # Default socket filename 23 | socket_file = '/tmp/haproxy_socket' 24 | 25 | 26 | def get_haproxy_socket_filename(): 27 | socket_file = None 28 | # Open the haproxy config and see if we can find the socket file 29 | for line in open('/etc/haproxy/haproxy.cfg'): 30 | splitline = line.strip().split() 31 | if len(splitline) > 2 and splitline[0] == 'stats' and splitline[ 32 | 1] == 'socket': 33 | socket_file = splitline[2] 34 | break 35 | return socket_file 36 | 37 | 38 | def query_haproxy_socket(socket_file='/tmp/haproxy', command='show stat'): 39 | if not command.endswith('\n'): 40 | command += '\n' 41 | s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 42 | s.connect(socket_file) 43 | s.send(command) 44 | data = "" 45 | d = s.recv(1024) 46 | while d: 47 | data += d 48 | d = s.recv(1024) 49 | #data = s.recv() 50 | s.close() 51 | return data 52 | 53 | 54 | def get_backends(vip, state='ALL'): 55 | hosts = [] 56 | for line in query_haproxy_socket(get_haproxy_socket_filename(), 57 | 'show stat').split('\n'): 58 | if not line.startswith('#') and len(line.strip()): 59 | splitline = line.strip().split(',') 60 | if splitline[0] == vip and splitline[1] != 'BACKEND': 61 | if state.upper() == 'ALL' or (splitline[17] == state): 62 | hosts.append(splitline[1]) 63 | return hosts 64 | 65 | 66 | state = 'ALL' 67 | if len(sys.argv) > 2 and sys.argv[2].upper() in ['UP', 'DOWN']: 68 | state = sys.argv[2].upper() 69 | print(json.dumps(get_backends(sys.argv[1], state), indent=4)) 70 | -------------------------------------------------------------------------------- /hostlists/plugins/plugintype.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | hostlists plugin to recursively query plugins based on type. 4 | 5 | This makes it possible to obtain lists of hosts by recursively 6 | querying multiple backends. 7 | 8 | For example: 9 | * Query dns for www.foo.com 10 | * Get a list of two hostnames back haproxy1.ny.foo.com and 11 | haproxy1.lax.foo.com. 12 | * Query reverse proxies and load balancers for the 13 | above two hostnames and the names of any hosts serving 14 | the traffic for them. haproxy1.ny.foo.com is a vip being 15 | served by apache1.ny.foo.com ad apache2.ny.foo.com. 16 | haproxy1.lax.foo.com is a vip being serviced by 17 | apache2.lax.foo.com, apache3.lax.foo.com and 18 | joesdesktop.foo.com. 19 | * Return apache[1-2].ny.foo.com, apache[2-3].lax.foo.com, 20 | joesdektop.foo.com 21 | """ 22 | 23 | # Copyright (c) 2010-2015 Yahoo! Inc. All rights reserved. 24 | # Licensed under the Apache License, Version 2.0 (the "License"); 25 | # you may not use this file except in compliance with the License. 26 | # You may obtain a copy of the License at 27 | # 28 | # http://www.apache.org/licenses/LICENSE-2.0 29 | # 30 | # Unless required by applicable law or agreed to in writing, software 31 | # distributed under the License is distributed on an "AS IS" BASIS, 32 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 33 | # See the License for the specific language governing permissions and 34 | # limitations under the License. See accompanying LICENSE file. 35 | 36 | 37 | from hostlists.plugin_manager import get_plugins 38 | 39 | 40 | def name(): 41 | return ['type', 'type_vip', 'type_vip_up', 'type_vip_down'] 42 | 43 | 44 | def expand(value, name=None): 45 | """ Try all plugins of a specific type for a result, if none 46 | are able to expand the value further then return just the value """ 47 | mod_type = 'vip' 48 | if not name: 49 | return [value] 50 | if name.lower() in ['type_vip']: 51 | mod_type = 'vip' 52 | filter_append = '' 53 | if name.lower() in ['type_vip_down']: 54 | mod_type = 'vip_down' 55 | filter_append = '_down' 56 | if name.lower() in ['type_vip_up']: 57 | mod_type = 'vip_up' 58 | filter_append = '_up' 59 | plugins = get_plugins() 60 | for plugin_name in plugins.keys(): 61 | if ( 62 | (filter_append != '' and plugin_name.endswith(filter_append)) or (filter_append == '' and plugin_name.find('_') == -1) 63 | ): 64 | try: 65 | if mod_type in plugins[plugin_name].type(): 66 | name = plugin_name + filter_append 67 | result = plugins[plugin_name].expand(value, name=name) 68 | if len(result): 69 | return result 70 | except AttributeError: 71 | pass 72 | return [value] 73 | -------------------------------------------------------------------------------- /tests/test_expand.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2010-2015 Yahoo! Inc. All rights reserved. 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. See accompanying LICENSE file. 14 | """ 15 | Unit tests of sshmap 16 | """ 17 | import hostlists 18 | import tempfile 19 | import unittest 20 | import os 21 | 22 | 23 | class TestHostlistsExpand(unittest.TestCase): 24 | """ 25 | hostmap unit tests 26 | """ 27 | 28 | class base_test_expand_string_input(unittest.TestCase): 29 | range_list = 'localhost' 30 | expected_result = ['localhost'] 31 | 32 | def __init__(self): 33 | result = hostlists.expand(self.range_list) 34 | self.assertIsInstance(result, list) 35 | self.assertListEqual(result, self.expected_result) 36 | 37 | class test_expand__string_input__single_host(base_test_expand_string_input): 38 | """ Expand a string with a single host 39 | """ 40 | range_list = 'localhost' 41 | expected_result = ['localhost'] 42 | 43 | class test_expand__string_input__multiple_host( 44 | base_test_expand_string_input 45 | ): 46 | """ 47 | Expand a string containing multiple comma seperated hosts 48 | """ 49 | range_list = 'localhost, foobar' 50 | expected_result = ['localhost', 'foobar'] 51 | 52 | class test_expand__string_input__multiple_host__range( 53 | base_test_expand_string_input 54 | ): 55 | range_list = 'localhost[3-5], foobar' 56 | expected_result = [ 57 | 'localhost3', 'localhost4', 'localhost5', 'foobar' 58 | ] 59 | 60 | class disabled_test_expand__string_input__multiple_host__range_gap( 61 | base_test_expand_string_input 62 | ): 63 | range_list = 'localhost[3-5,7], foobar' 64 | expected_result = [ 65 | 'localhost3', 'localhost4', 'localhost5', 'localhost7', 'foobar' 66 | ] 67 | 68 | class test_expand__string_input__file_plugin( 69 | base_test_expand_string_input 70 | ): 71 | expected_result = ['localhost'] 72 | 73 | def setUp(self): 74 | self.tfile = tempfile.NamedTemporaryFile(delete=False, mode='w') 75 | self.tfile.write('localhost\n') 76 | self.tfile.close() 77 | self.range_list = 'file:%s' % self.tfile.name 78 | 79 | def tearDown(self): 80 | if self.tfile: 81 | os.remove(self.tfile.name) 82 | 83 | 84 | if __name__ == '__main__': 85 | unittest.main() 86 | -------------------------------------------------------------------------------- /hostlists_plugins_default/range.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ hostlists plugin to get hosts from a range """ 3 | 4 | # Copyright (c) 2010-2013 Yahoo! Inc. All rights reserved. 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. See accompanying LICENSE file. 16 | from hostlists.plugin_base import HostlistsPlugin 17 | 18 | 19 | class HostlistsPluginRange(HostlistsPlugin): 20 | names = ['range'] 21 | 22 | def expand(self, value, name=None): 23 | """ 24 | Use Plugin to expand the value 25 | """ 26 | return self.expand_item(value) 27 | 28 | def block_to_list(self, block): 29 | """ Convert a range block into a numeric list 30 | input "1-3,17,19-20" 31 | output=[1,2,3,17,19,20] 32 | """ 33 | block += ',' 34 | result = [] 35 | val = val1 = '' 36 | in_range = False 37 | for letter in block: 38 | if letter in [',', '-']: 39 | if in_range: 40 | val2 = val 41 | val2_len = len(val2) 42 | # result+=range(int(val1),int(val2)+1) 43 | for value in range(int(val1), int(val2) + 1): 44 | if val1.startswith('0'): 45 | result.append(str(value).zfill(val2_len)) 46 | else: 47 | result.append(str(value)) 48 | val = '' 49 | val1 = None 50 | in_range = False 51 | else: 52 | val1 = val 53 | val1_len = len(val1) 54 | val = '' 55 | if letter == ',': 56 | if val1 is not None: 57 | result.append(val1.zfill(val1_len)) # pragma: no cover 58 | else: 59 | in_range = True 60 | else: 61 | val += letter 62 | return result 63 | 64 | 65 | def expand_item(self, item): 66 | result = [] 67 | in_block = False 68 | pre_block = '' 69 | for count in range(0, len(item)): 70 | letter = item[count] 71 | if letter == '[': 72 | in_block = True 73 | block = '' 74 | elif letter == ']' and in_block: 75 | in_block = False 76 | for value in self.block_to_list(block): 77 | result.append('%s%s%s' % (pre_block, value, item[count + 1:])) 78 | elif in_block: 79 | block += letter 80 | elif not in_block: 81 | pre_block += letter 82 | if len(result): 83 | return result 84 | else: 85 | return [item] # pragma: no cover 86 | -------------------------------------------------------------------------------- /hostlists/plugin_manager.py: -------------------------------------------------------------------------------- 1 | import imp 2 | import logging 3 | import os 4 | import sys 5 | import pkg_resources 6 | from .exceptions import HostListsError 7 | 8 | 9 | # Global plugin cache so we don't constantly reload the plugin modules 10 | global_plugins = {} 11 | pkg_resources_probed = False 12 | plugins_probed = False 13 | 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def _get_plugins__pkg_resources(): 19 | """Query pkg_resources for packages that registered with the plugin entrypoint""" 20 | global global_plugins 21 | global pkg_resources_probed 22 | if pkg_resources_probed: 23 | return global_plugins 24 | entry_points = pkg_resources.iter_entry_points(group='hostlists_plugins') 25 | if not entry_points: 26 | return global_plugins 27 | for entry_point in entry_points: 28 | plugin_class = entry_point.load() 29 | for name in plugin_class.names: 30 | global_plugins[name] = plugin_class() 31 | pkg_resources_probed = True 32 | return global_plugins 33 | 34 | 35 | def get_plugins(): 36 | """ Find all the hostlists plugins """ 37 | global global_plugins 38 | global plugins_probed 39 | 40 | if plugins_probed: 41 | return global_plugins 42 | 43 | plugins = _get_plugins__pkg_resources() 44 | 45 | pluginlist = [] 46 | plugin_path = [ 47 | '/home/y/lib/hostlists', 48 | os.path.dirname(__file__), 49 | '~/.hostlists', 50 | ] + sys.path 51 | for directory in plugin_path: 52 | if os.path.isdir(os.path.join(directory, 'plugins')): 53 | templist = os.listdir(os.path.join(directory, 'plugins')) 54 | for item in templist: 55 | pluginlist.append( 56 | os.path.join(os.path.join(directory, 'plugins'), item) 57 | ) 58 | pluginlist.sort() 59 | # Create a dict mapping the plugin name to the plugin method 60 | for item in pluginlist: 61 | if item.endswith('.py'): 62 | module_file = open(item) 63 | try: 64 | mod = imp.load_module( 65 | 'hostlists_plugins_%s' % os.path.basename(item[:-3]), 66 | module_file, 67 | item, 68 | ('.py', 'r', imp.PY_SOURCE) 69 | ) 70 | names = mod.name() 71 | if isinstance(names, str): 72 | names = [names] 73 | for name in names: 74 | if name not in plugins.keys(): 75 | plugins[name.lower()] = mod 76 | except (AttributeError, ImportError): 77 | # Error in module import, probably a plugin bug 78 | logger.debug( 79 | "Plugin import failed %s:" % item 80 | ) 81 | if module_file: 82 | module_file.close() 83 | plugins_probed = True 84 | return plugins 85 | 86 | 87 | def multiple_names(plugin): 88 | plugins = get_plugins() 89 | count = 0 90 | for item in plugins.keys(): 91 | if plugins[item] == plugin: 92 | count += 1 93 | if count > 1: # Pragma no cover 94 | return True 95 | else: 96 | return False 97 | 98 | 99 | def run_plugin_expand(name, value): 100 | """ 101 | Run a plugin's expand method 102 | """ 103 | plugins = get_plugins() 104 | if name not in plugins.keys(): 105 | raise HostListsError( 106 | 'plugin %s not found, valid plugins are: %s' % ( 107 | name, ','.join(plugins.keys()) 108 | ) 109 | ) 110 | logger.debug(plugins[name]) 111 | logger.debug(dir(plugins[name])) 112 | return plugins[name].expand(value, name=name) 113 | 114 | 115 | def installed_plugins(): 116 | plugins = [] 117 | for plugin in get_plugins(): 118 | if plugin: 119 | plugins.append(plugin) 120 | return plugins 121 | -------------------------------------------------------------------------------- /tests/test_hostlists.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2012-2015 Yahoo! Inc. All rights reserved. 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. See accompanying LICENSE file. 14 | """ 15 | Unit tests of sshmap 16 | """ 17 | import hostlists 18 | import json 19 | import os 20 | import types 21 | import unittest 22 | 23 | 24 | class TestHostlists(unittest.TestCase): 25 | """ 26 | hostlists_module.py unit tests 27 | """ 28 | def test_cmp_compat(self): 29 | self.assertEqual(hostlists.range.cmp_compat(1, 2), -1) 30 | self.assertEqual(hostlists.range.cmp_compat(2, 1), 1) 31 | 32 | def test_get_plugins(self): 33 | plugins = hostlists.get_plugins() 34 | self.assertIn('file', plugins.keys()) 35 | 36 | def test_get_setting_without_config_file(self): 37 | if os.path.exists('test_get_setting.conf'): 38 | os.remove('test_get_setting.conf') 39 | hostlists.CONF_FILE = os.path.abspath('test_get_setting.conf') 40 | result = hostlists.get_setting('key') 41 | self.assertIsNone(result) 42 | 43 | def test_get_setting_with_config_file(self): 44 | expected_dict = { 45 | 'key': 'value' 46 | } 47 | with open('test_get_setting.conf', 'w') as tf: 48 | json.dump(expected_dict, tf) 49 | hostlists.CONF_FILE = os.path.abspath('test_get_setting.conf') 50 | result = hostlists.get_setting('key') 51 | os.remove('test_get_setting.conf') 52 | self.assertEqual(result, 'value') 53 | 54 | def test_expand(self): 55 | """ 56 | Expand a list of lists and set operators into a final host lists 57 | >>> hostlists.expand(['foo[01-10]','-','foo[04-06]']) 58 | ['foo09', 'foo08', 'foo07', 'foo02', 'foo01', 'foo03', 'foo10'] 59 | >>> 60 | """ 61 | result = hostlists.expand(['foo[01-10]', '-', 'foo[04-06]']) 62 | expected_result = [ 63 | 'foo09', 'foo08', 'foo07', 'foo02', 'foo01', 'foo03', 'foo10'] 64 | result.sort() 65 | expected_result.sort() 66 | self.assertLessEqual(result, expected_result) 67 | 68 | def test_expand_invalid_plugin(self): 69 | with self.assertRaises(hostlists.HostListsError): 70 | hostlists.expand(['boozle:bar']) 71 | 72 | def test_expand_file(self): 73 | with open('test_expand_file.hostlist', 'w') as fh: 74 | fh.write('foo[1-2]\n') 75 | result = hostlists.expand(['file:test_expand_file.hostlist']) 76 | expected_result = ['foo1', 'foo2'] 77 | os.remove('test_expand_file.hostlist') 78 | result.sort() 79 | self.assertListEqual(result, expected_result) 80 | 81 | def test_expand_dns(self): 82 | result = hostlists.expand(['dns:yahoo.com']) 83 | self.assertGreater(len(result), 0) 84 | 85 | def test_expand_dnsip(self): 86 | result = hostlists.expand(['dnsip:yahoo.com']) 87 | self.assertGreater(len(result), 0) 88 | 89 | def test_compress(self): 90 | result = hostlists.compress(['foo1', 'foo3', 'foo4']) 91 | expected_result = ['foo1', 'foo[3-4]'] 92 | self.assertListEqual(result, expected_result) 93 | 94 | def test_range_split(self): 95 | result = hostlists.range_split('foo1, foo[3-9]') 96 | expected_result = ['foo1', 'foo[3-9]'] 97 | self.assertListEqual(result, expected_result) 98 | 99 | 100 | if __name__ == '__main__': 101 | unittest.main() 102 | -------------------------------------------------------------------------------- /hostlists/plugins/haproxy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | haproxy host plugin 4 | 5 | USAGE 6 | ----- 7 | This plugin requires a signifcant amount of pre-configuration on the haproxy 8 | server in order to work. This requires a configuration change to enable 9 | the stats socket in haproxy, setting permissions, and copying the 10 | get_haproxy_phys script into place. 11 | 12 | The setup steps are: 13 | 14 | 1. Enable the haproxy status socket, the script expects it to be /tmp/haproxy 15 | by adding: 16 | stats socket /tmp/haproxy 17 | to the global section of the /etc/haproxy/haproxy.cfg file 18 | 19 | 2. Restart haproxy to create the socket 20 | 21 | 3. Make sure the socket file is owned by the user that will be connecting 22 | to the haproxy server. 23 | 24 | 4. Copy the get_haproxy_phys from the sshmap hostlists_plugins directory 25 | into the root of the home directory of the user that will be connecing 26 | to the haproxy server. 27 | 5. Set up the user's ~/.ssh/authorized_keys file to allow access via ssh 28 | without password. 29 | 30 | NOTE 31 | ---- 32 | This plugin is primarily intended as a proof of concept of adding a load 33 | balancer as the host IP address source. 34 | 35 | THEORY OF OPERATION 36 | ------------------- 37 | Since haproxy does not provide an externally accessible means of accessing 38 | the backend names and states for hosts it is proxying this module uses ssh 39 | to call a helper script on the haproxy server to get the information. 40 | """ 41 | 42 | # Copyright (c) 2012 Yahoo! Inc. All rights reserved. 43 | # Licensed under the Apache License, Version 2.0 (the "License"); 44 | # you may not use this file except in compliance with the License. 45 | # You may obtain a copy of the License at 46 | # 47 | # http://www.apache.org/licenses/LICENSE-2.0 48 | # 49 | # Unless required by applicable law or agreed to in writing, software 50 | # distributed under the License is distributed on an "AS IS" BASIS, 51 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 52 | # See the License for the specific language governing permissions and 53 | # limitations under the License. See accompanying LICENSE file. 54 | 55 | import base64 56 | import hostlists 57 | import json 58 | import os 59 | import json.decoder 60 | import subprocess # nosec 61 | 62 | 63 | from urllib.request import urlopen 64 | from urllib.request import URLError 65 | from urllib.request import Request 66 | 67 | 68 | def name(): 69 | """ Name of plugins this plugin responds to """ 70 | return ['haproxy', 'haproxy_all', 'haproxy_up', 'haproxy_down'] 71 | 72 | 73 | def plugin_type(): 74 | """ Type of plugin this is """ 75 | return ['vip'] 76 | 77 | 78 | def server_setting(server='default', setting=None): 79 | if not setting: 80 | return None 81 | settings = hostlists.get_setting('haproxy_plugin') 82 | if not settings: 83 | return None 84 | if server in settings.keys(): 85 | if setting in settings[server].keys(): 86 | return settings[server][setting] 87 | return None 88 | 89 | 90 | # noinspection PyBroadException 91 | def expand(value, name='haproxy', method=None): 92 | state = 'ALL' 93 | if name in ['haproxy_up']: 94 | state = 'UP' 95 | if name in ['haproxy_down']: 96 | state = 'DOWN' 97 | temp = value.split(':') 98 | if len(temp): 99 | haproxy = temp[0] 100 | if len(temp) > 1: 101 | backend = temp[1] 102 | else: 103 | backend = 'all' 104 | else: 105 | return [] 106 | # Determine settings 107 | # the last setting that is found is the setting so we go 108 | # from most generic to least 109 | # first some reasonable hardwired defaults 110 | timeout = 2 111 | userid = None 112 | password = None 113 | # Next try and get setting defaults from the config file 114 | if server_setting('default', 'userid'): 115 | userid = server_setting('default', 'userid') 116 | if server_setting('default', 'password'): 117 | password = server_setting('default', 'password') 118 | if server_setting('default', 'timeout'): 119 | timeout = server_setting('default', 'timeout') 120 | if haproxy and not method and server_setting(haproxy, 'method'): 121 | method = server_setting(haproxy, 'method') 122 | if not method and server_setting('default', 'method'): 123 | method = server_setting('default', 'method') 124 | # Finally try settings specific to the server 125 | if haproxy: 126 | if server_setting(haproxy, 'userid'): 127 | userid = server_setting(haproxy, 'userid') 128 | if server_setting(haproxy, 'password'): 129 | password = server_setting(haproxy, 'password') 130 | if server_setting(haproxy, 'timeout'): 131 | timeout = server_setting(haproxy, 'timeout') 132 | tmplist = [] 133 | if method == 'ssh': 134 | try: 135 | hosts = subprocess.check_output(['ssh', haproxy, './get_haproxy_phys', backend, state]).decode(errors='ignore') # nosec 136 | except (FileNotFoundError, subprocess.CalledProcessError): 137 | return [] 138 | try: 139 | return json.loads(hosts) 140 | except ValueError: 141 | return [] 142 | else: 143 | url = "http://%s/haproxy?stats;csv" % haproxy 144 | request = Request(url) 145 | if userid and password: 146 | userid = userid.strip() 147 | password = password.strip() 148 | authbytes = f'{userid}:{password}'.encode(errors='ignore') 149 | base64string = base64.encodebytes(authbytes) 150 | request.add_header("Authorization", "Basic %s" % base64string) 151 | try: 152 | result = urlopen(request, timeout=timeout).read() # nosec 153 | for line in result.split('\n'): 154 | if not line.startswith('#') and len( 155 | line.strip()) and ',' in line: 156 | splitline = line.strip().split(',') 157 | if (splitline[0] == backend or backend.lower() == 'all') and \ 158 | splitline[1] not in ['BACKEND', 'FRONTEND']: 159 | if state.upper() == 'ALL' or (splitline[17] == state): 160 | tmplist.append(splitline[1]) 161 | return tmplist 162 | except URLError: 163 | return [] 164 | -------------------------------------------------------------------------------- /hostlists/range.py: -------------------------------------------------------------------------------- 1 | """hostlists range management functions""" 2 | import operator 3 | import re 4 | from .plugin_manager import run_plugin_expand 5 | 6 | 7 | # A list of operators we use for set options 8 | SET_OPERATORS = ['-'] 9 | 10 | 11 | def cmp_compat(a, b): 12 | """ 13 | Simple comparison function 14 | :param a: 15 | :param b: 16 | :return: 17 | """ 18 | return (a > b) - (a < b) 19 | 20 | 21 | def compress(hostnames): 22 | """ 23 | Compress a list of host into a more compact range representation 24 | """ 25 | domain_dict = {} 26 | result = [] 27 | for host in hostnames: 28 | if '.' in host: 29 | domain = '.'.join(host.split('.')[1:]) 30 | else: 31 | domain = '' 32 | try: 33 | domain_dict[domain].append(host) 34 | except KeyError: 35 | domain_dict[domain] = [host] 36 | domains = list(domain_dict.keys()) 37 | domains.sort() 38 | for domain in domains: 39 | hosts = compress_domain(domain_dict[domain]) 40 | result += hosts 41 | return result 42 | 43 | 44 | def compress_domain(hostnames): 45 | """ 46 | Compress a list of hosts in a domain into a more compact representation 47 | """ 48 | hostnames.sort() 49 | prev_dict = {'prefix': "", 'suffix': '', 'number': 0} 50 | items = [] 51 | items_block = [] 52 | new_hosts = [] 53 | for host in hostnames: 54 | try: 55 | parsed_dict = re.match( 56 | r"(?P[^0-9]+)(?P\d+)(?P.*).?", 57 | host 58 | ).groupdict() 59 | # To generate the range we need the entries sorted numerically 60 | # but to ensure we don't loose any leading 0s we don't want to 61 | # replace the number parameter that is a string with the leading 62 | # 0s. 63 | parsed_dict['number_int'] = int(parsed_dict['number']) 64 | new_hosts.append(parsed_dict) 65 | except AttributeError: 66 | if '.' not in host: 67 | host += '.' 68 | parsed_dict = {'host': compress([host])[0].strip('.')} 69 | else: 70 | parsed_dict = {'host': host} 71 | new_hosts.append(parsed_dict) 72 | new_hosts = multikeysort(new_hosts, ['prefix', 'number_int']) 73 | for parsed_dict in new_hosts: 74 | if 'host' in parsed_dict.keys() or \ 75 | parsed_dict['prefix'] != prev_dict['prefix'] or \ 76 | parsed_dict['suffix'] != prev_dict['suffix'] or \ 77 | int(parsed_dict['number']) != int(prev_dict['number']) + 1: 78 | if len(items_block): 79 | items.append(items_block) 80 | items_block = [parsed_dict] 81 | else: 82 | items_block.append(parsed_dict) 83 | prev_dict = parsed_dict 84 | items.append(items_block) 85 | result = [] 86 | for item in items: 87 | if len(item): 88 | if len(item) == 1 and 'host' in item[0].keys(): 89 | result.append(item[0]['host']) 90 | elif len(item) == 1: 91 | result.append( 92 | '%s%s%s' % ( 93 | item[0]['prefix'], item[0]['number'], item[0]['suffix'] 94 | ) 95 | ) 96 | else: 97 | result.append( 98 | '%s[%s-%s]%s' % ( 99 | item[0]['prefix'], 100 | item[0]['number'], 101 | item[-1]['number'], 102 | item[0]['suffix'] 103 | ) 104 | ) 105 | return result 106 | 107 | 108 | def multikeysort(items, columns): 109 | comparers = [ 110 | ((operator.itemgetter(col[1:].strip()), -1) if col.startswith('-') else (operator.itemgetter(col.strip()), 1)) for col in columns 111 | ] 112 | 113 | def comparer(left, right): 114 | for fn, mult in comparers: 115 | try: 116 | result = cmp_compat(fn(left), fn(right)) 117 | except KeyError: 118 | return 0 119 | if result: 120 | return mult * result 121 | else: 122 | return 0 123 | try: 124 | # noinspection PyArgumentList 125 | return sorted(items, cmp=comparer) 126 | except TypeError: 127 | # Python 3 removed the cmp parameter 128 | import functools 129 | return sorted(items, key=functools.cmp_to_key(comparer)) 130 | 131 | 132 | def range_split(hosts): 133 | """ 134 | Split up a range string, this needs to separate comma separated 135 | items unless they are within square brackets and split out set operations 136 | as separate items. 137 | """ 138 | in_brackets = False 139 | current = "" 140 | result_list = [] 141 | for c in hosts: 142 | if c in ['[']: 143 | in_brackets = True 144 | if c in [']']: 145 | in_brackets = False 146 | if not in_brackets and c == ',': 147 | result_list.append(current) 148 | current = "" 149 | # elif not in_brackets and c == '-': 150 | # result_list.append(current) 151 | # result_list.append('-') 152 | # current = "" 153 | elif not in_brackets and c in [','] and len(current) == 0: 154 | pass 155 | else: 156 | current += c 157 | current = current.strip().strip(',') 158 | if current: 159 | result_list.append(current) 160 | return result_list 161 | 162 | 163 | def expand(range_list, onepass=False): 164 | """ 165 | Expand a list of lists and set operators into a final host lists 166 | >>> expand(['foo[01-10]','-','foo[04-06]']) 167 | ['foo09', 'foo08', 'foo07', 'foo02', 'foo01', 'foo03', 'foo10'] 168 | >>> 169 | """ 170 | if isinstance(range_list, str): # pragma: no cover 171 | range_list = [h.strip() for h in range_list.split(',')] 172 | new_list = [] 173 | set1 = None 174 | operation = None 175 | for item in range_list: 176 | if set1 and operation: 177 | set2 = expand_item(item) 178 | new_list.append(list(set(set1).difference(set(set2)))) 179 | set1 = None 180 | operation = None 181 | elif item in SET_OPERATORS and len(new_list): 182 | set1 = new_list.pop() 183 | operation = item 184 | else: 185 | expanded_item = expand_item(item, onepass=onepass) 186 | new_list.append(expanded_item) 187 | new_list2 = [] 188 | for item in new_list: 189 | new_list2 += item 190 | return new_list2 191 | 192 | 193 | def expand_item(range_list, onepass=False): 194 | """ Expand a list of plugin:parameters into a list of hosts """ 195 | 196 | if isinstance(range_list, str): 197 | range_list = [range_list] 198 | 199 | # Iterate through our list 200 | newlist = [] 201 | found_plugin = False 202 | for item in range_list: 203 | # Is the item a plugin 204 | temp = item.split(':') 205 | found_plugin = False 206 | if len(temp) > 1: 207 | plugin = temp[0].lower() 208 | # Do we have a plugin that matches the passed plugin 209 | newlist += run_plugin_expand(plugin, ':'.join(temp[1:]).strip(':')) 210 | found_plugin = True 211 | else: 212 | # Default to running through the range plugin 213 | newlist += run_plugin_expand('range', temp[0]) 214 | 215 | # Recurse back through ourselves incase a plugin returns a value that 216 | # needs to be parsed 217 | # by another plugin. For example a dns resource that has an address that 218 | # points to a load balancer vip that may container a number of hosts that 219 | # need to be looked up via the load_balancer plugin. 220 | if found_plugin and not onepass: 221 | newlist = expand_item(newlist) 222 | return newlist 223 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # hostlists documentation build configuration file, created by 5 | # sphinx-quickstart on Thu Mar 5 15:56:28 2015. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import hostlists 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.intersphinx', 34 | 'sphinx.ext.ifconfig', 35 | ] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # The suffix of source filenames. 41 | source_suffix = '.rst' 42 | 43 | # The encoding of source files. 44 | #source_encoding = 'utf-8-sig' 45 | 46 | # The master toctree document. 47 | master_doc = 'index' 48 | 49 | # General information about the project. 50 | project = 'hostlists' 51 | copyright = '2015, Dwight Hubbard' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = hostlists.__version__ 59 | 60 | # The full version, including alpha/beta/rc tags. 61 | release = version 62 | 63 | # The language for content autogenerated by Sphinx. Refer to documentation 64 | # for a list of supported languages. 65 | #language = None 66 | 67 | # There are two options for replacing |today|: either, you set today to some 68 | # non-false value, then it is used: 69 | #today = '' 70 | # Else, today_fmt is used as the format for a strftime call. 71 | #today_fmt = '%B %d, %Y' 72 | 73 | # List of patterns, relative to source directory, that match files and 74 | # directories to ignore when looking for source files. 75 | exclude_patterns = [] 76 | 77 | # The reST default role (used for this markup: `text`) to use for all 78 | # documents. 79 | #default_role = None 80 | 81 | # If true, '()' will be appended to :func: etc. cross-reference text. 82 | #add_function_parentheses = True 83 | 84 | # If true, the current module name will be prepended to all description 85 | # unit titles (such as .. function::). 86 | #add_module_names = True 87 | 88 | # If true, sectionauthor and moduleauthor directives will be shown in the 89 | # output. They are ignored by default. 90 | #show_authors = False 91 | 92 | # The name of the Pygments (syntax highlighting) style to use. 93 | pygments_style = 'sphinx' 94 | 95 | # A list of ignored prefixes for module index sorting. 96 | #modindex_common_prefix = [] 97 | 98 | # If true, keep warnings as "system message" paragraphs in the built documents. 99 | #keep_warnings = False 100 | 101 | 102 | # -- Options for HTML output ---------------------------------------------- 103 | 104 | # The theme to use for HTML and HTML Help pages. See the documentation for 105 | # a list of builtin themes. 106 | html_theme = 'default' 107 | 108 | # Theme options are theme-specific and customize the look and feel of a theme 109 | # further. For a list of options available for each theme, see the 110 | # documentation. 111 | #html_theme_options = {} 112 | 113 | # Add any paths that contain custom themes here, relative to this directory. 114 | #html_theme_path = [] 115 | 116 | # The name for this set of Sphinx documents. If None, it defaults to 117 | # " v documentation". 118 | #html_title = None 119 | 120 | # A shorter title for the navigation bar. Default is the same as html_title. 121 | #html_short_title = None 122 | 123 | # The name of an image file (relative to this directory) to place at the top 124 | # of the sidebar. 125 | #html_logo = None 126 | 127 | # The name of an image file (within the static path) to use as favicon of the 128 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 129 | # pixels large. 130 | #html_favicon = None 131 | 132 | # Add any paths that contain custom static files (such as style sheets) here, 133 | # relative to this directory. They are copied after the builtin static files, 134 | # so a file named "default.css" will overwrite the builtin "default.css". 135 | html_static_path = ['_static'] 136 | 137 | # Add any extra paths that contain custom files (such as robots.txt or 138 | # .htaccess) here, relative to this directory. These files are copied 139 | # directly to the root of the documentation. 140 | #html_extra_path = [] 141 | 142 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 143 | # using the given strftime format. 144 | #html_last_updated_fmt = '%b %d, %Y' 145 | 146 | # If true, SmartyPants will be used to convert quotes and dashes to 147 | # typographically correct entities. 148 | #html_use_smartypants = True 149 | 150 | # Custom sidebar templates, maps document names to template names. 151 | #html_sidebars = {} 152 | 153 | # Additional templates that should be rendered to pages, maps page names to 154 | # template names. 155 | #html_additional_pages = {} 156 | 157 | # If false, no module index is generated. 158 | #html_domain_indices = True 159 | 160 | # If false, no index is generated. 161 | #html_use_index = True 162 | 163 | # If true, the index is split into individual pages for each letter. 164 | #html_split_index = False 165 | 166 | # If true, links to the reST sources are added to the pages. 167 | #html_show_sourcelink = True 168 | 169 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 170 | #html_show_sphinx = True 171 | 172 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 173 | #html_show_copyright = True 174 | 175 | # If true, an OpenSearch description file will be output, and all pages will 176 | # contain a tag referring to it. The value of this option must be the 177 | # base URL from which the finished HTML is served. 178 | #html_use_opensearch = '' 179 | 180 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 181 | #html_file_suffix = None 182 | 183 | # Output file base name for HTML help builder. 184 | htmlhelp_basename = 'hostlistsdoc' 185 | 186 | 187 | # -- Options for LaTeX output --------------------------------------------- 188 | 189 | latex_elements = { 190 | # The paper size ('letterpaper' or 'a4paper'). 191 | #'papersize': 'letterpaper', 192 | 193 | # The font size ('10pt', '11pt' or '12pt'). 194 | #'pointsize': '10pt', 195 | 196 | # Additional stuff for the LaTeX preamble. 197 | #'preamble': '', 198 | } 199 | 200 | # Grouping the document tree into LaTeX files. List of tuples 201 | # (source start file, target name, title, 202 | # author, documentclass [howto, manual, or own class]). 203 | latex_documents = [ 204 | ('index', 'hostlists.tex', 'hostlists Documentation', 205 | 'Dwight Hubbard', 'manual'), 206 | ] 207 | 208 | # The name of an image file (relative to this directory) to place at the top of 209 | # the title page. 210 | #latex_logo = None 211 | 212 | # For "manual" documents, if this is true, then toplevel headings are parts, 213 | # not chapters. 214 | #latex_use_parts = False 215 | 216 | # If true, show page references after internal links. 217 | #latex_show_pagerefs = False 218 | 219 | # If true, show URL addresses after external links. 220 | #latex_show_urls = False 221 | 222 | # Documents to append as an appendix to all manuals. 223 | #latex_appendices = [] 224 | 225 | # If false, no module index is generated. 226 | #latex_domain_indices = True 227 | 228 | 229 | # -- Options for manual page output --------------------------------------- 230 | 231 | # One entry per manual page. List of tuples 232 | # (source start file, name, description, authors, manual section). 233 | man_pages = [ 234 | ('index', 'hostlists', 'hostlists Documentation', 235 | ['Dwight Hubbard'], 1) 236 | ] 237 | 238 | # If true, show URL addresses after external links. 239 | #man_show_urls = False 240 | 241 | 242 | # -- Options for Texinfo output ------------------------------------------- 243 | 244 | # Grouping the document tree into Texinfo files. List of tuples 245 | # (source start file, target name, title, author, 246 | # dir menu entry, description, category) 247 | texinfo_documents = [ 248 | ('index', 'hostlists', 'hostlists Documentation', 249 | 'Dwight Hubbard', 'hostlists', 'One line description of project.', 250 | 'Miscellaneous'), 251 | ] 252 | 253 | # Documents to append as an appendix to all manuals. 254 | #texinfo_appendices = [] 255 | 256 | # If false, no module index is generated. 257 | #texinfo_domain_indices = True 258 | 259 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 260 | #texinfo_show_urls = 'footnote' 261 | 262 | # If true, do not generate a @detailmenu in the "Top" node's menu. 263 | #texinfo_no_detailmenu = False 264 | 265 | 266 | # Example configuration for intersphinx: refer to the Python standard library. 267 | intersphinx_mapping = {'http://docs.python.org/': None} 268 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Add files or directories to the blacklist. They should be base names, not 11 | # paths. 12 | ignore=CVS 13 | 14 | # Add files or directories matching the regex patterns to the blacklist. The 15 | # regex matches against base names, not paths. 16 | ignore-patterns= 17 | 18 | # Pickle collected data for later comparisons. 19 | persistent=yes 20 | 21 | # List of plugins (as comma separated values of python modules names) to load, 22 | # usually to register additional checkers. 23 | # load-plugins=pylint_django 24 | 25 | # Use multiple processes to speed up Pylint. 26 | jobs=1 27 | 28 | # Allow loading of arbitrary C extensions. Extensions are imported into the 29 | # active Python interpreter and may run arbitrary code. 30 | unsafe-load-any-extension=no 31 | 32 | # A comma-separated list of package or module names from where C extensions may 33 | # be loaded. Extensions are loading into the active Python interpreter and may 34 | # run arbitrary code 35 | extension-pkg-whitelist= 36 | 37 | # Allow optimization of some AST trees. This will activate a peephole AST 38 | # optimizer, which will apply various small optimizations. For instance, it can 39 | # be used to obtain the result of joining multiple strings with the addition 40 | # operator. Joining a lot of strings can lead to a maximum recursion error in 41 | # Pylint and this flag can prevent that. It has one side effect, the resulting 42 | # AST will be different than the one from reality. This option is deprecated 43 | # and it will be removed in Pylint 2.0. 44 | optimize-ast=no 45 | 46 | 47 | [MESSAGES CONTROL] 48 | 49 | # Only show warnings with the listed confidence levels. Leave empty to show 50 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 51 | confidence=HIGH, INFERENCE, INFERENCE_FAILURE 52 | 53 | # Disable the message, report, category or checker with the given id(s). You 54 | # can either give multiple identifiers separated by comma (,) or put this 55 | # option multiple times (only on the command line, not in the configuration 56 | # file where it should appear only once).You can also use "--disable=all" to 57 | # disable everything first and then reenable specific checks. For example, if 58 | # you want to run only the similarities checker, you can use "--disable=all 59 | # --enable=similarities". If you want to run only the classes checker, but have 60 | # no Warning level messages displayed, use"--disable=all --enable=classes 61 | # --disable=W" 62 | #disable=W,C,R,E0633,E1101,E1136 63 | # Disabled Warnings Descriptions 64 | # C - 65 | # C0103 - Invalid name 66 | # C0111 - Missing Docstring 67 | # C0411 - Wrong Import Order 68 | # C0412 - Ungrouped imports 69 | # E - All errors 70 | # E0633 - Unpacking non sequence 71 | # E1101 - No member 72 | # E1136 - Unscriptable object 73 | # R - 74 | # R0101 - To many nested blocks 75 | # R0102 - Simplifiable if statement 76 | # R0201 - No self use 77 | # R0204 - Redefined variable type 78 | # R0801 - Duplicate Code 79 | # R0901 - To many ancestors 80 | # R0902 - To many instance attributes 81 | # R0903 - To few public methods 82 | # R0904 - To many public methods 83 | # R0911 - To many return statements 84 | # R0912 - To many branches 85 | # R0913 - To many arguments 86 | # R0914 - To many locals 87 | # R0915 - To many statements 88 | # W - All Warnings 89 | # W0611 - Unused import 90 | # W1401 - Anomaous backslash in string 91 | 92 | # Disabled due to false positives when used with django 93 | # E0633 X 94 | # E1101 X 95 | # E1136 X 96 | 97 | # I0011 98 | 99 | # Disabled to make lint pass, need to be enabled and fixed over time 100 | # W 101 | # C 102 | # R0201 103 | # R0204 104 | # R0801 105 | # R0901 106 | # R0902 107 | # R0903 108 | # R0904 109 | # R0912 X 110 | # R0914 X 111 | # R0915 X 112 | disable=W,C,R,I,E0633,E1101,E1136,W0621,C0103 113 | 114 | # Enable the message, report, category or checker with the given id(s). You can 115 | # either give multiple identifier separated by comma (,) or put this option 116 | # multiple time (only on the command line, not in the configuration file where 117 | # it should appear only once). See also the "--disable" option for examples. 118 | # enable= 119 | 120 | 121 | [REPORTS] 122 | 123 | # Set the output format. Available formats are text, parseable, colorized, msvs 124 | # (visual studio) and html. You can also give a reporter class, eg 125 | # mypackage.mymodule.MyReporterClass. 126 | output-format=parseable 127 | 128 | # Put messages in a separate file for each module / package specified on the 129 | # command line instead of printing them on stdout. Reports (if any) will be 130 | # written in a file name "pylint_global.[txt|html]". This option is deprecated 131 | # and it will be removed in Pylint 2.0. 132 | files-output=no 133 | 134 | # Tells whether to display a full report or only the messages 135 | reports=no 136 | 137 | # Python expression which should return a note less than 10 (10 is the highest 138 | # note). You have access to the variables errors warning, statement which 139 | # respectively contain the number of errors / warnings messages and the total 140 | # number of statements analyzed. This is used by the global evaluation report 141 | # (RP0004). 142 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 143 | 144 | # Template used to display messages. This is a python new-style format string 145 | # used to format the message information. See doc for all details 146 | #msg-template= 147 | 148 | 149 | [BASIC] 150 | 151 | # Good variable names which should always be accepted, separated by a comma 152 | good-names=i,j,k,ex,Run,_ 153 | 154 | # Bad variable names which should always be refused, separated by a comma 155 | bad-names=foo,bar,baz,toto,tutu,tata 156 | 157 | # Colon-delimited sets of names that determine each other's naming style when 158 | # the name regexes allow several styles. 159 | name-group= 160 | 161 | # Include a hint for the correct naming format with invalid-name 162 | include-naming-hint=no 163 | 164 | # List of decorators that produce properties, such as abc.abstractproperty. Add 165 | # to this list to register other decorators that produce valid properties. 166 | property-classes=abc.abstractproperty 167 | 168 | # Regular expression matching correct function names 169 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 170 | 171 | # Naming hint for function names 172 | function-name-hint=[a-z_][a-z0-9_]{2,30}$ 173 | 174 | # Regular expression matching correct variable names 175 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 176 | 177 | # Naming hint for variable names 178 | variable-name-hint=[a-z_][a-z0-9_]{2,30}$ 179 | 180 | # Regular expression matching correct constant names 181 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 182 | 183 | # Naming hint for constant names 184 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 185 | 186 | # Regular expression matching correct attribute names 187 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 188 | 189 | # Naming hint for attribute names 190 | attr-name-hint=[a-z_][a-z0-9_]{2,30}$ 191 | 192 | # Regular expression matching correct argument names 193 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 194 | 195 | # Naming hint for argument names 196 | argument-name-hint=[a-z_][a-z0-9_]{2,30}$ 197 | 198 | # Regular expression matching correct class attribute names 199 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 200 | 201 | # Naming hint for class attribute names 202 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 203 | 204 | # Regular expression matching correct inline iteration names 205 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 206 | 207 | # Naming hint for inline iteration names 208 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 209 | 210 | # Regular expression matching correct class names 211 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 212 | 213 | # Naming hint for class names 214 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 215 | 216 | # Regular expression matching correct module names 217 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 218 | 219 | # Naming hint for module names 220 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 221 | 222 | # Regular expression matching correct method names 223 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 224 | 225 | # Naming hint for method names 226 | method-name-hint=[a-z_][a-z0-9_]{2,30}$ 227 | 228 | # Regular expression which should only match function or class names that do 229 | # not require a docstring. 230 | no-docstring-rgx=^_ 231 | 232 | # Minimum line length for functions/classes that require docstrings, shorter 233 | # ones are exempt. 234 | docstring-min-length=-1 235 | 236 | 237 | [ELIF] 238 | 239 | # Maximum number of nested blocks for function / method body 240 | max-nested-blocks=6 241 | 242 | 243 | [FORMAT] 244 | 245 | # Maximum number of characters on a single line. 246 | max-line-length=512 247 | 248 | # Regexp for a line that is allowed to be longer than the limit. 249 | ignore-long-lines=^\s*(# )??$ 250 | 251 | # Allow the body of an if to be on the same line as the test if there is no 252 | # else. 253 | single-line-if-stmt=no 254 | 255 | # List of optional constructs for which whitespace checking is disabled. `dict- 256 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 257 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 258 | # `empty-line` allows space-only lines. 259 | no-space-check=trailing-comma,dict-separator 260 | 261 | # Maximum number of lines in a module 262 | max-module-lines=1000 263 | 264 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 265 | # tab). 266 | indent-string=' ' 267 | 268 | # Number of spaces of indent required inside a hanging or continued line. 269 | indent-after-paren=4 270 | 271 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 272 | expected-line-ending-format= 273 | 274 | 275 | [LOGGING] 276 | 277 | # Logging modules to check that the string format arguments are in logging 278 | # function parameter format 279 | logging-modules=logging 280 | 281 | 282 | [MISCELLANEOUS] 283 | 284 | # List of note tags to take in consideration, separated by a comma. 285 | notes=FIXME,XXX,TODO 286 | 287 | 288 | [SIMILARITIES] 289 | 290 | # Minimum lines number of a similarity. 291 | min-similarity-lines=4 292 | 293 | # Ignore comments when computing similarities. 294 | ignore-comments=yes 295 | 296 | # Ignore docstrings when computing similarities. 297 | ignore-docstrings=yes 298 | 299 | # Ignore imports when computing similarities. 300 | ignore-imports=no 301 | 302 | 303 | [SPELLING] 304 | 305 | # Spelling dictionary name. Available dictionaries: none. To make it working 306 | # install python-enchant package. 307 | spelling-dict= 308 | 309 | # List of comma separated words that should not be checked. 310 | spelling-ignore-words= 311 | 312 | # A path to a file that contains private dictionary; one word per line. 313 | spelling-private-dict-file= 314 | 315 | # Tells whether to store unknown words to indicated private dictionary in 316 | # --spelling-private-dict-file option instead of raising a message. 317 | spelling-store-unknown-words=no 318 | 319 | 320 | [TYPECHECK] 321 | 322 | # Tells whether missing members accessed in mixin class should be ignored. A 323 | # mixin class is detected if its name ends with "mixin" (case insensitive). 324 | ignore-mixin-members=yes 325 | 326 | # List of module names for which member attributes should not be checked 327 | # (useful for modules/projects where namespaces are manipulated during runtime 328 | # and thus existing member attributes cannot be deduced by static analysis. It 329 | # supports qualified module names, as well as Unix pattern matching. 330 | ignored-modules= 331 | 332 | # List of class names for which member attributes should not be checked (useful 333 | # for classes with dynamically set attributes). This supports the use of 334 | # qualified names. 335 | ignored-classes=optparse.Values,thread._local,_thread._local 336 | 337 | # List of members which are set dynamically and missed by pylint inference 338 | # system, and so shouldn't trigger E1101 when accessed. Python regular 339 | # expressions are accepted. 340 | generated-members= 341 | 342 | # List of decorators that produce context managers, such as 343 | # contextlib.contextmanager. Add to this list to register other decorators that 344 | # produce valid context managers. 345 | contextmanager-decorators=contextlib.contextmanager 346 | 347 | 348 | [VARIABLES] 349 | 350 | # Tells whether we should check for unused import in __init__ files. 351 | init-import=no 352 | 353 | # A regular expression matching the name of dummy variables (i.e. expectedly 354 | # not used). 355 | dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy 356 | 357 | # List of additional names supposed to be defined in builtins. Remember that 358 | # you should avoid to define new builtins when possible. 359 | additional-builtins= 360 | 361 | # List of strings which can identify a callback function by name. A callback 362 | # name must start or end with one of those strings. 363 | callbacks=cb_,_cb 364 | 365 | # List of qualified module names which can have objects that can redefine 366 | # builtins. 367 | redefining-builtins-modules=six.moves,future.builtins 368 | 369 | 370 | [CLASSES] 371 | 372 | # List of method names used to declare (i.e. assign) instance attributes. 373 | defining-attr-methods=__init__,__new__,setUp 374 | 375 | # List of valid names for the first argument in a class method. 376 | valid-classmethod-first-arg=cls 377 | 378 | # List of valid names for the first argument in a metaclass class method. 379 | valid-metaclass-classmethod-first-arg=mcs 380 | 381 | # List of member names, which should be excluded from the protected access 382 | # warning. 383 | exclude-protected=_asdict,_fields,_replace,_source,_make 384 | 385 | 386 | [DESIGN] 387 | 388 | # Maximum number of arguments for function / method 389 | max-args=5 390 | 391 | # Argument names that match this expression will be ignored. Default to name 392 | # with leading underscore 393 | ignored-argument-names=_.* 394 | 395 | # Maximum number of locals for function / method body 396 | max-locals=15 397 | 398 | # Maximum number of return / yield for function / method body 399 | max-returns=6 400 | 401 | # Maximum number of branch for function / method body 402 | max-branches=12 403 | 404 | # Maximum number of statements in function / method body 405 | max-statements=50 406 | 407 | # Maximum number of parents for a class (see R0901). 408 | max-parents=7 409 | 410 | # Maximum number of attributes for a class (see R0902). 411 | max-attributes=7 412 | 413 | # Minimum number of public methods for a class (see R0903). 414 | min-public-methods=2 415 | 416 | # Maximum number of public methods for a class (see R0904). 417 | max-public-methods=20 418 | 419 | # Maximum number of boolean expressions in a if statement 420 | max-bool-expr=5 421 | 422 | 423 | [IMPORTS] 424 | 425 | # Deprecated modules which should not be used, separated by a comma 426 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 427 | 428 | # Create a graph of every (i.e. internal and external) dependencies in the 429 | # given file (report RP0402 must not be disabled) 430 | import-graph= 431 | 432 | # Create a graph of external dependencies in the given file (report RP0402 must 433 | # not be disabled) 434 | ext-import-graph= 435 | 436 | # Create a graph of internal dependencies in the given file (report RP0402 must 437 | # not be disabled) 438 | int-import-graph= 439 | 440 | # Force import order to recognize a module as part of the standard 441 | # compatibility libraries. 442 | known-standard-library= 443 | 444 | # Force import order to recognize a module as part of a third party library. 445 | known-third-party=enchant 446 | 447 | # Analyse import fallback blocks. This can be used to support both Python 2 and 448 | # 3 compatible code, which means that the block might have code that exists 449 | # only in one or another interpreter, leading to false positives when analysed. 450 | analyse-fallback-blocks=no 451 | 452 | 453 | [EXCEPTIONS] 454 | 455 | # Exceptions that will emit a warning when being caught. Defaults to 456 | # "Exception" 457 | overgeneral-exceptions=Exception 458 | --------------------------------------------------------------------------------