├── .dockerignore ├── .editorconfig ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.rst ├── docker-entrypoint.sh ├── docs ├── Makefile ├── _static │ └── honeycomb.png ├── cli.rst ├── conf.py ├── full_api.rst ├── honeycomb.commands.integration.rst ├── honeycomb.commands.service.rst ├── honeycomb.decoymanager.rst ├── honeycomb.integrationmanager.rst ├── honeycomb.rst ├── honeycomb.servicemanager.rst ├── honeycomb.utils.rst ├── index.rst ├── plugin_api.rst └── using-docker.rst ├── honeycomb.yml ├── honeycomb ├── __init__.py ├── __main__.py ├── cli.py ├── commands │ ├── __init__.py │ ├── integration │ │ ├── __init__.py │ │ ├── configure.py │ │ ├── install.py │ │ ├── list.py │ │ ├── show.py │ │ ├── test.py │ │ └── uninstall.py │ └── service │ │ ├── __init__.py │ │ ├── install.py │ │ ├── list.py │ │ ├── logs.py │ │ ├── run.py │ │ ├── show.py │ │ ├── status.py │ │ ├── stop.py │ │ ├── test.py │ │ └── uninstall.py ├── decoymanager │ ├── __init__.py │ └── models.py ├── defs.py ├── error_messages.py ├── exceptions.py ├── integrationmanager │ ├── __init__.py │ ├── defs.py │ ├── error_messages.py │ ├── exceptions.py │ ├── integration_utils.py │ ├── models.py │ ├── registration.py │ └── tasks.py ├── servicemanager │ ├── __init__.py │ ├── base_service.py │ ├── defs.py │ ├── error_messages.py │ ├── exceptions.py │ ├── models.py │ └── registration.py └── utils │ ├── __init__.py │ ├── config_utils.py │ ├── daemon.py │ ├── plugin_utils.py │ ├── tailer.py │ └── wait.py ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── test_honeycomb.py └── utils │ ├── __init__.py │ ├── defs.py │ ├── syslog.py │ └── test_utils.py └── tox.ini /.dockerignore: -------------------------------------------------------------------------------- 1 | # git 2 | .git 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[ocd] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | .static_storage/ 60 | .media/ 61 | local_settings.py 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # Environments 89 | .env 90 | .venv 91 | env/ 92 | venv/ 93 | venv3/ 94 | ENV/ 95 | env.bak/ 96 | venv.bak/ 97 | 98 | # Spyder project settings 99 | .spyderproject 100 | .spyproject 101 | 102 | # Rope project settings 103 | .ropeproject 104 | 105 | # mkdocs documentation 106 | /site 107 | 108 | # mypy 109 | .mypy_cache/ 110 | 111 | # IDEs 112 | .vscode 113 | 114 | # macos 115 | .DS_Store 116 | 117 | 118 | # Honeycomb 119 | .args.json 120 | *.zip 121 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | end_of_line = lf 9 | charset = utf-8 10 | trim_trailing_whitespace = true 11 | insert_final_newline = true 12 | 13 | [{*.json,.*.yml}] 14 | indent_size = 2 15 | 16 | [*.py] 17 | max_line_length = 120 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[ocd] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | .static_storage/ 57 | .media/ 58 | local_settings.py 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # Environments 86 | .env 87 | .venv 88 | env/ 89 | venv/ 90 | venv3/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | 108 | # IDEs 109 | .vscode 110 | .idea 111 | 112 | # macos 113 | .DS_Store 114 | 115 | 116 | # Honeycomb 117 | .args.json 118 | honeycomb.yml 119 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - 2.7 5 | - 3.6 6 | 7 | install: pip install -U tox-travis pip setuptools 8 | script: tox 9 | 10 | stages: 11 | - lint 12 | - test 13 | 14 | jobs: 15 | include: 16 | - stage: lint 17 | python: 3.6 18 | script: tox -e flake8 19 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Honeycomb 2 | 3 | Pull requests are welcome! Please make sure to follow coding style requirements 4 | and confirm all tests pass before opening a PR. 5 | 6 | Honeycomb uses the latest stable versions of Python2/3, this guide uses pyenv 7 | to make managing python versions easier but feel free to use whatever you like 8 | as long as you pass the tests :) 9 | 10 | ## Set up development environment 11 | 12 | Make sure you have both python 2.7 and 3.6 available as they are required for testing. 13 | 14 | ### Install pyenv 15 | See https://github.com/pyenv/pyenv#installation for full instructions on how to install pyenv. 16 | ======= 17 | Follow instructions to init pyenv and add it to your bashrc/zshrc file. 18 | 19 | ### Install python 2 and 3 20 | $ pyenv install 2.7.14 21 | $ pyenv install 3.6.3 22 | $ pyenv global 2.7.14 3.6.3 23 | 24 | 25 | ### Set up virtualenv and install honeycomb in editable mode 26 | $ git clone git@github.com:Cymmetria/honeycomb.git 27 | $ cd honeycomb 28 | $ virtualenv venv 29 | $ source venv/bin/activate 30 | $ pip install -r requirements-dev.txt # will install tox 31 | $ pip install --editable . 32 | 33 | 34 | ### Make sure tests are working 35 | $ tox 36 | 37 | This will run all the existing tests, do not start coding before you resolve any local configuration issues. 38 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | # build-base and python3-dev might be required by honeycomb plugins 4 | RUN apk add --no-cache build-base python3-dev tini bash && \ 5 | wget -qO- https://bootstrap.pypa.io/get-pip.py | python3 && \ 6 | pip install virtualenv 7 | 8 | # ensure honeycomb user exists 9 | RUN set -x && \ 10 | addgroup -g 1000 -S honeycomb && \ 11 | adduser -u 1000 -D -S -G honeycomb honeycomb 12 | 13 | # set default home and permissions 14 | ENV HC_HOME /usr/share/honeycomb 15 | RUN mkdir ${HC_HOME} && chown -vR 1000:1000 ${HC_HOME} 16 | 17 | # install honeycomb 18 | COPY requirements.txt /app/requirements.txt 19 | WORKDIR /app 20 | RUN virtualenv /app/venv && \ 21 | /app/venv/bin/pip install -r requirements.txt 22 | ENV PATH /app/venv/bin:${PATH} 23 | 24 | COPY . /app/ 25 | RUN pip install --editable . 26 | 27 | COPY docker-entrypoint.sh /docker-entrypoint.sh 28 | 29 | # fix permissions and drop privileges 30 | RUN chown 1000:1000 -R /app 31 | USER 1000 32 | 33 | ENTRYPOINT ["/docker-entrypoint.sh"] 34 | 35 | VOLUME /usr/share/honeycomb 36 | CMD ["honeycomb", "--config", "${HC_HOME}/honeycomb.yml"] 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Honeycomb 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | |GitHub license| |PyPI| |Read the Docs| |Travis| |Updates| |Codecov| |Gitter| 2 | 3 | .. |honeycomb_logo| image:: https://i.imgur.com/x9rdRlF.png 4 | :alt: Honeycomb 5 | :target: https://github.com/Cymmetria/honeycomb 6 | .. |GitHub license| image:: https://img.shields.io/github/license/Cymmetria/honeycomb.svg 7 | :alt: GitHub license 8 | :target: https://github.com/Cymmetria/honeycomb/blob/master/LICENSE 9 | .. |PyPI| image:: https://img.shields.io/pypi/v/honeycomb-framework.svg 10 | :alt: PyPI 11 | :target: https://pypi.org/project/honeycomb-framework/ 12 | .. |Read the Docs| image:: https://img.shields.io/readthedocs/honeycomb/master.svg 13 | :alt: Read the Docs 14 | :target: http://honeycomb.cymmetria.com 15 | .. |Travis| image:: https://img.shields.io/travis/Cymmetria/honeycomb.svg 16 | :alt: Travis 17 | :target: https://travis-ci.com/Cymmetria/honeycomb 18 | .. |Updates| image:: https://pyup.io/repos/github/Cymmetria/honeycomb/shield.svg 19 | :target: https://pyup.io/repos/github/Cymmetria/honeycomb/ 20 | :alt: Updates 21 | .. |Codecov| image:: https://img.shields.io/codecov/c/github/Cymmetria/honeycomb.svg 22 | :alt: Codecov 23 | :target: https://codecov.io/gh/Cymmetria/honeycomb 24 | .. |Gitter| image:: https://badges.gitter.im/cymmetria/honeycomb.svg 25 | :alt: Join the chat at https://gitter.im/cymmetria/honeycomb 26 | :target: https://gitter.im/cymmetria/honeycomb 27 | 28 | |honeycomb_logo| 29 | 30 | Honeycomb - An extensible honeypot framework 31 | ============================================ 32 | 33 | Honeycomb is an open-source honeypot framework created by Cymmetria_. 34 | 35 | .. _Cymmetria: https://cymmetria.com 36 | 37 | Honeycomb allows running honeypots with various integrations from a public library of plugins at https://github.com/Cymmetria/honeycomb_plugins 38 | 39 | Writing new honeypot services and integrations for honeycomb is super easy! 40 | See the `plugins repo `_ for more info. 41 | 42 | Full CLI documentation can be found at http://honeycomb.cymmetria.com/en/latest/cli.html 43 | 44 | Usage 45 | ----- 46 | 47 | Using pip:: 48 | 49 | $ pip install honeycomb-framework 50 | $ honeycomb --help 51 | 52 | Using Docker:: 53 | 54 | $ docker run -v honeycomb.yml:/usr/share/honeycomb/honeycomb.yml cymmetria/honeycomb 55 | -------------------------------------------------------------------------------- /docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | isCommand() { 4 | for cmd in "service" \ 5 | "intergration" 6 | do 7 | if [ -z "${cmd#"$1"}" ]; then 8 | return 0 9 | fi 10 | done 11 | 12 | return 1 13 | } 14 | 15 | # check if the first argument passed in looks like a flag 16 | if [ "${1:0:1}" = '-' ]; then 17 | set -- /sbin/tini -- honeycomb "$@" 18 | 19 | # check if the first argument passed in is honeycomb 20 | elif [ "$1" = 'honeycomb' ]; then 21 | set -- /sbin/tini -- "$@" 22 | 23 | # check if the first argument passed in matches a known command 24 | elif isCommand "$1"; then 25 | set -- /sbin/tini -- honeycomb "$@" 26 | fi 27 | 28 | exec "$@" 29 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = Honeycomb 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/_static/honeycomb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cymmetria/honeycomb/33ea91b5cf675000e4e85dd02efe580ea6e95c86/docs/_static/honeycomb.png -------------------------------------------------------------------------------- /docs/cli.rst: -------------------------------------------------------------------------------- 1 | Running Honeycomb from command line 2 | =================================== 3 | 4 | .. click:: honeycomb.cli:cli 5 | :prog: honeycomb 6 | :show-nested: 7 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Configuration file for the Sphinx documentation builder. 4 | 5 | This file does only contain a selection of the most common options. For a 6 | full list see the documentation: 7 | http://www.sphinx-doc.org/en/stable/config 8 | """ 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | import honeycomb 16 | 17 | # -- Project information ----------------------------------------------------- 18 | 19 | project = u'Honeycomb' 20 | copyright = u'2018, Cymmetria' 21 | author = u'Omer Cohen' 22 | 23 | # The short X.Y version 24 | version = honeycomb.__version__ 25 | # The full version, including alpha/beta/rc tags 26 | release = honeycomb.__version__ 27 | 28 | 29 | # -- General configuration --------------------------------------------------- 30 | 31 | # If your documentation needs a minimal Sphinx version, state it here. 32 | # 33 | # needs_sphinx = '1.0' 34 | 35 | # Add any Sphinx extension module names here, as strings. They can be 36 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 37 | # ones. 38 | extensions = [ 39 | 'sphinx.ext.autodoc', 40 | 'sphinx.ext.doctest', 41 | 'sphinx.ext.todo', 42 | 'sphinx.ext.coverage', 43 | 'sphinx.ext.viewcode', 44 | 'sphinx.ext.githubpages', 45 | 'sphinx_click.ext', 46 | ] 47 | 48 | # Add any paths that contain templates here, relative to this directory. 49 | templates_path = ['_templates'] 50 | 51 | # The suffix(es) of source filenames. 52 | # You can specify multiple suffix as a list of string: 53 | # 54 | # source_suffix = ['.rst', '.md'] 55 | source_suffix = '.rst' 56 | 57 | # The master toctree document. 58 | master_doc = 'index' 59 | 60 | # The language for content autogenerated by Sphinx. Refer to documentation 61 | # for a list of supported languages. 62 | # 63 | # This is also used if you do content translation via gettext catalogs. 64 | # Usually you set "language" from the command line for these cases. 65 | language = None 66 | 67 | # List of patterns, relative to source directory, that match files and 68 | # directories to ignore when looking for source files. 69 | # This pattern also affects html_static_path and html_extra_path . 70 | exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store'] 71 | 72 | # The name of the Pygments (syntax highlighting) style to use. 73 | pygments_style = 'sphinx' 74 | 75 | 76 | # -- Options for HTML output ------------------------------------------------- 77 | 78 | # The theme to use for HTML and HTML Help pages. See the documentation for 79 | # a list of builtin themes. 80 | # 81 | html_theme = 'alabaster' 82 | html_sidebars = { 83 | "**": [ 84 | "about.html", 85 | "navigation.html", 86 | "relations.html", 87 | "searchbox.html", 88 | "donate.html", 89 | ] 90 | } 91 | html_theme_options = { 92 | "logo": "honeycomb.png", 93 | "description": "An extensible honeypot framework", 94 | "github_user": "Cymmetria", 95 | "github_repo": "honeycomb", 96 | "fixed_sidebar": True, 97 | } 98 | 99 | # Add any paths that contain custom static files (such as style sheets) here, 100 | # relative to this directory. They are copied after the builtin static files, 101 | # so a file named "default.css" will overwrite the builtin "default.css". 102 | html_static_path = ['_static'] 103 | 104 | # Custom sidebar templates, must be a dictionary that maps document names 105 | # to template names. 106 | # 107 | # The default sidebars (for documents that don't match any pattern) are 108 | # defined by theme itself. Builtin themes are using these templates by 109 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 110 | # 'searchbox.html']``. 111 | # 112 | # html_sidebars = {} 113 | 114 | 115 | # -- Options for HTMLHelp output --------------------------------------------- 116 | 117 | # Output file base name for HTML help builder. 118 | htmlhelp_basename = 'Honeycombdoc' 119 | 120 | 121 | # -- Options for LaTeX output ------------------------------------------------ 122 | 123 | latex_elements = { 124 | # The paper size ('letterpaper' or 'a4paper'). 125 | # 126 | # 'papersize': 'letterpaper', 127 | 128 | # The font size ('10pt', '11pt' or '12pt'). 129 | # 130 | # 'pointsize': '10pt', 131 | 132 | # Additional stuff for the LaTeX preamble. 133 | # 134 | # 'preamble': '', 135 | 136 | # Latex figure (float) alignment 137 | # 138 | # 'figure_align': 'htbp', 139 | } 140 | 141 | # Grouping the document tree into LaTeX files. List of tuples 142 | # (source start file, target name, title, 143 | # author, documentclass [howto, manual, or own class]). 144 | latex_documents = [ 145 | (master_doc, 'Honeycomb.tex', u'Honeycomb Documentation', 146 | u'Cymmetria', 'manual'), 147 | ] 148 | 149 | 150 | # -- Options for manual page output ------------------------------------------ 151 | 152 | # One entry per manual page. List of tuples 153 | # (source start file, name, description, authors, manual section). 154 | man_pages = [ 155 | (master_doc, 'honeycomb', u'Honeycomb Documentation', 156 | [author], 1) 157 | ] 158 | 159 | 160 | # -- Options for Texinfo output ---------------------------------------------- 161 | 162 | # Grouping the document tree into Texinfo files. List of tuples 163 | # (source start file, target name, title, author, 164 | # dir menu entry, description, category) 165 | texinfo_documents = [ 166 | (master_doc, 'Honeycomb', u'Honeycomb Documentation', 167 | author, 'Honeycomb', 'One line description of project.', 168 | 'Miscellaneous'), 169 | ] 170 | 171 | 172 | # -- Extension configuration ------------------------------------------------- 173 | 174 | # -- Options for todo extension ---------------------------------------------- 175 | 176 | # If true, `todo` and `todoList` produce output, else they produce nothing. 177 | todo_include_todos = True 178 | 179 | autoclass_content = 'both' 180 | -------------------------------------------------------------------------------- /docs/full_api.rst: -------------------------------------------------------------------------------- 1 | Honeycomb API Reference 2 | ======================= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | honeycomb 8 | -------------------------------------------------------------------------------- /docs/honeycomb.commands.integration.rst: -------------------------------------------------------------------------------- 1 | honeycomb.commands.integration package 2 | ====================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | honeycomb.commands.integration.configure module 8 | ----------------------------------------------- 9 | 10 | .. automodule:: honeycomb.commands.integration.configure 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | honeycomb.commands.integration.install module 16 | --------------------------------------------- 17 | 18 | .. automodule:: honeycomb.commands.integration.install 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | honeycomb.commands.integration.list module 24 | ------------------------------------------ 25 | 26 | .. automodule:: honeycomb.commands.integration.list 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | honeycomb.commands.integration.show module 32 | ------------------------------------------ 33 | 34 | .. automodule:: honeycomb.commands.integration.show 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | honeycomb.commands.integration.test module 40 | ------------------------------------------ 41 | 42 | .. automodule:: honeycomb.commands.integration.test 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | honeycomb.commands.integration.uninstall module 48 | ----------------------------------------------- 49 | 50 | .. automodule:: honeycomb.commands.integration.uninstall 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | -------------------------------------------------------------------------------- /docs/honeycomb.commands.service.rst: -------------------------------------------------------------------------------- 1 | honeycomb.commands.service package 2 | ================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | honeycomb.commands.service.install module 8 | ----------------------------------------- 9 | 10 | .. automodule:: honeycomb.commands.service.install 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | honeycomb.commands.service.list module 16 | -------------------------------------- 17 | 18 | .. automodule:: honeycomb.commands.service.list 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | honeycomb.commands.service.logs module 24 | -------------------------------------- 25 | 26 | .. automodule:: honeycomb.commands.service.logs 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | honeycomb.commands.service.run module 32 | ------------------------------------- 33 | 34 | .. automodule:: honeycomb.commands.service.run 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | honeycomb.commands.service.show module 40 | -------------------------------------- 41 | 42 | .. automodule:: honeycomb.commands.service.show 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | honeycomb.commands.service.status module 48 | ---------------------------------------- 49 | 50 | .. automodule:: honeycomb.commands.service.status 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | honeycomb.commands.service.stop module 56 | -------------------------------------- 57 | 58 | .. automodule:: honeycomb.commands.service.stop 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | honeycomb.commands.service.test module 64 | -------------------------------------- 65 | 66 | .. automodule:: honeycomb.commands.service.test 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | honeycomb.commands.service.uninstall module 72 | ------------------------------------------- 73 | 74 | .. automodule:: honeycomb.commands.service.uninstall 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | -------------------------------------------------------------------------------- /docs/honeycomb.decoymanager.rst: -------------------------------------------------------------------------------- 1 | honeycomb.decoymanager package 2 | ============================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | honeycomb.decoymanager.models module 8 | ------------------------------------ 9 | 10 | .. automodule:: honeycomb.decoymanager.models 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: honeycomb.decoymanager 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/honeycomb.integrationmanager.rst: -------------------------------------------------------------------------------- 1 | honeycomb.integrationmanager package 2 | ==================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | honeycomb.integrationmanager.defs module 8 | ---------------------------------------- 9 | 10 | .. automodule:: honeycomb.integrationmanager.defs 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | honeycomb.integrationmanager.error\_messages module 16 | --------------------------------------------------- 17 | 18 | .. automodule:: honeycomb.integrationmanager.error_messages 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | honeycomb.integrationmanager.exceptions module 24 | ---------------------------------------------- 25 | 26 | .. automodule:: honeycomb.integrationmanager.exceptions 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | honeycomb.integrationmanager.integration\_utils module 32 | ------------------------------------------------------ 33 | 34 | .. automodule:: honeycomb.integrationmanager.integration_utils 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | honeycomb.integrationmanager.models module 40 | ------------------------------------------ 41 | 42 | .. automodule:: honeycomb.integrationmanager.models 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | honeycomb.integrationmanager.registration module 48 | ------------------------------------------------ 49 | 50 | .. automodule:: honeycomb.integrationmanager.registration 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | honeycomb.integrationmanager.tasks module 56 | ----------------------------------------- 57 | 58 | .. automodule:: honeycomb.integrationmanager.tasks 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | 64 | Module contents 65 | --------------- 66 | 67 | .. automodule:: honeycomb.integrationmanager 68 | :members: 69 | :undoc-members: 70 | :show-inheritance: 71 | -------------------------------------------------------------------------------- /docs/honeycomb.rst: -------------------------------------------------------------------------------- 1 | honeycomb package 2 | ================= 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | honeycomb.commands.service 10 | honeycomb.commands.integration 11 | honeycomb.decoymanager 12 | honeycomb.integrationmanager 13 | honeycomb.servicemanager 14 | honeycomb.utils 15 | 16 | Submodules 17 | ---------- 18 | 19 | honeycomb.cli module 20 | -------------------- 21 | 22 | .. automodule:: honeycomb.cli 23 | :members: 24 | :undoc-members: 25 | :show-inheritance: 26 | 27 | honeycomb.defs module 28 | --------------------- 29 | 30 | .. automodule:: honeycomb.defs 31 | :members: 32 | :undoc-members: 33 | :show-inheritance: 34 | 35 | honeycomb.error\_messages module 36 | -------------------------------- 37 | 38 | .. automodule:: honeycomb.error_messages 39 | :members: 40 | :undoc-members: 41 | :show-inheritance: 42 | 43 | honeycomb.exceptions module 44 | --------------------------- 45 | 46 | .. automodule:: honeycomb.exceptions 47 | :members: 48 | :undoc-members: 49 | :show-inheritance: 50 | -------------------------------------------------------------------------------- /docs/honeycomb.servicemanager.rst: -------------------------------------------------------------------------------- 1 | honeycomb.servicemanager package 2 | ================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | honeycomb.servicemanager.base\_service module 8 | --------------------------------------------- 9 | 10 | .. automodule:: honeycomb.servicemanager.base_service 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | honeycomb.servicemanager.defs module 16 | ------------------------------------ 17 | 18 | .. automodule:: honeycomb.servicemanager.defs 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | honeycomb.servicemanager.error\_messages module 24 | ----------------------------------------------- 25 | 26 | .. automodule:: honeycomb.servicemanager.error_messages 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | honeycomb.servicemanager.exceptions module 32 | ------------------------------------------ 33 | 34 | .. automodule:: honeycomb.servicemanager.exceptions 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | honeycomb.servicemanager.models module 40 | -------------------------------------- 41 | 42 | .. automodule:: honeycomb.servicemanager.models 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | honeycomb.servicemanager.registration module 48 | -------------------------------------------- 49 | 50 | .. automodule:: honeycomb.servicemanager.registration 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | 56 | Module contents 57 | --------------- 58 | 59 | .. automodule:: honeycomb.servicemanager 60 | :members: 61 | :undoc-members: 62 | :show-inheritance: 63 | -------------------------------------------------------------------------------- /docs/honeycomb.utils.rst: -------------------------------------------------------------------------------- 1 | honeycomb.utils package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | honeycomb.utils.config\_utils module 8 | ------------------------------------ 9 | 10 | .. automodule:: honeycomb.utils.config_utils 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | honeycomb.utils.daemon module 16 | ----------------------------- 17 | 18 | .. automodule:: honeycomb.utils.daemon 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | honeycomb.utils.plugin\_utils module 24 | ------------------------------------ 25 | 26 | .. automodule:: honeycomb.utils.plugin_utils 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | honeycomb.utils.tailer module 32 | ----------------------------- 33 | 34 | .. automodule:: honeycomb.utils.tailer 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | honeycomb.utils.wait module 40 | --------------------------- 41 | 42 | .. automodule:: honeycomb.utils.wait 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: honeycomb.utils 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | 3 | .. toctree:: 4 | :maxdepth: 3 5 | 6 | cli 7 | using-docker 8 | plugin_api 9 | full_api 10 | -------------------------------------------------------------------------------- /docs/plugin_api.rst: -------------------------------------------------------------------------------- 1 | Plugin API Reference 2 | ==================== 3 | 4 | honeycomb.servicemanager.base\_service module 5 | --------------------------------------------- 6 | 7 | .. automodule:: honeycomb.servicemanager.base_service 8 | :noindex: 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | 13 | 14 | honeycomb.integrationmanager.integration\_utils module 15 | ------------------------------------------------------ 16 | 17 | .. automodule:: honeycomb.integrationmanager.integration_utils 18 | :noindex: 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /docs/using-docker.rst: -------------------------------------------------------------------------------- 1 | Running Honeycomb in a container 2 | ================================ 3 | 4 | The rationale of container support is to allow rapid configuration and deployment so that launching honeypots would be simple and easy. 5 | 6 | Since Honeycomb is a standalone runner for services and integrations, it doesn't make sense for it to orchestrate deployment of external honeypots using docker. Instead, Honeycomb itself could be run as a container. 7 | 8 | This means the goal is to allow simple configuration that can be passed on to Honeycomb and launch services with integrations easily. 9 | 10 | To launch a Honeycomb service with a configured integration, the user needs to type in several commands to install a service, install an integration, configure that integration and finally run the service with optional parameters. 11 | 12 | This actually resembles configuring a docker environment, where the user needs to type in several commands to define volumes, networks, and finally run the desired container. 13 | 14 | A yml configuration that specifies all of the desired configurations (services, integrations, etc.) will be supplied to Honeycomb, and it will work like a state-machine to reach the desired state before finally running the service. 15 | 16 | An example Honeycomb file can be found on `github `._ 17 | 18 | .. literalinclude:: ../honeycomb.yml 19 | :linenos: 20 | -------------------------------------------------------------------------------- /honeycomb.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 1 3 | 4 | services: 5 | simple_http: 6 | parameters: 7 | port: 1234 8 | 9 | integrations: 10 | syslog: 11 | parameters: 12 | address: "127.0.0.1" 13 | port: 5514 14 | protocol: tcp 15 | -------------------------------------------------------------------------------- /honeycomb/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | version = (0, 1, 1) 7 | 8 | __version__ = ".".join(str(x) for x in version) 9 | 10 | __all__ = ["cli"] 11 | -------------------------------------------------------------------------------- /honeycomb/__main__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb main entry point. 3 | 4 | This file allows running honeycomb as a module directly without calling a method 5 | .. code-block:: bash 6 | $ python -m honeycomb --help 7 | """ 8 | 9 | from __future__ import absolute_import 10 | 11 | import sys 12 | 13 | from honeycomb.cli import cli 14 | 15 | 16 | def main(): 17 | """Provide an entry point for setup.py console_scripts.""" 18 | return sys.exit(cli()) 19 | 20 | 21 | if __name__ == "__main__": 22 | main() 23 | -------------------------------------------------------------------------------- /honeycomb/cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb Command Line Interface.""" 3 | from __future__ import absolute_import 4 | 5 | import os 6 | import ctypes 7 | import logging.config 8 | 9 | import six 10 | import click 11 | from pythonjsonlogger import jsonlogger 12 | 13 | from honeycomb import __version__ 14 | from honeycomb.defs import DEBUG_LOG_FILE, INTEGRATIONS, SERVICES 15 | from honeycomb.commands import commands_list 16 | from honeycomb.utils.config_utils import process_config 17 | 18 | 19 | CONTEXT_SETTINGS = dict( 20 | obj={}, 21 | auto_envvar_prefix="HC", # all parameters will be taken from HC_PARAMETER first 22 | max_content_width=120, 23 | help_option_names=["-h", "--help"], 24 | ) 25 | 26 | logger = logging.getLogger(__name__) 27 | 28 | 29 | @click.group(commands=commands_list, context_settings=CONTEXT_SETTINGS, invoke_without_command=True, 30 | no_args_is_help=True) 31 | @click.option("--home", "-H", default=click.get_app_dir("honeycomb"), 32 | help="Honeycomb home path", type=click.Path(), show_default=True) 33 | @click.option("--iamroot", is_flag=True, default=False, help="Force run as root (NOT RECOMMENDED!)") 34 | # TODO: --config help needs rephrasing 35 | @click.option("--config", "-c", type=click.Path(exists=True, dir_okay=False, resolve_path=True), 36 | help="Path to a honeycomb.yml file that provides instructions") 37 | @click.option("--verbose", "-v", envvar="DEBUG", is_flag=True, default=False, help="Enable verbose logging") 38 | @click.pass_context 39 | @click.version_option(version=__version__, message="Honeycomb, version %(version)s") 40 | def cli(ctx, home, iamroot, config, verbose): 41 | """Honeycomb is a honeypot framework.""" 42 | _mkhome(home) 43 | setup_logging(home, verbose) 44 | 45 | logger.debug("Honeycomb v%s", __version__, extra={"version": __version__}) 46 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 47 | extra={"command": ctx.command.name, "params": ctx.params}) 48 | 49 | try: 50 | is_admin = os.getuid() == 0 51 | except AttributeError: 52 | is_admin = ctypes.windll.shell32.IsUserAnAdmin() 53 | 54 | if is_admin: 55 | if not iamroot: 56 | raise click.ClickException("Honeycomb should not run as a privileged user, if you are just " 57 | "trying to bind to a low port try running `setcap 'cap_net_bind_service=+ep' " 58 | "$(which honeycomb)` instead. If you insist, use --iamroot") 59 | logger.warn("running as root!") 60 | 61 | ctx.obj["HOME"] = home 62 | 63 | logger.debug("ctx: {}".format(ctx.obj)) 64 | 65 | if config: 66 | return process_config(ctx, config) 67 | 68 | 69 | class MyLogger(logging.Logger): 70 | """Custom Logger.""" 71 | 72 | def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None): 73 | """Override default logger to allow overriding of internal attributes.""" 74 | # See below commented section for a simple example of what the docstring refers to 75 | if six.PY2: 76 | rv = logging.LogRecord(name, level, fn, lno, msg, args, exc_info, func) 77 | else: 78 | rv = logging.LogRecord(name, level, fn, lno, msg, args, exc_info, func, sinfo) 79 | 80 | if extra is None: 81 | extra = dict() 82 | extra.update({"pid": os.getpid(), "uid": os.getuid(), "gid": os.getgid(), "ppid": os.getppid()}) 83 | 84 | for key in extra: 85 | # if (key in ["message", "asctime"]) or (key in rv.__dict__): 86 | # raise KeyError("Attempt to overwrite %r in LogRecord" % key) 87 | rv.__dict__[key] = extra[key] 88 | return rv 89 | 90 | 91 | def setup_logging(home, verbose): 92 | """Configure logging for honeycomb.""" 93 | logging.setLoggerClass(MyLogger) 94 | logging.config.dictConfig({ 95 | "version": 1, 96 | "disable_existing_loggers": False, 97 | "formatters": { 98 | "console": { 99 | "format": "%(levelname)-8s [%(asctime)s %(name)s] %(filename)s:%(lineno)s %(funcName)s: %(message)s", 100 | }, 101 | "json": { 102 | "()": jsonlogger.JsonFormatter, 103 | "format": "%(levelname)s %(asctime)s %(name)s %(filename)s %(lineno)s %(funcName)s %(message)s", 104 | }, 105 | }, 106 | "handlers": { 107 | "default": { 108 | "level": "DEBUG" if verbose else "INFO", 109 | "class": "logging.StreamHandler", 110 | "formatter": "console", 111 | }, 112 | "file": { 113 | "level": "DEBUG", 114 | "class": "logging.handlers.WatchedFileHandler", 115 | "filename": os.path.join(home, DEBUG_LOG_FILE), 116 | "formatter": "json", 117 | }, 118 | }, 119 | "loggers": { 120 | "": { 121 | "handlers": ["default", "file"], 122 | "level": "DEBUG", 123 | "propagate": True, 124 | }, 125 | } 126 | }) 127 | 128 | 129 | def _mkhome(home): 130 | def mkdir_if_not_exists(path): 131 | try: 132 | if not os.path.exists(path): 133 | os.makedirs(path) 134 | except OSError as exc: 135 | logging.exception(exc) 136 | raise click.ClickException("Unable to create Honeycomb home dirs") 137 | 138 | home = os.path.realpath(home) 139 | for path in [home, 140 | os.path.join(home, SERVICES), 141 | os.path.join(home, INTEGRATIONS)]: 142 | mkdir_if_not_exists(path) 143 | -------------------------------------------------------------------------------- /honeycomb/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb commands.""" 3 | 4 | import os 5 | import importlib 6 | 7 | import click 8 | 9 | commands_list = {} 10 | cwd = os.path.dirname(__file__) 11 | 12 | 13 | class MyGroup(click.Group): 14 | """Dynamic group loader class.""" 15 | 16 | folder = None 17 | 18 | def __init__(self, folder, **kwargs): 19 | """Create a standard group, adding the folder parameter. 20 | 21 | :param folder: Path to folder with command .py files 22 | """ 23 | click.Group.__init__(self, **kwargs) 24 | self.folder = os.path.join(cwd, folder) 25 | self.name = folder 26 | 27 | def list_commands(self, ctx): 28 | """List commands from folder.""" 29 | rv = [] 30 | files = [_ for _ in next(os.walk(self.folder))[2] if not _.startswith("_") and _.endswith(".py")] 31 | for filename in files: 32 | rv.append(filename[:-3]) 33 | rv.sort() 34 | return rv 35 | 36 | def get_command(self, ctx, name): 37 | """Fetch command from folder.""" 38 | plugin = os.path.basename(self.folder) 39 | try: 40 | command = importlib.import_module("honeycomb.commands.{}.{}".format(plugin, name)) 41 | except ImportError: 42 | raise click.UsageError("No such command {} {}\n\n{}".format(plugin, name, self.get_help(ctx))) 43 | return getattr(command, name) 44 | 45 | 46 | for command in [_ for _ in next(os.walk(os.path.realpath(cwd)))[1]]: 47 | if command.startswith("_"): 48 | continue 49 | command_module = "honeycomb.commands.{}".format(command) 50 | commands_list[command] = MyGroup(folder=command, help=importlib.import_module(command_module).__doc__) 51 | -------------------------------------------------------------------------------- /honeycomb/commands/integration/__init__.py: -------------------------------------------------------------------------------- 1 | """Honeycomb integration commands.""" 2 | -------------------------------------------------------------------------------- /honeycomb/commands/integration/configure.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb integration run command.""" 3 | 4 | import os 5 | import json 6 | import logging 7 | 8 | import click 9 | 10 | from honeycomb import defs 11 | from honeycomb.utils import plugin_utils, config_utils 12 | from honeycomb.integrationmanager.registration import register_integration 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | @click.command(short_help="Configure an integration with default parameters") 18 | @click.pass_context 19 | @click.argument("integration") 20 | @click.argument("args", nargs=-1) 21 | @click.option("-e", "--editable", is_flag=True, default=False, 22 | help="Load integration directly from unspecified path without installing (mainly for dev)") 23 | @click.option("-a", "--show_args", is_flag=True, default=False, help="Show available integration arguments") 24 | def configure(ctx, integration, args, show_args, editable): 25 | """Configure an integration with default parameters. 26 | 27 | You can still provide one-off integration arguments to :func:`honeycomb.commands.service.run` if required. 28 | """ 29 | home = ctx.obj["HOME"] 30 | integration_path = plugin_utils.get_plugin_path(home, defs.INTEGRATIONS, integration, editable) 31 | 32 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 33 | extra={"command": ctx.command.name, "params": ctx.params}) 34 | 35 | logger.debug("loading {} ({})".format(integration, integration_path)) 36 | integration = register_integration(integration_path) 37 | 38 | if show_args: 39 | return plugin_utils.print_plugin_args(integration_path) 40 | 41 | # get our integration class instance 42 | integration_args = plugin_utils.parse_plugin_args(args, config_utils.get_config_parameters(integration_path)) 43 | 44 | args_file = os.path.join(integration_path, defs.ARGS_JSON) 45 | with open(args_file, "w") as f: 46 | data = json.dumps(integration_args) 47 | logger.debug("writing %s to %s", data, args_file) 48 | f.write(json.dumps(integration_args)) 49 | 50 | click.secho("[*] {0} has been configured, make sure to test it with `honeycomb integration test {0}`" 51 | .format(integration.name)) 52 | -------------------------------------------------------------------------------- /honeycomb/commands/integration/install.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb integration install command.""" 3 | 4 | import os 5 | import errno 6 | import logging 7 | 8 | import click 9 | 10 | from honeycomb import exceptions 11 | from honeycomb.defs import INTEGRATION, INTEGRATIONS 12 | from honeycomb.utils import plugin_utils 13 | from honeycomb.integrationmanager.registration import register_integration 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | @click.command(short_help="Install an integration") 19 | @click.pass_context 20 | @click.argument("integrations", nargs=-1) 21 | def install(ctx, integrations, delete_after_install=False): 22 | """Install a honeycomb integration from the online library, local path or zipfile.""" 23 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 24 | extra={"command": ctx.command.name, "params": ctx.params}) 25 | 26 | home = ctx.obj["HOME"] 27 | integrations_path = os.path.join(home, INTEGRATIONS) 28 | 29 | installed_all_plugins = True 30 | for integration in integrations: 31 | try: 32 | plugin_utils.install_plugin(integration, INTEGRATION, integrations_path, register_integration) 33 | except exceptions.PluginAlreadyInstalled as exc: 34 | click.echo(exc) 35 | installed_all_plugins = False 36 | 37 | if not installed_all_plugins: 38 | raise ctx.exit(errno.EEXIST) 39 | -------------------------------------------------------------------------------- /honeycomb/commands/integration/list.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb integration list command.""" 3 | 4 | import os 5 | import logging 6 | 7 | import click 8 | 9 | from honeycomb.defs import INTEGRATIONS 10 | from honeycomb.utils.plugin_utils import list_remote_plugins, list_local_plugins 11 | from honeycomb.integrationmanager.registration import register_integration 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | @click.command(short_help="List available integrations") 17 | @click.pass_context 18 | @click.option("-r", "--remote", is_flag=True, default=False, 19 | help="Include available integrations from online repository") 20 | def list(ctx, remote): 21 | """List integrations.""" 22 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 23 | extra={"command": ctx.command.name, "params": ctx.params}) 24 | 25 | click.secho("[*] Installed integrations:") 26 | home = ctx.obj["HOME"] 27 | integrations_path = os.path.join(home, INTEGRATIONS) 28 | plugin_type = "integration" 29 | 30 | def get_integration_details(integration_name): 31 | logger.debug("loading {}".format(integration_name)) 32 | integration = register_integration(os.path.join(integrations_path, integration_name)) 33 | supported_event_types = integration.supported_event_types 34 | if not supported_event_types: 35 | supported_event_types = "All" 36 | 37 | return "{:s} ({:s}) [Supported event types: {}]".format(integration.name, integration.description, 38 | supported_event_types) 39 | 40 | installed_integrations = list_local_plugins(plugin_type, integrations_path, get_integration_details) 41 | 42 | if remote: 43 | list_remote_plugins(installed_integrations, plugin_type) 44 | else: 45 | click.secho("\n[*] Try running `honeycomb integrations list -r` " 46 | "to see integrations available from our repository") 47 | -------------------------------------------------------------------------------- /honeycomb/commands/integration/show.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb integration show command.""" 3 | 4 | import os 5 | import logging 6 | 7 | import click 8 | import requests 9 | from requests.adapters import HTTPAdapter 10 | 11 | from honeycomb import defs 12 | from honeycomb.utils import plugin_utils 13 | from honeycomb.integrationmanager.defs import DISPLAY_NAME 14 | from honeycomb.integrationmanager.registration import register_integration 15 | 16 | PKG_INFO_TEMPLATE = """Name: {name} 17 | Installed: {installed} 18 | Version: {commit_revision} Updated: {commit_date} 19 | Summary: {label} 20 | Location: {location} 21 | Requires: {requirements}""" 22 | 23 | logger = logging.getLogger(__name__) 24 | 25 | 26 | @click.command(short_help="Show detailed information about a integration") 27 | @click.pass_context 28 | @click.argument("integration") 29 | @click.option("-r", "--remote", is_flag=True, default=False, help="Show information only from remote repository") 30 | def show(ctx, integration, remote): 31 | """Show detailed information about a package.""" 32 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 33 | extra={"command": ctx.command.name, "params": ctx.params}) 34 | 35 | home = ctx.obj["HOME"] 36 | integration_path = plugin_utils.get_plugin_path(home, defs.INTEGRATIONS, integration) 37 | 38 | def collect_local_info(integration, integration_path): 39 | logger.debug("loading {} from {}".format(integration, integration_path)) 40 | integration = register_integration(integration_path) 41 | try: 42 | with open(os.path.join(integration_path, "requirements.txt"), "r") as fh: 43 | info["requirements"] = " ".join(fh.readlines()) 44 | except IOError: 45 | pass 46 | info["name"] = integration.name 47 | info["label"] = integration.label 48 | info["location"] = integration_path 49 | 50 | return info 51 | 52 | def collect_remote_info(integration): 53 | rsession = requests.Session() 54 | rsession.mount("https://", HTTPAdapter(max_retries=3)) 55 | 56 | try: 57 | r = rsession.get(defs.GITHUB_RAW_URL.format(plugin_type=defs.INTEGRATIONS, 58 | plugin=integration, filename="config.json")) 59 | integration_config = r.json() 60 | info["name"] = integration_config[DISPLAY_NAME] 61 | info["label"] = integration_config[defs.DESCRIPTION] 62 | info["location"] = defs.GITHUB_URL.format(plugin_type=defs.INTEGRATIONS, plugin=info["name"]) 63 | except requests.exceptions.HTTPError as exc: 64 | logger.debug(str(exc), exc_info=True) 65 | raise click.ClickException("Cannot find package {}".format(integration)) 66 | except requests.exceptions.ConnectionError as exc: 67 | logger.debug(str(exc), exc_info=True) 68 | raise click.ClickException("Unable to reach remote repository {}".format(integration)) 69 | 70 | try: 71 | r = rsession.get(defs.GITHUB_RAW_URL.format(plugin_type=defs.INTEGRATIONS, 72 | plugin=info["name"], filename="requirements.txt")) 73 | r.raise_for_status() 74 | info["requirements"] = " ".join(r.text.split("\n")) 75 | except requests.exceptions.HTTPError as exc: 76 | logger.debug(str(exc), exc_info=True) 77 | info["requirements"] = None 78 | except requests.exceptions.ConnectionError as exc: 79 | logger.debug(str(exc), exc_info=True) 80 | raise click.ClickException("Unable to reach remote repository {}".format(integration)) 81 | 82 | return info 83 | 84 | info = {"commit_revision": "N/A", "commit_date": "N/A", "requirements": "None"} 85 | 86 | if os.path.exists(integration_path): 87 | info["installed"] = True 88 | if remote: 89 | click.secho("[*] Fetching info from online repository") 90 | info.update(collect_remote_info(integration)) 91 | else: 92 | info.update(collect_local_info(integration, integration_path)) 93 | else: 94 | logger.debug("cannot find {} locally".format(integration)) 95 | if not remote: 96 | click.secho("[*] Cannot find integration locally, checking online repository") 97 | info["installed"] = False 98 | info.update(collect_remote_info(integration)) 99 | logger.debug(info) 100 | click.secho(PKG_INFO_TEMPLATE.format(**info)) 101 | -------------------------------------------------------------------------------- /honeycomb/commands/integration/test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb integration test command.""" 3 | 4 | import os 5 | import json 6 | import logging 7 | 8 | import click 9 | 10 | from honeycomb.defs import INTEGRATIONS, ARGS_JSON 11 | from honeycomb.utils import plugin_utils 12 | from honeycomb.integrationmanager.exceptions import IntegrationTestFailed 13 | from honeycomb.integrationmanager.registration import register_integration, get_integration_module 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | @click.command(short_help="Test an integration") 19 | @click.pass_context 20 | @click.argument("integrations", nargs=-1) 21 | @click.option("-e", "--editable", is_flag=True, default=False, 22 | help="Run integration directly from specified path (main for dev)") 23 | def test(ctx, integrations, editable): 24 | """Execute the integration's internal test method to verify it's working as intended.""" 25 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 26 | extra={"command": ctx.command.name, "params": ctx.params}) 27 | 28 | home = ctx.obj["HOME"] 29 | 30 | for integration in integrations: 31 | integration_path = plugin_utils.get_plugin_path(home, INTEGRATIONS, integration, editable) 32 | 33 | logger.debug("loading {} ({})".format(integration, integration_path)) 34 | integration = register_integration(integration_path) 35 | integration_module = get_integration_module(integration_path) 36 | 37 | if not integration.test_connection_enabled: 38 | raise click.ClickException("Sorry, {} integration does not support testing.".format(integration.name)) 39 | 40 | try: 41 | with open(os.path.join(integration_path, ARGS_JSON)) as f: 42 | integration_args = json.loads(f.read()) 43 | except IOError: 44 | raise click.ClickException("Cannot load integration args, please configure it first.") 45 | logger.debug("testing integration {} with args {}".format(integration, integration_args)) 46 | click.secho("[*] Testing {} with args {}".format(integration.name, integration_args)) 47 | integration_obj = integration_module.IntegrationActionsClass(integration_args) 48 | 49 | success, response = integration_obj.test_connection(integration_args) 50 | if success: 51 | click.secho("Integration test: {}, Extra details: {}".format("OK" if success else "FAIL", response)) 52 | else: 53 | raise IntegrationTestFailed(response) 54 | -------------------------------------------------------------------------------- /honeycomb/commands/integration/uninstall.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb integration uninstall command.""" 3 | 4 | import logging 5 | 6 | import click 7 | 8 | from honeycomb.defs import INTEGRATIONS 9 | from honeycomb.utils import plugin_utils 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | @click.command(short_help="Uninstall an integration") 15 | @click.pass_context 16 | @click.option("-y", "--yes", is_flag=True, default=False, help="Don't ask for confirmation of uninstall deletions.") 17 | @click.argument("integrations", nargs=-1) 18 | def uninstall(ctx, yes, integrations): 19 | """Uninstall a integration.""" 20 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 21 | extra={"command": ctx.command.name, "params": ctx.params}) 22 | 23 | home = ctx.obj["HOME"] 24 | 25 | for integration in integrations: 26 | integration_path = plugin_utils.get_plugin_path(home, INTEGRATIONS, integration) 27 | plugin_utils.uninstall_plugin(integration_path, yes) 28 | -------------------------------------------------------------------------------- /honeycomb/commands/service/__init__.py: -------------------------------------------------------------------------------- 1 | """Honeycomb service commands.""" 2 | -------------------------------------------------------------------------------- /honeycomb/commands/service/install.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service install command.""" 3 | 4 | import os 5 | import errno 6 | import logging 7 | 8 | import click 9 | 10 | from honeycomb import exceptions 11 | from honeycomb.defs import SERVICE, SERVICES 12 | from honeycomb.utils import plugin_utils 13 | from honeycomb.servicemanager.registration import register_service 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | @click.command(short_help="Install a service") 19 | @click.pass_context 20 | @click.argument("services", nargs=-1) 21 | def install(ctx, services, delete_after_install=False): 22 | """Install a honeypot service from the online library, local path or zipfile.""" 23 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 24 | extra={"command": ctx.command.name, "params": ctx.params}) 25 | 26 | home = ctx.obj["HOME"] 27 | services_path = os.path.join(home, SERVICES) 28 | 29 | installed_all_plugins = True 30 | for service in services: 31 | try: 32 | plugin_utils.install_plugin(service, SERVICE, services_path, register_service) 33 | except exceptions.PluginAlreadyInstalled as exc: 34 | click.echo(exc) 35 | installed_all_plugins = False 36 | 37 | if not installed_all_plugins: 38 | raise ctx.exit(errno.EEXIST) 39 | -------------------------------------------------------------------------------- /honeycomb/commands/service/list.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service list command.""" 3 | 4 | import os 5 | import logging 6 | 7 | import click 8 | 9 | from honeycomb.defs import SERVICES 10 | from honeycomb.utils.plugin_utils import list_remote_plugins, list_local_plugins 11 | from honeycomb.servicemanager.registration import register_service 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | @click.command(short_help="List available services") 17 | @click.pass_context 18 | @click.option("-r", "--remote", is_flag=True, default=False, 19 | help="Include available services from online repository") 20 | def list(ctx, remote): 21 | """List services.""" 22 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 23 | extra={"command": ctx.command.name, "params": ctx.params}) 24 | 25 | click.secho("[*] Installed services:") 26 | home = ctx.obj["HOME"] 27 | services_path = os.path.join(home, SERVICES) 28 | plugin_type = "service" 29 | 30 | def get_service_details(service_name): 31 | logger.debug("loading {}".format(service_name)) 32 | service = register_service(os.path.join(services_path, service_name)) 33 | if service.ports: 34 | ports = ", ".join("{}/{}".format(port["port"], port["protocol"]) for port in service.ports) 35 | else: 36 | ports = "Undefined" 37 | return "{:s} (Ports: {}) [Alerts: {}]".format(service.name, ports, 38 | ", ".join([_.name for _ in service.alert_types])) 39 | 40 | installed_services = list_local_plugins(plugin_type, services_path, get_service_details) 41 | 42 | if remote: 43 | list_remote_plugins(installed_services, plugin_type) 44 | else: 45 | click.secho("\n[*] Try running `honeycomb services list -r` " 46 | "to see services available from our repository") 47 | -------------------------------------------------------------------------------- /honeycomb/commands/service/logs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service logs command.""" 3 | 4 | import os 5 | import logging 6 | import threading 7 | 8 | import click 9 | 10 | from honeycomb.defs import SERVICES 11 | from honeycomb.utils.tailer import Tailer 12 | from honeycomb.servicemanager.defs import STDOUTLOG, LOGS_DIR 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | @click.command(short_help="Show logs for a daemonized service.") 18 | @click.option("-n", "--num", type=int, default=10, help="Number of lines to read from end of file", show_default=True) 19 | @click.option("-f", "--follow", is_flag=True, default=False, help="Follow log output") 20 | @click.argument("services", required=True, nargs=-1) 21 | @click.pass_context 22 | def logs(ctx, services, num, follow): 23 | """Show logs of daemonized service.""" 24 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 25 | extra={"command": ctx.command.name, "params": ctx.params}) 26 | 27 | home = ctx.obj["HOME"] 28 | services_path = os.path.join(home, SERVICES) 29 | 30 | tail_threads = [] 31 | for service in services: 32 | logpath = os.path.join(services_path, service, LOGS_DIR, STDOUTLOG) 33 | if os.path.exists(logpath): 34 | logger.debug("tailing %s", logpath) 35 | # TODO: Print log lines from multiple services sorted by timestamp 36 | t = threading.Thread(target=Tailer, kwargs={"name": service, 37 | "nlines": num, 38 | "filepath": logpath, 39 | "follow": follow}) 40 | t.daemon = True 41 | t.start() 42 | tail_threads.append(t) 43 | 44 | if tail_threads: 45 | while tail_threads[0].isAlive(): 46 | tail_threads[0].join(0.1) 47 | -------------------------------------------------------------------------------- /honeycomb/commands/service/run.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service run command.""" 3 | 4 | import os 5 | import json 6 | import signal 7 | import logging 8 | 9 | import click 10 | 11 | from honeycomb.defs import SERVICES, INTEGRATIONS, ARGS_JSON 12 | from honeycomb.utils import plugin_utils, config_utils 13 | from honeycomb.utils.daemon import myRunner 14 | from honeycomb.integrationmanager.tasks import configure_integration 15 | from honeycomb.servicemanager.defs import STDOUTLOG, STDERRLOG, LOGS_DIR 16 | from honeycomb.servicemanager.registration import register_service, get_service_module 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | @click.command(short_help="Load and run a specific service") 22 | @click.pass_context 23 | @click.argument("service", nargs=1) 24 | @click.argument("args", nargs=-1) 25 | @click.option("-d", "--daemon", is_flag=True, default=False, help="Run service in daemon mode") 26 | @click.option("-e", "--editable", is_flag=True, default=False, 27 | help="Load service directly from specified path without installing (mainly for dev)") 28 | @click.option("-a", "--show-args", is_flag=True, default=False, help="Show available service arguments") 29 | @click.option("-i", "--integration", multiple=True, help="Enable an integration") 30 | def run(ctx, service, args, show_args, daemon, editable, integration): 31 | """Load and run a specific service.""" 32 | home = ctx.obj["HOME"] 33 | service_path = plugin_utils.get_plugin_path(home, SERVICES, service, editable) 34 | service_log_path = os.path.join(service_path, LOGS_DIR) 35 | 36 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 37 | extra={"command": ctx.command.name, "params": ctx.params}) 38 | 39 | logger.debug("loading {} ({})".format(service, service_path)) 40 | service = register_service(service_path) 41 | 42 | if show_args: 43 | return plugin_utils.print_plugin_args(service_path) 44 | 45 | # get our service class instance 46 | service_module = get_service_module(service_path) 47 | service_args = plugin_utils.parse_plugin_args(args, config_utils.get_config_parameters(service_path)) 48 | service_obj = service_module.service_class(alert_types=service.alert_types, service_args=service_args) 49 | 50 | if not os.path.exists(service_log_path): 51 | os.mkdir(service_log_path) 52 | 53 | # prepare runner 54 | if daemon: 55 | runner = myRunner(service_obj, 56 | pidfile=service_path + ".pid", 57 | stdout=open(os.path.join(service_log_path, STDOUTLOG), "ab"), 58 | stderr=open(os.path.join(service_log_path, STDERRLOG), "ab")) 59 | 60 | files_preserve = [] 61 | for handler in logging.getLogger().handlers: 62 | if hasattr(handler, "stream"): 63 | if hasattr(handler.stream, "fileno"): 64 | files_preserve.append(handler.stream.fileno()) 65 | if hasattr(handler, "socket"): 66 | files_preserve.append(handler.socket.fileno()) 67 | 68 | runner.daemon_context.files_preserve = files_preserve 69 | runner.daemon_context.signal_map.update({ 70 | signal.SIGTERM: service_obj._on_server_shutdown, 71 | signal.SIGINT: service_obj._on_server_shutdown, 72 | }) 73 | logger.debug("daemon_context", extra={"daemon_context": vars(runner.daemon_context)}) 74 | 75 | for integration_name in integration: 76 | integration_path = plugin_utils.get_plugin_path(home, INTEGRATIONS, integration_name, editable) 77 | configure_integration(integration_path) 78 | 79 | click.secho("[+] Launching {} {}".format(service.name, "in daemon mode" if daemon else "")) 80 | try: 81 | # save service_args for external reference (see test) 82 | with open(os.path.join(service_path, ARGS_JSON), "w") as f: 83 | f.write(json.dumps(service_args)) 84 | runner._start() if daemon else service_obj.run() 85 | except KeyboardInterrupt: 86 | service_obj._on_server_shutdown() 87 | 88 | click.secho("[*] {} has stopped".format(service.name)) 89 | -------------------------------------------------------------------------------- /honeycomb/commands/service/show.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service show command.""" 3 | 4 | import os 5 | import logging 6 | 7 | import click 8 | import requests 9 | from requests.adapters import HTTPAdapter 10 | 11 | from honeycomb import defs 12 | from honeycomb.utils import plugin_utils 13 | from honeycomb.servicemanager.registration import register_service 14 | 15 | PKG_INFO_TEMPLATE = """Name: {name} 16 | Installed: {installed} 17 | Version: {commit_revision} Updated: {commit_date} 18 | Summary: {label} 19 | Location: {location} 20 | Requires: {requirements}""" 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | @click.command(short_help="Show detailed information about a service") 26 | @click.pass_context 27 | @click.argument("service") 28 | @click.option("-r", "--remote", is_flag=True, default=False, help="Show information only from remote repository") 29 | def show(ctx, service, remote): 30 | """Show detailed information about a package.""" 31 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 32 | extra={"command": ctx.command.name, "params": ctx.params}) 33 | 34 | home = ctx.obj["HOME"] 35 | service_path = plugin_utils.get_plugin_path(home, defs.SERVICES, service) 36 | 37 | def collect_local_info(service, service_path): 38 | logger.debug("loading {} from {}".format(service, service_path)) 39 | service = register_service(service_path) 40 | try: 41 | with open(os.path.join(service_path, "requirements.txt"), "r") as fh: 42 | info["requirements"] = " ".join(fh.readlines()) 43 | except IOError: 44 | pass 45 | info["name"] = service.name 46 | info["label"] = service.label 47 | info["location"] = service_path 48 | 49 | return info 50 | 51 | def collect_remote_info(service): 52 | rsession = requests.Session() 53 | rsession.mount("https://", HTTPAdapter(max_retries=3)) 54 | 55 | try: 56 | r = rsession.get(defs.GITHUB_RAW_URL.format(plugin_type=defs.SERVICES, 57 | plugin=service, filename="config.json")) 58 | service_config = r.json() 59 | info["name"] = service_config[defs.SERVICE][defs.NAME] 60 | info["label"] = service_config[defs.SERVICE][defs.LABEL] 61 | info["location"] = defs.GITHUB_URL.format(plugin_type=defs.SERVICES, plugin=info["name"]) 62 | except requests.exceptions.HTTPError as exc: 63 | logger.debug(str(exc), exc_info=True) 64 | raise click.ClickException("Cannot find package {}".format(service)) 65 | except requests.exceptions.ConnectionError as exc: 66 | logger.debug(str(exc), exc_info=True) 67 | raise click.ClickException("Unable to reach remote repository {}".format(service)) 68 | 69 | try: 70 | r = rsession.get(defs.GITHUB_RAW_URL.format(plugin_type=defs.SERVICES, plugin=info["name"], 71 | filename="requirements.txt")) 72 | r.raise_for_status() 73 | info["requirements"] = " ".join(r.text.split("\n")) 74 | except requests.exceptions.HTTPError as exc: 75 | logger.debug(str(exc), exc_info=True) 76 | info["requirements"] = None 77 | except requests.exceptions.ConnectionError as exc: 78 | logger.debug(str(exc), exc_info=True) 79 | raise click.ClickException("Unable to reach remote repository {}".format(service)) 80 | 81 | return info 82 | 83 | info = {"commit_revision": "N/A", "commit_date": "N/A", "requirements": "None"} 84 | 85 | if os.path.exists(service_path): 86 | info["installed"] = True 87 | if remote: 88 | click.secho("[*] Fetching info from online repository") 89 | info.update(collect_remote_info(service)) 90 | else: 91 | info.update(collect_local_info(service, service_path)) 92 | else: 93 | logger.debug("cannot find {} locally".format(service)) 94 | if not remote: 95 | click.secho("[*] Cannot find service locally, checking online repository") 96 | info["installed"] = False 97 | info.update(collect_remote_info(service)) 98 | logger.debug(info) 99 | click.secho(PKG_INFO_TEMPLATE.format(**info)) 100 | -------------------------------------------------------------------------------- /honeycomb/commands/service/status.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service status command.""" 3 | 4 | import os 5 | import logging 6 | 7 | import click 8 | 9 | from honeycomb.defs import SERVICES 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | @click.command(short_help="Shows status of installed service(s)") 15 | @click.pass_context 16 | @click.argument("services", nargs=-1) 17 | @click.option("-a", "--show-all", is_flag=True, default=False, help="Show status for all services") 18 | def status(ctx, services, show_all): 19 | """Show status of installed service(s).""" 20 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 21 | extra={"command": ctx.command.name, "params": ctx.params}) 22 | 23 | home = ctx.obj["HOME"] 24 | services_path = os.path.join(home, SERVICES) 25 | 26 | def print_status(service): 27 | service_dir = os.path.join(services_path, service) 28 | if os.path.exists(service_dir): 29 | pidfile = service_dir + ".pid" 30 | if os.path.exists(pidfile): 31 | try: 32 | with open(pidfile) as fh: 33 | pid = int(fh.read().strip()) 34 | os.kill(pid, 0) 35 | status = "running (pid: {})".format(pid) 36 | except OSError: 37 | status = "not running (stale pidfile)" 38 | else: 39 | status = "not running" 40 | else: 41 | status = "no such service" 42 | click.secho("{} - {}".format(service, status)) 43 | 44 | if show_all: 45 | for service in next(os.walk(services_path))[1]: 46 | print_status(service) 47 | elif services: 48 | for service in services: 49 | print_status(service) 50 | else: 51 | raise click.UsageError("You must specify a service name or use --show-all") 52 | -------------------------------------------------------------------------------- /honeycomb/commands/service/stop.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service stop command.""" 3 | 4 | import os 5 | import json 6 | import logging 7 | 8 | import click 9 | import daemon.runner 10 | 11 | from honeycomb.defs import SERVICES, ARGS_JSON 12 | from honeycomb.utils import plugin_utils 13 | from honeycomb.utils.daemon import myRunner 14 | from honeycomb.servicemanager.registration import get_service_module, register_service 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | @click.command(short_help="Stop a running service daemon") 20 | @click.argument("service") 21 | @click.option("-e", "--editable", is_flag=True, default=False, 22 | help="Load service directly from specified path without installing (mainly for dev)") 23 | @click.pass_context 24 | def stop(ctx, service, editable): 25 | """Stop a running service daemon.""" 26 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 27 | extra={"command": ctx.command.name, "params": ctx.params}) 28 | 29 | home = ctx.obj["HOME"] 30 | service_path = plugin_utils.get_plugin_path(home, SERVICES, service, editable) 31 | 32 | logger.debug("loading {}".format(service)) 33 | service = register_service(service_path) 34 | 35 | try: 36 | with open(os.path.join(service_path, ARGS_JSON)) as f: 37 | service_args = json.loads(f.read()) 38 | except IOError as exc: 39 | logger.debug(str(exc), exc_info=True) 40 | raise click.ClickException("Cannot load service args, are you sure server is running?") 41 | 42 | # get our service class instance 43 | service_module = get_service_module(service_path) 44 | service_obj = service_module.service_class(alert_types=service.alert_types, service_args=service_args) 45 | 46 | # prepare runner 47 | runner = myRunner(service_obj, 48 | pidfile=service_path + ".pid", 49 | stdout=open(os.path.join(service_path, "stdout.log"), "ab"), 50 | stderr=open(os.path.join(service_path, "stderr.log"), "ab")) 51 | 52 | click.secho("[*] Stopping {}".format(service.name)) 53 | try: 54 | runner._stop() 55 | except daemon.runner.DaemonRunnerStopFailureError as exc: 56 | logger.debug(str(exc), exc_info=True) 57 | raise click.ClickException("Unable to stop service, are you sure it is running?") 58 | -------------------------------------------------------------------------------- /honeycomb/commands/service/test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service test command.""" 3 | 4 | import os 5 | import json 6 | import socket 7 | import logging 8 | 9 | import click 10 | 11 | from honeycomb.defs import DEBUG_LOG_FILE, SERVICES, ARGS_JSON 12 | from honeycomb.utils import plugin_utils 13 | from honeycomb.utils.wait import wait_until, search_json_log, TimeoutException 14 | from honeycomb.servicemanager.defs import EVENT_TYPE 15 | from honeycomb.servicemanager.registration import register_service, get_service_module 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | @click.command(short_help="Test a running service") 21 | @click.pass_context 22 | @click.argument("services", nargs=-1) 23 | @click.option("-f", "--force", is_flag=True, default=False, help="Do not check if service is running before testing") 24 | @click.option("-e", "--editable", is_flag=True, default=False, 25 | help="Run service directly from specified path (main for dev)") 26 | def test(ctx, services, force, editable): 27 | """Execute the service's internal test method to verify it's working as intended. 28 | 29 | If there's no such method, honeycomb will attempt to connect to the port listed in config.json 30 | """ 31 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 32 | extra={"command": ctx.command.name, "params": ctx.params}) 33 | 34 | home = ctx.obj["HOME"] 35 | 36 | for service in services: 37 | service_path = plugin_utils.get_plugin_path(home, SERVICES, service, editable) 38 | 39 | logger.debug("loading {} ({})".format(service, service_path)) 40 | service = register_service(service_path) 41 | service_module = get_service_module(service_path) 42 | 43 | if not force: 44 | if os.path.exists(service_path): 45 | pidfile = service_path + ".pid" 46 | if os.path.exists(pidfile): 47 | try: 48 | with open(pidfile) as fh: 49 | pid = int(fh.read().strip()) 50 | os.kill(pid, 0) 51 | logger.debug("service is running (pid: {})".format(pid)) 52 | except OSError: 53 | logger.debug("service is not running (stale pidfile, pid: {})".format(pid), exc_info=True) 54 | raise click.ClickException("Unable to test {} because it is not running".format(service.name)) 55 | else: 56 | logger.debug("service is not running (no pidfile)") 57 | raise click.ClickException("Unable to test {} because it is not running".format(service.name)) 58 | 59 | try: 60 | with open(os.path.join(service_path, ARGS_JSON)) as f: 61 | service_args = json.loads(f.read()) 62 | except IOError as exc: 63 | logger.debug(str(exc), exc_info=True) 64 | raise click.ClickException("Cannot load service args, are you sure server is running?") 65 | logger.debug("loading service {} with args {}".format(service, service_args)) 66 | service_obj = service_module.service_class(alert_types=service.alert_types, service_args=service_args) 67 | logger.debug("loaded service {}".format(service_obj)) 68 | 69 | if hasattr(service_obj, "test"): 70 | click.secho("[+] Executing internal test method for service..") 71 | logger.debug("executing internal test method for service") 72 | event_types = service_obj.test() 73 | for event_type in event_types: 74 | try: 75 | wait_until(search_json_log, filepath=os.path.join(home, DEBUG_LOG_FILE), 76 | total_timeout=10, key=EVENT_TYPE, value=event_type) 77 | except TimeoutException: 78 | raise click.ClickException("failed to test alert: {}".format(event_type)) 79 | 80 | click.secho("{} alert tested successfully".format(event_type)) 81 | 82 | elif hasattr(service, "ports") and len(service.ports) > 0: 83 | click.secho("[+] No internal test method found, only testing ports are open") 84 | logger.debug("no internal test method found, testing ports: {}".format(service.ports)) 85 | for port in service.ports: 86 | socktype = socket.SOCK_DGRAM if port["protocol"] == "udp" else socket.SOCK_STREAM 87 | s = socket.socket(socket.AF_INET, socktype) 88 | try: 89 | s.connect(("127.0.0.1", port["port"])) 90 | s.shutdown(2) 91 | except Exception as exc: 92 | logger.debug(str(exc), exc_info=True) 93 | raise click.ClickException("Unable to connect to service port {}".format(port["port"])) 94 | -------------------------------------------------------------------------------- /honeycomb/commands/service/uninstall.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service uninstall command.""" 3 | 4 | import logging 5 | 6 | import click 7 | 8 | from honeycomb.defs import SERVICES 9 | from honeycomb.utils import plugin_utils 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | @click.command(short_help="Uninstall a service") 15 | @click.pass_context 16 | @click.option("-y", "--yes", is_flag=True, default=False, help="Don't ask for confirmation of uninstall deletions.") 17 | @click.argument("services", nargs=-1) 18 | def uninstall(ctx, yes, services): 19 | """Uninstall a service.""" 20 | logger.debug("running command %s (%s)", ctx.command.name, ctx.params, 21 | extra={"command": ctx.command.name, "params": ctx.params}) 22 | 23 | home = ctx.obj["HOME"] 24 | 25 | for service in services: 26 | service_path = plugin_utils.get_plugin_path(home, SERVICES, service) 27 | plugin_utils.uninstall_plugin(service_path, yes) 28 | -------------------------------------------------------------------------------- /honeycomb/decoymanager/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb Decoy Manager.""" 3 | from __future__ import unicode_literals, absolute_import 4 | -------------------------------------------------------------------------------- /honeycomb/decoymanager/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb defs and constants.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import platform 7 | from uuid import uuid4 8 | 9 | from attr import attrs, attrib, validators, Factory 10 | from datetime import datetime 11 | 12 | from honeycomb.servicemanager.models import ServiceType 13 | 14 | 15 | @attrs(slots=True) 16 | class AlertType(object): 17 | """Alert Type.""" 18 | 19 | name = attrib(type=str) 20 | label = attrib(type=str) 21 | service_type = attrib(type=ServiceType) 22 | 23 | 24 | @attrs(slots=True) 25 | class Alert(object): 26 | """Alert object.""" 27 | 28 | STATUS_IGNORED = 0 29 | STATUS_MUTED = 1 30 | STATUS_ALERT = 2 31 | ALERT_STATUS = ( 32 | (STATUS_IGNORED, "Ignore"), # == Generate alert in logs but don't send to integrations 33 | (STATUS_MUTED, "Mute"), # Generate alert but send only to integrations that accept muted alerts 34 | (STATUS_ALERT, "Alert") # Generate alert and send to all integrations 35 | ) 36 | 37 | alert_type = attrib(type=AlertType) 38 | 39 | id = attrib(type=str, default=Factory(uuid4)) 40 | status = attrib(type=int, default=STATUS_ALERT, validator=validators.in_([_[0] for _ in ALERT_STATUS])) 41 | timestamp = attrib(type=datetime, default=Factory(datetime.now)) 42 | 43 | event_type = attrib(init=False, type=str) 44 | manufacturer = attrib(init=False, type=str) 45 | event_description = attrib(init=False, type=str, default=Factory(lambda self: self.alert_type.label, 46 | takes_self=True)) 47 | 48 | request = attrib(init=False, type=str) 49 | dest_ip = attrib(init=False) 50 | dest_port = attrib(init=False) 51 | file_accessed = attrib(init=False) 52 | originating_ip = attrib(init=False) 53 | originating_port = attrib(init=False) 54 | transport_protocol = attrib(init=False) 55 | originating_hostname = attrib(init=False) 56 | originating_mac_address = attrib(init=False) 57 | 58 | domain = attrib(init=False) 59 | username = attrib(init=False) 60 | password = attrib(init=False) 61 | image_md5 = attrib(init=False) 62 | image_path = attrib(init=False) 63 | image_file = attrib(init=False) 64 | image_sha256 = attrib(init=False) 65 | 66 | cmd = attrib(init=False) 67 | pid = attrib(init=False) 68 | uid = attrib(init=False) 69 | ppid = attrib(init=False) 70 | address = attrib(init=False) 71 | end_timestamp = attrib(init=False) 72 | 73 | # decoy (service) fields: 74 | decoy_os = attrib(init=False, default=Factory(platform.system)) 75 | decoy_ipv4 = attrib(init=False) 76 | decoy_name = attrib(init=False) 77 | decoy_hostname = attrib(init=False) 78 | 79 | # Extra fields: 80 | additional_fields = attrib(init=False) 81 | -------------------------------------------------------------------------------- /honeycomb/defs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb defs and constants.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import six 7 | from attr import attrs, attrib 8 | 9 | 10 | @attrs 11 | class BaseNameLabel(object): 12 | """Generic name/label class.""" 13 | 14 | name = attrib() 15 | label = attrib() 16 | 17 | 18 | @attrs 19 | class IBaseType(object): 20 | """Abstract type interface, provides BaseNameLabel collection methods.""" 21 | 22 | @classmethod 23 | def all_names(cls): 24 | """Return list of all property names.""" 25 | return [v.name for (k, v) in six.iteritems(cls.__dict__) if isinstance(v, BaseNameLabel)] 26 | 27 | @classmethod 28 | def all_labels(cls): 29 | """Return list of all property labels.""" 30 | return [v.label for (k, v) in six.iteritems(cls.__dict__) if isinstance(v, BaseNameLabel)] 31 | 32 | 33 | @attrs 34 | class BaseCollection(object): 35 | """Abstract type collection mixin, should hold BaseNameLabel attributes.""" 36 | 37 | 38 | @attrs 39 | class ConfigField(object): 40 | """Config Validator. 41 | 42 | error_message is also a function to calculate the error when we ran the validator_func 43 | """ 44 | 45 | validator_func = attrib() 46 | get_error_message = attrib() 47 | 48 | 49 | DEPS_DIR = "venv" 50 | DEBUG_LOG_FILE = "honeycomb.debug.log" 51 | 52 | SERVICE = "service" 53 | SERVICES = "{}s".format(SERVICE) 54 | INTEGRATION = "integration" 55 | INTEGRATIONS = "{}s".format(INTEGRATION) 56 | 57 | GITHUB_URL = "https://github.com/Cymmetria/honeycomb_plugins/tree/master/{plugin_type}/{plugin}" 58 | GITHUB_RAW = "https://cymmetria.github.io/honeycomb_plugins" 59 | GITHUB_RAW_URL = "https://raw.githubusercontent.com/Cymmetria/honeycomb_plugins/master/" \ 60 | "{plugin_type}/{plugin}/{filename}" 61 | 62 | 63 | """Config constants.""" 64 | NAME = "name" 65 | LABEL = "label" 66 | DESCRIPTION = "description" 67 | CONFIG_FILE_NAME = "config.json" 68 | 69 | 70 | """Parameters constants.""" 71 | PARAMETERS = "parameters" 72 | 73 | ARGS_JSON = ".args.json" 74 | 75 | MIN = "min" 76 | MAX = "max" 77 | TYPE = "type" 78 | VALUE = "value" 79 | ITEMS = "items" 80 | DEFAULT = "default" 81 | REQUIRED = "required" 82 | HELP_TEXT = "help_text" 83 | VALIDATOR = "validator" 84 | FIELD_LABEL = LABEL 85 | 86 | TEXT_TYPE = "text" 87 | STRING_TYPE = "string" 88 | SELECT_TYPE = "select" 89 | BOOLEAN_TYPE = "boolean" 90 | INTEGER_TYPE = "integer" 91 | PASSWORD_TYPE = "password" 92 | FILE_TYPE = "file" 93 | -------------------------------------------------------------------------------- /honeycomb/error_messages.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb generic error messages.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | from honeycomb.defs import CONFIG_FILE_NAME 7 | 8 | MISSING_FILE_ERROR = "Missing file {}" 9 | PATH_NOT_FOUND_ERROR = "Cannot find path {}" 10 | MALFORMED_CONFIG_FILE = "{} is not a valid json file".format(CONFIG_FILE_NAME) 11 | CUSTOM_MESSAGE_ERROR_VALIDATION = "Failed to import config. error in field {} with value {}: {}" 12 | FIELD_MISSING = "field {} is missing from config file" 13 | 14 | CONFIG_FIELD_TYPE_ERROR = "Config error: '{}' is not a valid {}" 15 | PARAMETERS_FIELD_ERROR = "Parameters: '{}' is not a valid {}" 16 | PARAMETERS_DEFAULT_DOESNT_MATCH_TYPE = "Parameters: Bad value for {}={} (must be {})" 17 | PARAMETERS_REQUIRED_FIELD_MISSING = "Parameters: '{}' is missing (use --show_args to see all parameters)" 18 | 19 | PLUGIN_ALREADY_INSTALLED = "{} is already installed" 20 | PLUGIN_NOT_FOUND_IN_ONLINE_REPO = "Cannot find {} in online repository" 21 | PLUGIN_REPO_CONNECTION_ERROR = "Unable to access online repository (check debug logs for detailed info)" 22 | -------------------------------------------------------------------------------- /honeycomb/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb Exceptions.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import os 7 | import sys 8 | import logging 9 | import traceback 10 | 11 | import click 12 | 13 | from honeycomb import error_messages 14 | 15 | 16 | class BaseHoneycombException(click.ClickException): 17 | """Base Exception.""" 18 | 19 | msg_format = None 20 | 21 | def __init__(self, *args, **kwargs): 22 | """Raise ClickException and log msg with relevant debugging info from the frame that raised the exception.""" 23 | try: 24 | raise ZeroDivisionError 25 | except ZeroDivisionError: 26 | exception_frame = sys.exc_info()[2].tb_frame.f_back.f_back 27 | 28 | exception_stack = traceback.extract_stack(exception_frame, limit=1)[0] 29 | filename, lineno, funcName, tb_msg = exception_stack 30 | 31 | extra = {"filename": os.path.basename(filename), "lineno": lineno, "funcName": funcName} 32 | msg = self.msg_format.format(*args) 33 | logging.getLogger(__name__).debug(msg, extra=extra) 34 | if kwargs.get("exc_info") or os.environ.get("DEBUG", False): 35 | traceback.print_stack(exception_frame) 36 | super(BaseHoneycombException, self).__init__(click.style("[-] {}".format(msg), fg="red")) 37 | 38 | 39 | class PathNotFound(BaseHoneycombException): 40 | """Specified path was not found.""" 41 | 42 | msg_format = error_messages.PATH_NOT_FOUND_ERROR 43 | 44 | 45 | class PluginError(BaseHoneycombException): 46 | """Base Plugin Exception.""" 47 | 48 | 49 | class ConfigFileNotFound(PluginError): 50 | """Config file not found.""" 51 | 52 | msg_format = error_messages.MISSING_FILE_ERROR 53 | 54 | 55 | class RequiredFieldMissing(PluginError): 56 | """Required parameter is missing.""" 57 | 58 | msg_format = error_messages.PARAMETERS_REQUIRED_FIELD_MISSING 59 | 60 | 61 | class PluginAlreadyInstalled(PluginError): 62 | """Plugin already installed.""" 63 | 64 | msg_format = error_messages.PLUGIN_ALREADY_INSTALLED 65 | 66 | 67 | class PluginNotFoundInOnlineRepo(PluginError): 68 | """Plugin not found in online repo.""" 69 | 70 | msg_format = error_messages.PLUGIN_NOT_FOUND_IN_ONLINE_REPO 71 | 72 | 73 | class PluginRepoConnectionError(PluginError): 74 | """Connection error when trying to connect to plugin repo.""" 75 | 76 | msg_format = error_messages.PLUGIN_REPO_CONNECTION_ERROR 77 | 78 | 79 | class ConfigValidationError(BaseHoneycombException): 80 | """Base config validation error.""" 81 | 82 | 83 | class ConfigFieldMissing(ConfigValidationError): 84 | """Field is missing from config file.""" 85 | 86 | msg_format = error_messages.FIELD_MISSING 87 | 88 | 89 | class ConfigFieldTypeMismatch(ConfigValidationError): 90 | """Config field does not match specified type.""" 91 | 92 | msg_format = error_messages.PARAMETERS_DEFAULT_DOESNT_MATCH_TYPE 93 | 94 | 95 | class ConfigFieldValidationError(ConfigValidationError): 96 | """Error validating config field.""" 97 | 98 | msg_format = error_messages.CUSTOM_MESSAGE_ERROR_VALIDATION 99 | 100 | 101 | class ParametersFieldError(ConfigValidationError): 102 | """Error validating parameter.""" 103 | 104 | msg_format = error_messages.PARAMETERS_FIELD_ERROR 105 | -------------------------------------------------------------------------------- /honeycomb/integrationmanager/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb Output Manager.""" 3 | from __future__ import unicode_literals, absolute_import 4 | -------------------------------------------------------------------------------- /honeycomb/integrationmanager/defs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb integrations definitions and constants.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import six 7 | 8 | from honeycomb import defs 9 | from honeycomb.utils import config_utils 10 | from honeycomb.error_messages import CONFIG_FIELD_TYPE_ERROR 11 | 12 | ACTIONS_FILE_NAME = "integration.py" 13 | 14 | SEND_ALERT_DATA_INTERVAL = 5 15 | MAX_SEND_RETRIES = 5 16 | 17 | SUPPORTED_FIELD_TYPES = [defs.PASSWORD_TYPE, defs.BOOLEAN_TYPE, defs.INTEGER_TYPE, defs.STRING_TYPE, defs.SELECT_TYPE] 18 | 19 | DISPLAY_NAME = "display_name" 20 | INTEGRATION_TYPE = "integration_type" 21 | REQUIRED_FIELDS = "required_fields" 22 | MAX_SEND_RETRIES = "max_send_retries" 23 | POLLING_ENABLED = "polling_enabled" 24 | POLLING_DURATION = "polling_duration" 25 | SUPPORTED_EVENT_TYPES = "supported_event_types" 26 | TEST_CONNECTION_ENABLED = "test_connection_enabled" 27 | 28 | INTEGRATION_FIELDS_TO_CREATE_OBJECT = [DISPLAY_NAME, defs.DESCRIPTION, MAX_SEND_RETRIES, 29 | defs.PARAMETERS, INTEGRATION_TYPE, REQUIRED_FIELDS, 30 | POLLING_ENABLED, SUPPORTED_EVENT_TYPES, TEST_CONNECTION_ENABLED] 31 | 32 | INTEGRATION_PARAMETERS_ALLOWED_KEYS = [defs.VALUE, defs.LABEL, defs.DEFAULT, defs.TYPE, 33 | defs.HELP_TEXT, defs.REQUIRED, defs.MIN, defs.MAX, defs.VALIDATOR, defs.ITEMS] 34 | 35 | INTEGRATION_PARAMETERS_ALLOWED_TYPES = [defs.STRING_TYPE, defs.INTEGER_TYPE, defs.BOOLEAN_TYPE, defs.SELECT_TYPE] 36 | 37 | 38 | class IntegrationTypes(defs.IBaseType): 39 | """Integration types. 40 | 41 | Currently only output event is supported. 42 | """ 43 | 44 | EVENT_OUTPUT = defs.BaseNameLabel("event_output", "Event output") 45 | 46 | 47 | class IntegrationAlertStatuses(defs.IBaseType): 48 | """Provides information about the alert status in queue.""" 49 | 50 | PENDING = defs.BaseNameLabel("pending", "Pending") 51 | POLLING = defs.BaseNameLabel("polling", "Polling") 52 | IN_POLLING = defs.BaseNameLabel("in_polling", "Polling") 53 | DONE = defs.BaseNameLabel("done", "Done") 54 | ERROR_MISSING_SEND_FIELDS = defs.BaseNameLabel("error_missing", "Error. Missing required alert data.") 55 | ERROR_SENDING = defs.BaseNameLabel("error_sending", "Error sending") 56 | ERROR_POLLING = defs.BaseNameLabel("error_polling", "Error polling") 57 | ERROR_SENDING_FORMATTING = defs.BaseNameLabel("error_sending_formatting", 58 | "Error sending. Result format not recognized.") 59 | ERROR_POLLING_FORMATTING = defs.BaseNameLabel("error_polling_formatting", 60 | "Error polling. Result format not recognized.") 61 | 62 | 63 | VALID_INTEGRATION_ALERT_OUTPUT_STATUSES = [IntegrationAlertStatuses.POLLING.name, 64 | IntegrationAlertStatuses.DONE.name, 65 | IntegrationAlertStatuses.ERROR_POLLING.name, 66 | IntegrationAlertStatuses.ERROR_POLLING_FORMATTING.name] 67 | 68 | INTEGRATION_VALIDATE_CONFIG_FIELDS = { 69 | DISPLAY_NAME: config_utils.config_field_type(DISPLAY_NAME, six.string_types), 70 | INTEGRATION_TYPE: defs.ConfigField( 71 | lambda value: value in IntegrationTypes.all_names(), 72 | lambda: "Invalid {} must be one of: {}".format(INTEGRATION_TYPE, IntegrationTypes.all_names()) 73 | ), 74 | SUPPORTED_EVENT_TYPES: defs.ConfigField( 75 | lambda event_types: (isinstance(event_types, list) and 76 | all([isinstance(_, six.string_types) for _ in event_types])), 77 | lambda: CONFIG_FIELD_TYPE_ERROR.format(SUPPORTED_EVENT_TYPES, "list of strings") 78 | ), 79 | REQUIRED_FIELDS: defs.ConfigField( 80 | lambda required_fields: (isinstance(required_fields, list) and 81 | all([isinstance(_, six.string_types) for _ in required_fields])), 82 | lambda: CONFIG_FIELD_TYPE_ERROR.format(SUPPORTED_EVENT_TYPES, "list of strings") 83 | ), 84 | MAX_SEND_RETRIES: config_utils.config_field_type(MAX_SEND_RETRIES, int), 85 | POLLING_ENABLED: config_utils.config_field_type(POLLING_ENABLED, bool), 86 | defs.PARAMETERS: config_utils.config_field_type(defs.PARAMETERS, list), 87 | } 88 | -------------------------------------------------------------------------------- /honeycomb/integrationmanager/error_messages.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb integration error messages.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | from honeycomb.integrationmanager.defs import ACTIONS_FILE_NAME 7 | 8 | REQUIRED_FIELD = "This field may not be blank" 9 | 10 | TEST_CONNECTION_REQUIRED = "This field is required for the connection test" 11 | TEST_CONNECTION_NOT_SUPPORTED = "Test connection is not supported" 12 | TEST_CONNECTION_GENERAL_ERROR = "An error occurred while testing connection" 13 | 14 | INVALID_INTEGRATIONS_ACTIONS_FILE = "Invalid {} file".format(ACTIONS_FILE_NAME) 15 | INVALID_INTEGER = "Value must be an integer" 16 | 17 | ERROR_SENDING_PREFIX = "Sending alert data to '{}' failed" 18 | 19 | INTEGRATION_NOT_FOUND_ERROR = "Cannot find integration named {}, try installing it?" 20 | 21 | INTEGRATION_TEST_FAILED = "Integration test failed, details: {}" 22 | 23 | INTEGRATION_SEND_EVENT_ERROR = "Error sending integration event: {}" 24 | -------------------------------------------------------------------------------- /honeycomb/integrationmanager/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb Output Integration Exceptions.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | from honeycomb.exceptions import PluginError 7 | from honeycomb.integrationmanager import error_messages 8 | 9 | 10 | class IntegrationSendEventError(PluginError): 11 | """IntegrationSendEventError.""" 12 | 13 | msg_format = error_messages.INTEGRATION_SEND_EVENT_ERROR 14 | 15 | 16 | class IntegrationMissingRequiredFieldError(PluginError): 17 | """IntegrationMissingRequiredFieldError.""" 18 | 19 | 20 | class IntegrationPollEventError(PluginError): 21 | """IntegrationPollEventError.""" 22 | 23 | 24 | class IntegrationOutputFormatError(PluginError): 25 | """IntegrationOutputFormatError.""" 26 | 27 | 28 | class IntegrationPackageError(PluginError): 29 | """IntegrationPackageError.""" 30 | 31 | 32 | class IntegrationNoMethodImplementationError(PluginError): 33 | """IntegrationNoMethodImplementationError.""" 34 | 35 | 36 | class IntegrationNotFound(PluginError): 37 | """Integration not found.""" 38 | 39 | msg_format = error_messages.INTEGRATION_NOT_FOUND_ERROR 40 | 41 | 42 | class IntegrationTestFailed(PluginError): 43 | """Integration not found.""" 44 | 45 | msg_format = error_messages.INTEGRATION_TEST_FAILED 46 | -------------------------------------------------------------------------------- /honeycomb/integrationmanager/integration_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb Integration Manager.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import logging 7 | from abc import ABCMeta, abstractmethod 8 | 9 | from honeycomb.integrationmanager.exceptions import IntegrationNoMethodImplementationError 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class BaseIntegration(object): 15 | """Base Output Integration Class.""" 16 | 17 | __metaclass__ = ABCMeta 18 | 19 | def __init__(self, integration_data): 20 | """Use :func:`__init__` to set up any prerequisites needed before sending events, validate paramaters, etc. 21 | 22 | :param integration_data: Integration parameters 23 | :type integration_data: dict 24 | :raises IntegrationMissingRequiredFieldError: If a required field is missing. 25 | """ 26 | self.integration_data = integration_data 27 | self.logger = logging.getLogger(__name__) 28 | 29 | @abstractmethod 30 | def send_event(self, alert_dict): 31 | """Send alert event to external integration. 32 | 33 | :param alert_dict: A dictionary with all the alert fields. 34 | :rtype: tuple(dict(output_data), object(output_file)) 35 | :raises IntegrationSendEventError: If there's a problem sending the event. 36 | :raises IntegrationMissingRequiredFieldError: If a required field is missing. 37 | :return: A tuple where the first value is a dictionary with information to display in the UI, and the second is 38 | an optional file to be attached. If polling is enabled, the returned output_data will be passed to 39 | :func:`poll_for_updates`. If your integration returns nothing, you should return ({}, None). 40 | """ 41 | 42 | @abstractmethod 43 | def format_output_data(self, output_data): 44 | """Process and format the output_data returned by :func:`send_event` before display. 45 | 46 | This is currently only relevant for MazeRunner, if you don't return an output this should return output_data 47 | without change. 48 | 49 | :param output_data: As returned by :func:`send_event` 50 | :rtype: dict 51 | :return: MazeRunner compatible UI output. 52 | :raises .IntegrationOutputFormatError: If there's a problem formatting the output data. 53 | """ 54 | 55 | def test_connection(self, integration_data): 56 | """Perform a test to ensure the integration is configured correctly. 57 | 58 | This could include testing authentication or performing a test query. 59 | 60 | :param integration_data: Integration arguments. 61 | :returns: `success` 62 | :rtype: tuple(bool(success), str(response)) 63 | """ 64 | raise IntegrationNoMethodImplementationError() 65 | 66 | def poll_for_updates(self, integration_output_data): 67 | """Poll external service for updates. 68 | 69 | If service has enabled polling, this method will be called periodically and should act like :func:`send_event` 70 | 71 | :param integration_output_data: Output data returned by previous :func:`send_event` or :func:`poll_for_updates` 72 | :return: See :func:`send_event` 73 | :raises .IntegrationPollEventError: If there's a problem polling for updates. 74 | """ 75 | raise IntegrationNoMethodImplementationError() 76 | -------------------------------------------------------------------------------- /honeycomb/integrationmanager/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb integration models.""" 3 | from __future__ import unicode_literals, absolute_import 4 | 5 | 6 | from datetime import datetime, timedelta 7 | 8 | from attr import attrs, attrib, Factory 9 | 10 | from honeycomb.decoymanager.models import Alert 11 | 12 | 13 | @attrs 14 | class Integration(object): 15 | """Integration model.""" 16 | 17 | parameters = attrib(type=str) 18 | display_name = attrib(type=str) 19 | required_fields = attrib(type=list) 20 | polling_enabled = attrib(type=bool) 21 | integration_type = attrib(type=str) 22 | max_send_retries = attrib(type=int) 23 | supported_event_types = attrib(type=list) 24 | test_connection_enabled = attrib(type=bool) 25 | 26 | module = attrib(default=None) 27 | description = attrib(type=str, default=None) 28 | polling_duration = attrib(type=timedelta, default=0) 29 | 30 | # TODO: Fix schema differences between custom service and integration config.json 31 | name = attrib(type=str, init=False, default=Factory(lambda self: self.display_name.lower().replace(" ", "_"), 32 | takes_self=True)) 33 | label = attrib(type=str, init=False, default=Factory(lambda self: self.description, takes_self=True)) 34 | 35 | 36 | @attrs 37 | class ConfiguredIntegration(object): 38 | """Configured integration model.""" 39 | 40 | name = attrib(type=str) 41 | path = attrib(type=str) 42 | integration = attrib(type=Integration) 43 | 44 | data = attrib(type=str, init=False) 45 | send_muted = attrib(type=bool, default=False) 46 | created_at = attrib(type=datetime, default=Factory(datetime.now)) 47 | 48 | # status = attrib(type=str, init=False) 49 | # configuring = attrib(type=bool, default=False) 50 | 51 | 52 | @attrs 53 | class IntegrationAlert(object): 54 | """Integration alert model.""" 55 | 56 | alert = attrib(type=Alert) 57 | status = attrib(type=str) 58 | retries = attrib(type=int) 59 | configured_integration = attrib(type=ConfiguredIntegration) 60 | 61 | send_time = attrib(type=datetime, init=False) 62 | output_data = attrib(type=str, init=False) 63 | -------------------------------------------------------------------------------- /honeycomb/integrationmanager/registration.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service manager.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import os 7 | import sys 8 | import json 9 | import logging 10 | import importlib 11 | 12 | import six 13 | 14 | from honeycomb.defs import DEPS_DIR, CONFIG_FILE_NAME, INTEGRATION 15 | 16 | from honeycomb.exceptions import ConfigFileNotFound 17 | from honeycomb.utils.config_utils import validate_config, validate_config_parameters 18 | from honeycomb.integrationmanager import defs 19 | from honeycomb.integrationmanager.models import Integration 20 | from honeycomb.integrationmanager.exceptions import IntegrationNotFound 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | def get_integration_module(integration_path): 26 | """Add custom paths to sys and import integration module. 27 | 28 | :param integration_path: Path to integration folder 29 | """ 30 | # add custom paths so imports would work 31 | paths = [ 32 | os.path.join(__file__, "..", ".."), # to import integrationmanager 33 | os.path.join(integration_path, ".."), # to import integration itself 34 | os.path.join(integration_path, DEPS_DIR), # to import integration deps 35 | ] 36 | 37 | for path in paths: 38 | path = os.path.realpath(path) 39 | logger.debug("adding %s to path", path) 40 | sys.path.insert(0, path) 41 | 42 | # get our integration class instance 43 | integration_name = os.path.basename(integration_path) 44 | logger.debug("importing %s", ".".join([integration_name, INTEGRATION])) 45 | return importlib.import_module(".".join([integration_name, INTEGRATION])) 46 | 47 | 48 | def register_integration(package_folder): 49 | """Register a honeycomb integration. 50 | 51 | :param package_folder: Path to folder with integration to load 52 | :returns: Validated integration object 53 | :rtype: :func:`honeycomb.utils.defs.Integration` 54 | """ 55 | logger.debug("registering integration %s", package_folder) 56 | package_folder = os.path.realpath(package_folder) 57 | if not os.path.exists(package_folder): 58 | raise IntegrationNotFound(os.path.basename(package_folder)) 59 | 60 | json_config_path = os.path.join(package_folder, CONFIG_FILE_NAME) 61 | if not os.path.exists(json_config_path): 62 | raise ConfigFileNotFound(json_config_path) 63 | 64 | with open(json_config_path, "r") as f: 65 | config_json = json.load(f) 66 | 67 | # Validate integration and alert config 68 | validate_config(config_json, defs.INTEGRATION_VALIDATE_CONFIG_FIELDS) 69 | validate_config_parameters(config_json, 70 | defs.INTEGRATION_PARAMETERS_ALLOWED_KEYS, 71 | defs.INTEGRATION_PARAMETERS_ALLOWED_TYPES) 72 | 73 | integration_type = _create_integration_object(config_json) 74 | 75 | return integration_type 76 | 77 | 78 | def _create_integration_object(config): 79 | integration_type_create_kwargs = { 80 | key: value for key, value in six.iteritems(config) 81 | if key in defs.INTEGRATION_FIELDS_TO_CREATE_OBJECT 82 | } 83 | 84 | obj = Integration(**integration_type_create_kwargs) 85 | if config[defs.POLLING_ENABLED]: 86 | setattr(obj, defs.POLLING_DURATION, config[defs.POLLING_DURATION]) 87 | return obj 88 | -------------------------------------------------------------------------------- /honeycomb/integrationmanager/tasks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb integration tasks.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import os 7 | import json 8 | import logging 9 | import threading 10 | from time import sleep 11 | from datetime import datetime, tzinfo 12 | 13 | import click 14 | 15 | from honeycomb.defs import ARGS_JSON 16 | from honeycomb.integrationmanager import exceptions 17 | from honeycomb.integrationmanager.defs import (IntegrationTypes, IntegrationAlertStatuses, 18 | SEND_ALERT_DATA_INTERVAL, MAX_SEND_RETRIES) 19 | from honeycomb.integrationmanager.models import IntegrationAlert, ConfiguredIntegration 20 | from honeycomb.integrationmanager.registration import register_integration, get_integration_module 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | configured_integrations = list() 25 | polling_integration_alerts = list() 26 | 27 | 28 | class _UTC(tzinfo): 29 | def utcoffset(self, dt): 30 | return datetime.timedelta(0) 31 | 32 | def tzname(self, dt): 33 | return "UTC" 34 | 35 | def dst(self, dt): 36 | return datetime.timedelta(0) 37 | 38 | 39 | def configure_integration(path): 40 | """Configure and enable an integration.""" 41 | integration = register_integration(path) 42 | integration_args = {} 43 | try: 44 | with open(os.path.join(path, ARGS_JSON)) as f: 45 | integration_args = json.loads(f.read()) 46 | except Exception as exc: 47 | logger.debug(str(exc), exc_info=True) 48 | raise click.ClickException("Cannot load {} integration args, please configure it first." 49 | .format(os.path.basename(path))) 50 | 51 | click.secho("[*] Adding integration {}".format(integration.name)) 52 | logger.debug("Adding integration %s", integration.name, 53 | extra={"integration": integration.name, "args": integration_args}) 54 | configured_integration = ConfiguredIntegration(name=integration.name, integration=integration, path=path) 55 | configured_integration.data = integration_args 56 | configured_integration.integration.module = get_integration_module(path).IntegrationActionsClass(integration_args) 57 | 58 | configured_integrations.append(configured_integration) 59 | 60 | 61 | def send_alert_to_subscribed_integrations(alert): 62 | """Send Alert to relevant integrations.""" 63 | valid_configured_integrations = get_valid_configured_integrations(alert) 64 | 65 | for configured_integration in valid_configured_integrations: 66 | threading.Thread(target=create_integration_alert_and_call_send, args=(alert, configured_integration)).start() 67 | 68 | 69 | def get_current_datetime_utc(): 70 | """Return a datetime object localized to UTC.""" 71 | return datetime.utcnow().replace(tzinfo=_UTC()) 72 | 73 | 74 | def get_valid_configured_integrations(alert): 75 | """Return a list of integrations for alert filtered by alert_type. 76 | 77 | :returns: A list of relevant integrations 78 | """ 79 | if not configured_integrations: 80 | return [] 81 | 82 | # Collect all integrations that are configured for specific alert_type 83 | # or have no specific supported_event_types (i.e., all alert types) 84 | valid_configured_integrations = [ 85 | _ for _ in configured_integrations if _.integration.integration_type == IntegrationTypes.EVENT_OUTPUT.name and 86 | (not _.integration.supported_event_types or alert.alert_type in _.integration.supported_event_types) 87 | ] 88 | 89 | return valid_configured_integrations 90 | 91 | 92 | def create_integration_alert_and_call_send(alert, configured_integration): 93 | """Create an IntegrationAlert object and send it to Integration.""" 94 | integration_alert = IntegrationAlert( 95 | alert=alert, 96 | configured_integration=configured_integration, 97 | status=IntegrationAlertStatuses.PENDING.name, 98 | retries=configured_integration.integration.max_send_retries 99 | ) 100 | 101 | send_alert_to_configured_integration(integration_alert) 102 | 103 | 104 | def send_alert_to_configured_integration(integration_alert): 105 | """Send IntegrationAlert to configured integration.""" 106 | try: 107 | alert = integration_alert.alert 108 | configured_integration = integration_alert.configured_integration 109 | integration = configured_integration.integration 110 | integration_actions_instance = configured_integration.integration.module 111 | 112 | alert_fields = dict() 113 | if integration.required_fields: 114 | if not all([hasattr(alert, _) for _ in integration.required_fields]): 115 | logger.debug("Alert does not have all required_fields (%s) for integration %s, skipping", 116 | integration.required_fields, 117 | integration.name) 118 | return 119 | 120 | exclude_fields = ["alert_type", "service_type"] 121 | alert_fields = {} 122 | for field in alert.__slots__: 123 | if hasattr(alert, field) and field not in exclude_fields: 124 | alert_fields[field] = getattr(alert, field) 125 | 126 | logger.debug("Sending alert %s to %s", alert_fields, integration.name) 127 | output_data, output_file_content = integration_actions_instance.send_event(alert_fields) 128 | 129 | if integration.polling_enabled: 130 | integration_alert.status = IntegrationAlertStatuses.POLLING.name 131 | polling_integration_alerts.append(integration_alert) 132 | else: 133 | integration_alert.status = IntegrationAlertStatuses.DONE.name 134 | 135 | integration_alert.send_time = get_current_datetime_utc() 136 | integration_alert.output_data = json.dumps(output_data) 137 | # TODO: do something with successfully handled alerts? They are all written to debug log file 138 | 139 | except exceptions.IntegrationMissingRequiredFieldError as exc: 140 | logger.exception("Send response formatting for integration alert %s failed. Missing required fields", 141 | integration_alert, 142 | exc.message) 143 | 144 | integration_alert.status = IntegrationAlertStatuses.ERROR_MISSING_SEND_FIELDS.name 145 | 146 | except exceptions.IntegrationOutputFormatError: 147 | logger.exception("Send response formatting for integration alert %s failed", integration_alert) 148 | 149 | integration_alert.status = IntegrationAlertStatuses.ERROR_SENDING_FORMATTING.name 150 | 151 | except exceptions.IntegrationSendEventError as exc: 152 | integration_send_retries = integration_alert.retries if integration_alert.retries <= MAX_SEND_RETRIES \ 153 | else MAX_SEND_RETRIES # making sure we do not exceed celery max retries 154 | send_retries_left = integration_send_retries - 1 155 | integration_alert.retries = send_retries_left 156 | 157 | logger.error("Sending integration alert %s failed. Message: %s. Retries left: %s", 158 | integration_alert, 159 | exc.message, 160 | send_retries_left) 161 | 162 | if send_retries_left == 0: 163 | integration_alert.status = IntegrationAlertStatuses.ERROR_SENDING.name 164 | 165 | if send_retries_left > 0: 166 | sleep(SEND_ALERT_DATA_INTERVAL) 167 | send_alert_to_configured_integration(integration_alert) 168 | 169 | 170 | def poll_integration_information_for_waiting_integration_alerts(): 171 | """poll_integration_information_for_waiting_integration_alerts.""" 172 | if not polling_integration_alerts: 173 | return 174 | 175 | logger.debug("Polling information for waiting integration alerts") 176 | 177 | for integration_alert in polling_integration_alerts: 178 | configured_integration = integration_alert.configured_integration 179 | integration = configured_integration.integration 180 | polling_duration = integration.polling_duration 181 | 182 | if get_current_datetime_utc() - integration_alert.send_time > polling_duration: 183 | logger.debug("Polling duration expired for integration alert %s", integration_alert) 184 | integration_alert.status = IntegrationAlertStatuses.ERROR_POLLING.name 185 | else: 186 | integration_alert.status = IntegrationAlertStatuses.IN_POLLING.name 187 | 188 | poll_integration_alert_data(integration_alert) 189 | 190 | 191 | def poll_integration_alert_data(integration_alert): 192 | """Poll for updates on waiting IntegrationAlerts.""" 193 | logger.info("Polling information for integration alert %s", integration_alert) 194 | try: 195 | configured_integration = integration_alert.configured_integration 196 | integration_actions_instance = configured_integration.integration.module 197 | 198 | output_data, output_file_content = integration_actions_instance.poll_for_updates( 199 | json.loads(integration_alert.output_data) 200 | ) 201 | 202 | integration_alert.status = IntegrationAlertStatuses.DONE.name 203 | integration_alert.output_data = json.dumps(output_data) 204 | polling_integration_alerts.remove(integration_alert) 205 | 206 | except exceptions.IntegrationNoMethodImplementationError: 207 | logger.error("No poll_for_updates function found for integration alert %s", integration_alert) 208 | 209 | integration_alert.status = IntegrationAlertStatuses.ERROR_POLLING.name 210 | 211 | except exceptions.IntegrationPollEventError: 212 | # This does not always indicate an error, this is also raised when need to try again later 213 | logger.debug("Polling for integration alert %s failed", integration_alert) 214 | 215 | except exceptions.IntegrationOutputFormatError: 216 | logger.error("Integration alert %s formatting error", integration_alert) 217 | 218 | integration_alert.status = IntegrationAlertStatuses.ERROR_POLLING_FORMATTING.name 219 | 220 | except Exception: 221 | logger.exception("Error polling integration alert %s", integration_alert) 222 | 223 | integration_alert.status = IntegrationAlertStatuses.ERROR_POLLING.name 224 | -------------------------------------------------------------------------------- /honeycomb/servicemanager/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb Service Manager.""" 3 | from __future__ import unicode_literals, absolute_import 4 | -------------------------------------------------------------------------------- /honeycomb/servicemanager/base_service.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Custom Service implementation from MazeRunner.""" 3 | from __future__ import unicode_literals, absolute_import 4 | 5 | import os 6 | import sys 7 | import time 8 | import logging 9 | from threading import Thread 10 | from multiprocessing import Process 11 | 12 | import six 13 | import docker 14 | from attr import attrs, attrib 15 | from six.moves.queue import Queue, Full, Empty 16 | 17 | from honeycomb.decoymanager.models import Alert 18 | from honeycomb.servicemanager.defs import SERVICE_ALERT_QUEUE_SIZE 19 | from honeycomb.servicemanager.error_messages import INVALID_ALERT_TYPE 20 | from honeycomb.integrationmanager.tasks import send_alert_to_subscribed_integrations 21 | 22 | 23 | @attrs 24 | class ServerCustomService(Process): 25 | """Custom Service Class. 26 | 27 | This class provides a basic wrapper for honeycomb (and mazerunner) services. 28 | """ 29 | 30 | alerts_queue = None 31 | thread_server = None 32 | 33 | logger = logging.getLogger(__name__) 34 | """Logger to be used by plugins and collected by main logger.""" 35 | 36 | alert_types = attrib(type=list) 37 | """List of alert types, parsed from config.json""" 38 | 39 | service_args = attrib(type=dict, default={}) 40 | """Validated dictionary of service arguments (see: :func:`honeycomb.utils.plugin_utils.parse_plugin_args`)""" 41 | 42 | logger.setLevel(logging.DEBUG) 43 | 44 | def signal_ready(self): 45 | """Signal the service manager this service is ready for incoming connections.""" 46 | self.logger.debug("service is ready") 47 | 48 | def on_server_start(self): 49 | """Service run loop function. 50 | 51 | The service manager will call this function in a new thread. 52 | 53 | .. note:: Must call :func:`signal_ready` after finishing configuration 54 | """ 55 | raise NotImplementedError 56 | 57 | def on_server_shutdown(self): 58 | """Shutdown function of the server. 59 | 60 | Override this and take care to gracefully shut down your service (e.g., close files) 61 | """ 62 | raise NotImplementedError 63 | 64 | def run_service(self): 65 | """Run the service and start an alert processing queue. 66 | 67 | .. seealso:: Use :func:`on_server_start` and :func:`on_server_shutdown` for starting and shutting down 68 | your service 69 | """ 70 | self.alerts_queue = Queue(maxsize=SERVICE_ALERT_QUEUE_SIZE) 71 | self.thread_server = Thread(target=self._on_server_start) 72 | # self.thread_server.daemon = True 73 | self.thread_server.start() 74 | 75 | try: 76 | while self.thread_server.is_alive(): 77 | try: 78 | new_alert = self.alerts_queue.get(timeout=1) 79 | self.emit(**new_alert) 80 | except Empty: 81 | continue 82 | except KeyboardInterrupt: 83 | self.logger.debug("Caught KeyboardInterrupt, shutting service down gracefully") 84 | raise 85 | except Exception as exc: 86 | self.logger.exception(exc) 87 | finally: 88 | self._on_server_shutdown() 89 | 90 | def run(self): 91 | """Daemon entry point.""" 92 | self.run_service() 93 | 94 | def emit(self, **kwargs): 95 | """Send alerts to logfile. 96 | 97 | :param kwargs: Fields to pass to :py:class:`honeycomb.decoymanager.models.Alert` 98 | """ 99 | try: 100 | alert_type = next(_ for _ in self.alert_types if _.name == kwargs["event_type"]) 101 | except StopIteration: 102 | self.logger.error(INVALID_ALERT_TYPE, kwargs["event_type"]) 103 | return 104 | 105 | self.logger.critical(kwargs) 106 | 107 | alert = Alert(alert_type) 108 | for key, value in six.iteritems(kwargs): 109 | setattr(alert, key, value) 110 | 111 | send_alert_to_subscribed_integrations(alert) 112 | 113 | def add_alert_to_queue(self, alert_dict): 114 | """Log alert and send to integrations.""" 115 | try: 116 | self.alerts_queue.put(alert_dict, block=False) 117 | except Full: 118 | self.logger.warning("Queue (size=%d) is full and can't process messages", SERVICE_ALERT_QUEUE_SIZE) 119 | except Exception as exc: 120 | self.logger.exception(exc) 121 | 122 | def _on_server_start(self): 123 | try: 124 | self.on_server_start() 125 | except Exception as exc: 126 | self.logger.exception(exc) 127 | 128 | def _on_server_shutdown(self, signum=None, frame=None): 129 | if signum: 130 | sys.stderr.write("Terminating on signal {}".format(signum)) 131 | self.logger.debug("Terminating on signal %s", signum) 132 | self.on_server_shutdown() 133 | raise SystemExit() 134 | 135 | 136 | class DockerService(ServerCustomService): 137 | """Provides an ability to run a Docker container that will be monitored for events.""" 138 | 139 | def __init__(self, *args, **kwargs): 140 | super(DockerService, self).__init__(*args, **kwargs) 141 | self._container = None 142 | self._docker_client = docker.from_env() 143 | 144 | @property 145 | def docker_params(self): 146 | """Return a dictionary of docker run parameters. 147 | 148 | .. seealso:: 149 | Docker run: https://docs.docker.com/engine/reference/run/ 150 | 151 | :return: Dictionary, e.g., :code:`dict(ports={80: 80})` 152 | """ 153 | return {} 154 | 155 | @property 156 | def docker_image_name(self): 157 | """Return docker image name.""" 158 | raise NotImplementedError 159 | 160 | def parse_line(self, line): 161 | """Parse line and return dictionary if its an alert, else None / {}.""" 162 | raise NotImplementedError 163 | 164 | def get_lines(self): 165 | """Fetch log lines from the docker service. 166 | 167 | :return: A blocking logs generator 168 | """ 169 | return self._container.logs(stream=True) 170 | 171 | def read_lines(self, file_path, empty_lines=False, signal_ready=True): 172 | """Fetch lines from file. 173 | 174 | In case the file handler changes (logrotate), reopen the file. 175 | 176 | :param file_path: Path to file 177 | :param empty_lines: Return empty lines 178 | :param signal_ready: Report signal ready on start 179 | """ 180 | file_handler, file_id = self._get_file(file_path) 181 | file_handler.seek(0, os.SEEK_END) 182 | 183 | if signal_ready: 184 | self.signal_ready() 185 | 186 | while self.thread_server.is_alive(): 187 | line = six.text_type(file_handler.readline(), "utf-8") 188 | if line: 189 | yield line 190 | continue 191 | elif empty_lines: 192 | yield line 193 | 194 | time.sleep(0.1) 195 | 196 | if file_id != self._get_file_id(os.stat(file_path)) and os.path.isfile(file_path): 197 | file_handler, file_id = self._get_file(file_path) 198 | 199 | @staticmethod 200 | def _get_file_id(file_stat): 201 | if os.name == "posix": 202 | # st_dev: Device inode resides on. 203 | # st_ino: Inode number. 204 | return "%xg%x" % (file_stat.st_dev, file_stat.st_ino) 205 | return "%f" % file_stat.st_ctime 206 | 207 | def _get_file(self, file_path): 208 | file_handler = open(file_path, "rb") 209 | file_id = self._get_file_id(os.fstat(file_handler.fileno())) 210 | return file_handler, file_id 211 | 212 | def on_server_start(self): 213 | """Service run loop function. 214 | 215 | Run the desired docker container with parameters and start parsing the monitored file for alerts. 216 | """ 217 | self._container = self._docker_client.containers.run(self.docker_image_name, detach=True, **self.docker_params) 218 | self.signal_ready() 219 | 220 | for log_line in self.get_lines(): 221 | try: 222 | alert_dict = self.parse_line(log_line) 223 | if alert_dict: 224 | self.add_alert_to_queue(alert_dict) 225 | except Exception: 226 | self.logger.exception(None) 227 | 228 | def on_server_shutdown(self): 229 | """Stop the container before shutting down.""" 230 | if not self._container: 231 | return 232 | self._container.stop() 233 | self._container.remove(v=True, force=True) 234 | -------------------------------------------------------------------------------- /honeycomb/servicemanager/defs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb services definitions and constants.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import six 7 | 8 | from honeycomb.defs import (ConfigField, NAME, LABEL, DEFAULT, VALUE, 9 | TYPE, FIELD_LABEL, HELP_TEXT, REQUIRED, TEXT_TYPE, INTEGER_TYPE, BOOLEAN_TYPE, FILE_TYPE) 10 | from honeycomb.utils import config_utils 11 | from honeycomb.error_messages import FIELD_MISSING, CONFIG_FIELD_TYPE_ERROR 12 | from honeycomb.decoymanager.models import Alert 13 | from honeycomb.servicemanager.models import OSFamilies 14 | 15 | EVENT_TYPE = "event_type" 16 | SERVICE_CONFIG_SECTION_KEY = "service" 17 | ALERT_CONFIG_SECTION_KEY = "event_types" 18 | SERVICE_ALERT_QUEUE_SIZE = 1000 19 | 20 | LOGS_DIR = "logs" 21 | STDOUTLOG = "stdout.log" 22 | STDERRLOG = "stderr.log" 23 | 24 | """Service section.""" 25 | PORT = "port" 26 | PORTS = "ports" 27 | FIELDS = "fields" 28 | POLICY = "policy" 29 | WILDCARD_PORT = "*" 30 | PROTOCOL = "protocol" 31 | ALLOW_MANY = "allow_many" 32 | ALERT_TYPES = "alert_types" 33 | CONFLICTS_WITH = "conflicts_with" 34 | SUPPORTED_OS_FAMILIES = "supported_os_families" 35 | 36 | TCP = "TCP" 37 | UDP = "UDP" 38 | ALLOWED_PROTOCOLS = [TCP, UDP] 39 | 40 | 41 | """Parameters.""" 42 | SERVICE_ALLOWED_PARAMTER_KEYS = [VALUE, DEFAULT, TYPE, FIELD_LABEL, HELP_TEXT, REQUIRED] 43 | SERVICE_ALLOWED_PARAMTER_TYPES = [TEXT_TYPE, INTEGER_TYPE, BOOLEAN_TYPE, FILE_TYPE] 44 | 45 | SERVICE_FIELDS_TO_CREATE_OBJECT = [NAME, PORTS, LABEL, ALLOW_MANY, ALERT_TYPES, SUPPORTED_OS_FAMILIES] 46 | 47 | 48 | SERVICE_ALERT_VALIDATE_FIELDS = { 49 | SERVICE_CONFIG_SECTION_KEY: ConfigField( 50 | lambda field: True, 51 | lambda: FIELD_MISSING.format(SERVICE_CONFIG_SECTION_KEY) 52 | ), 53 | ALERT_CONFIG_SECTION_KEY: ConfigField( 54 | lambda field: True, 55 | lambda: FIELD_MISSING.format(ALERT_CONFIG_SECTION_KEY) 56 | ), 57 | } 58 | 59 | SERVICE_CONFIG_VALIDATE_FIELDS = { 60 | ALLOW_MANY: config_utils.config_field_type(ALLOW_MANY, bool), 61 | SUPPORTED_OS_FAMILIES: ConfigField( 62 | lambda family: family in OSFamilies.all_names(), 63 | lambda: "Operating system family must be one of the following: {}".format( 64 | ",".join(OSFamilies.all_names())) 65 | ), 66 | PORTS: ConfigField( 67 | lambda ports: isinstance(ports, list) and all( 68 | [port.get(PROTOCOL, False) in ALLOWED_PROTOCOLS and 69 | (isinstance(port.get(PORT, False), int) or 70 | port.get(PORT, "") == WILDCARD_PORT) for port in ports]), 71 | lambda: "Ports configuration invalid, please consult docs." 72 | ), 73 | NAME: config_utils.config_field_type(NAME, six.string_types), 74 | 75 | LABEL: config_utils.config_field_type(LABEL, six.string_types), 76 | 77 | CONFLICTS_WITH: ConfigField( 78 | lambda conflicts_with: (isinstance(conflicts_with, list) and 79 | all([isinstance(_, six.string_types) for _ in conflicts_with])), 80 | lambda: CONFIG_FIELD_TYPE_ERROR.format(CONFLICTS_WITH, "list of strings") 81 | ), 82 | 83 | } 84 | 85 | ALERT_CONFIG_VALIDATE_FIELDS = { 86 | NAME: ConfigField( 87 | lambda name: isinstance(name, six.string_types), 88 | lambda: "Alert name already used" 89 | ), 90 | LABEL: ConfigField( 91 | lambda label: isinstance(label, six.string_types), 92 | lambda: "Alert label already used" 93 | ), 94 | POLICY: ConfigField( 95 | lambda policy: isinstance(policy, six.string_types) and policy in [alert_status[1] 96 | for alert_status in Alert.ALERT_STATUS], 97 | lambda: "Alert policy must be one of the following: {}".format([_[1] for _ in Alert.ALERT_STATUS]) 98 | ), 99 | FIELDS: ConfigField( 100 | lambda fields: isinstance(fields, list) and all([field in Alert.__slots__ for field in fields]), 101 | lambda: "Alert fields must be one of the following: {}".format([Alert.__slots___]) 102 | ), 103 | } 104 | -------------------------------------------------------------------------------- /honeycomb/servicemanager/error_messages.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb services error messages.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | UNSUPPORTED_OS_ERROR = "Service requires running on {} and you are using {}" 7 | SERVICE_NOT_FOUND_ERROR = "Cannot find service named {}, try installing it?" 8 | 9 | INVALID_ALERT_TYPE = "%s is not a valid event_type, check event_types in config.json" 10 | -------------------------------------------------------------------------------- /honeycomb/servicemanager/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb Service Manager Exceptions.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import honeycomb.exceptions 7 | from honeycomb.servicemanager import error_messages 8 | 9 | 10 | class ServiceManagerException(honeycomb.exceptions.PluginError): 11 | """Generic Service Manager Exception.""" 12 | 13 | 14 | class ServiceNotFound(ServiceManagerException): 15 | """Specified service does not exist.""" 16 | 17 | msg_format = error_messages.SERVICE_NOT_FOUND_ERROR 18 | 19 | 20 | class UnsupportedOS(ServiceManagerException): 21 | """Specified service does not exist.""" 22 | 23 | msg_format = error_messages.UNSUPPORTED_OS_ERROR 24 | -------------------------------------------------------------------------------- /honeycomb/servicemanager/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service models.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | from attr import attrib, attrs 7 | 8 | from honeycomb.defs import BaseNameLabel, IBaseType 9 | 10 | 11 | @attrs 12 | class ServiceType(object): 13 | """Holds loaded service metadata.""" 14 | 15 | name = attrib(type=str) 16 | ports = attrib(type=list) 17 | label = attrib(type=str) 18 | allow_many = attrib(type=bool) 19 | supported_os_families = attrib(type=list) 20 | 21 | alert_types = attrib(type=list, default=[]) 22 | 23 | 24 | class OSFamilies(IBaseType): 25 | """Defines supported platforms for services.""" 26 | 27 | LINUX = BaseNameLabel("Linux", "Linux") 28 | MACOS = BaseNameLabel("Darwin", "Darwin") 29 | WINDOWS = BaseNameLabel("Windows", "Windows") 30 | ALL = BaseNameLabel("All", "All") 31 | -------------------------------------------------------------------------------- /honeycomb/servicemanager/registration.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service manager.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import os 7 | import sys 8 | import json 9 | import logging 10 | import platform 11 | import importlib 12 | 13 | import six 14 | 15 | from honeycomb.defs import NAME, LABEL, CONFIG_FILE_NAME, DEPS_DIR 16 | from honeycomb.utils import config_utils 17 | from honeycomb.exceptions import ConfigFileNotFound 18 | from honeycomb.decoymanager.models import AlertType 19 | from honeycomb.servicemanager import defs 20 | from honeycomb.servicemanager.models import ServiceType, OSFamilies 21 | from honeycomb.servicemanager.exceptions import ServiceNotFound, UnsupportedOS 22 | 23 | logger = logging.getLogger(__name__) 24 | 25 | 26 | def get_service_module(service_path): 27 | """Add custom paths to sys and import service module. 28 | 29 | :param service_path: Path to service folder 30 | """ 31 | # add custom paths so imports would work 32 | paths = [ 33 | os.path.dirname(__file__), # this folder, to catch base_service 34 | os.path.realpath(os.path.join(service_path, "..")), # service's parent folder for import 35 | os.path.realpath(os.path.join(service_path)), # service's folder for local imports 36 | os.path.realpath(os.path.join(service_path, DEPS_DIR)), # deps dir 37 | ] 38 | 39 | for path in paths: 40 | path = os.path.realpath(path) 41 | logger.debug("adding %s to path", path) 42 | sys.path.insert(0, path) 43 | 44 | # get our service class instance 45 | service_name = os.path.basename(service_path) 46 | module = ".".join([service_name, service_name + "_service"]) 47 | logger.debug("importing %s", module) 48 | return importlib.import_module(module) 49 | 50 | 51 | def register_service(package_folder): 52 | """Register a honeycomb service. 53 | 54 | :param package_folder: Path to folder with service to load 55 | :returns: Validated service object 56 | :rtype: :func:`honeycomb.utils.defs.ServiceType` 57 | """ 58 | logger.debug("registering service %s", package_folder) 59 | package_folder = os.path.realpath(package_folder) 60 | if not os.path.exists(package_folder): 61 | raise ServiceNotFound(os.path.basename(package_folder)) 62 | 63 | json_config_path = os.path.join(package_folder, CONFIG_FILE_NAME) 64 | if not os.path.exists(json_config_path): 65 | raise ConfigFileNotFound(json_config_path) 66 | 67 | with open(json_config_path, "r") as f: 68 | config_json = json.load(f) 69 | 70 | # Validate service and alert config 71 | config_utils.validate_config(config_json, defs.SERVICE_ALERT_VALIDATE_FIELDS) 72 | 73 | config_utils.validate_config(config_json.get(defs.SERVICE_CONFIG_SECTION_KEY, {}), 74 | defs.SERVICE_CONFIG_VALIDATE_FIELDS) 75 | _validate_supported_platform(config_json) 76 | _validate_alert_configs(config_json) 77 | config_utils.validate_config_parameters(config_json, 78 | defs.SERVICE_ALLOWED_PARAMTER_KEYS, 79 | defs.SERVICE_ALLOWED_PARAMTER_TYPES) 80 | 81 | service_type = _create_service_object(config_json) 82 | service_type.alert_types = _create_alert_types(config_json, service_type) 83 | 84 | return service_type 85 | 86 | 87 | def _validate_supported_platform(config_json): 88 | current_platform = platform.system() 89 | supported_platform = config_json[defs.SERVICE_CONFIG_SECTION_KEY][defs.SUPPORTED_OS_FAMILIES] 90 | 91 | if supported_platform == OSFamilies.ALL.name: 92 | return current_platform 93 | elif supported_platform == OSFamilies.LINUX.name and \ 94 | current_platform in [OSFamilies.LINUX.name, OSFamilies.MACOS.name]: 95 | return current_platform 96 | elif supported_platform == OSFamilies.WINDOWS.name == current_platform: 97 | return current_platform 98 | 99 | raise UnsupportedOS(supported_platform, current_platform) 100 | 101 | 102 | def _validate_alert_configs(config_json): 103 | alert_types = config_json[defs.ALERT_CONFIG_SECTION_KEY] 104 | for alert_type in alert_types: 105 | config_utils.validate_config(alert_type, defs.ALERT_CONFIG_VALIDATE_FIELDS) 106 | 107 | 108 | def _create_service_object(config_json): 109 | service_config = config_json[defs.SERVICE_CONFIG_SECTION_KEY] 110 | 111 | service_type_create_kwargs = { 112 | key: value for key, value in six.iteritems(service_config) 113 | if key in defs.SERVICE_FIELDS_TO_CREATE_OBJECT 114 | } 115 | 116 | obj = ServiceType(**service_type_create_kwargs) 117 | return obj 118 | 119 | 120 | def _create_alert_types(config_json, service_type): 121 | alert_types = [] 122 | for alert_type in config_json.get(defs.ALERT_CONFIG_SECTION_KEY, []): 123 | _alert_type = AlertType(name=alert_type[NAME], label=alert_type[LABEL], service_type=service_type) 124 | alert_types.append(_alert_type) 125 | return alert_types 126 | -------------------------------------------------------------------------------- /honeycomb/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb Utils.""" 3 | from __future__ import unicode_literals, absolute_import 4 | -------------------------------------------------------------------------------- /honeycomb/utils/config_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb Config Utilities.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import os 7 | import re 8 | import json 9 | import logging 10 | 11 | import six 12 | import yaml 13 | 14 | from honeycomb import defs, exceptions 15 | from honeycomb.error_messages import CONFIG_FIELD_TYPE_ERROR 16 | 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | def config_field_type(field, cls): 22 | """Validate a config field against a type. 23 | 24 | Similar functionality to :func:`validate_field_matches_type` but returns :obj:`honeycomb.defs.ConfigField` 25 | """ 26 | return defs.ConfigField(lambda _: isinstance(_, cls), 27 | lambda: CONFIG_FIELD_TYPE_ERROR.format(field, cls.__name__)) 28 | 29 | 30 | def validate_config(config_json, fields): 31 | """Validate a JSON file configuration against list of :obj:`honeycomb.defs.ConfigField`.""" 32 | for field_name, validator_obj in six.iteritems(fields): 33 | field_value = config_json.get(field_name, None) 34 | if field_value is None: 35 | raise exceptions.ConfigFieldMissing(field_name) 36 | 37 | if not validator_obj.validator_func(field_value): 38 | raise exceptions.ConfigFieldValidationError(field_name, field_value, validator_obj.get_error_message()) 39 | 40 | 41 | def get_config_parameters(plugin_path): 42 | """Return the parameters section from config.json.""" 43 | json_config_path = os.path.join(plugin_path, defs.CONFIG_FILE_NAME) 44 | with open(json_config_path, "r") as f: 45 | config = json.load(f) 46 | return config.get(defs.PARAMETERS, []) 47 | 48 | 49 | def validate_config_parameters(config_json, allowed_keys, allowed_types): 50 | """Validate parameters in config file.""" 51 | custom_fields = config_json.get(defs.PARAMETERS, []) 52 | for field in custom_fields: 53 | validate_field(field, allowed_keys, allowed_types) 54 | default = field.get(defs.DEFAULT) 55 | field_type = field.get(defs.TYPE) 56 | if default: 57 | validate_field_matches_type(field[defs.VALUE], default, field_type) 58 | 59 | 60 | def validate_field_matches_type(field, value, field_type, select_items=None, _min=None, _max=None): 61 | """Validate a config field against a specific type.""" 62 | if (field_type == defs.TEXT_TYPE and not isinstance(value, six.string_types)) or \ 63 | (field_type == defs.STRING_TYPE and not isinstance(value, six.string_types)) or \ 64 | (field_type == defs.BOOLEAN_TYPE and not isinstance(value, bool)) or \ 65 | (field_type == defs.INTEGER_TYPE and not isinstance(value, int)): 66 | raise exceptions.ConfigFieldTypeMismatch(field, value, field_type) 67 | 68 | if field_type == defs.INTEGER_TYPE: 69 | if _min and value < _min: 70 | raise exceptions.ConfigFieldTypeMismatch(field, value, "must be higher than {}".format(_min)) 71 | if _max and value > _max: 72 | raise exceptions.ConfigFieldTypeMismatch(field, value, "must be lower than {}".format(_max)) 73 | 74 | if field_type == defs.SELECT_TYPE: 75 | from honeycomb.utils.plugin_utils import get_select_items 76 | items = get_select_items(select_items) 77 | if value not in items: 78 | raise exceptions.ConfigFieldTypeMismatch(field, value, "one of: {}".format(", ".join(items))) 79 | 80 | 81 | def get_truetype(value): 82 | """Convert a string to a pythonized parameter.""" 83 | if value in ["true", "True", "y", "Y", "yes"]: 84 | return True 85 | if value in ["false", "False", "n", "N", "no"]: 86 | return False 87 | if value.isdigit(): 88 | return int(value) 89 | return str(value) 90 | 91 | 92 | def validate_field(field, allowed_keys, allowed_types): 93 | """Validate field is allowed and valid.""" 94 | for key, value in field.items(): 95 | if key not in allowed_keys: 96 | raise exceptions.ParametersFieldError(key, "property") 97 | if key == defs.TYPE: 98 | if value not in allowed_types: 99 | raise exceptions.ParametersFieldError(value, key) 100 | if key == defs.VALUE: 101 | if not is_valid_field_name(value): 102 | raise exceptions.ParametersFieldError(value, "field name") 103 | 104 | 105 | def is_valid_field_name(value): 106 | """Ensure field name is valid.""" 107 | leftovers = re.sub(r"\w", "", value) 108 | leftovers = re.sub(r"-", "", leftovers) 109 | if leftovers != "" or value[0].isdigit() or value[0] in ["-", "_"] or " " in value: 110 | return False 111 | return True 112 | 113 | 114 | def process_config(ctx, configfile): 115 | """Process a yaml config with instructions. 116 | 117 | This is a heavy method that loads lots of content, so we only run the imports if its called. 118 | """ 119 | from honeycomb.commands.service.run import run as service_run 120 | # from honeycomb.commands.service.logs import logs as service_logs 121 | from honeycomb.commands.service.install import install as service_install 122 | from honeycomb.commands.integration.install import install as integration_install 123 | from honeycomb.commands.integration.configure import configure as integration_configure 124 | 125 | VERSION = "version" 126 | SERVICES = defs.SERVICES 127 | INTEGRATIONS = defs.INTEGRATIONS 128 | 129 | required_top_keys = [VERSION, SERVICES] 130 | supported_versions = [1] 131 | 132 | def validate_yml(config): 133 | for key in required_top_keys: 134 | if key not in config: 135 | raise exceptions.ConfigFieldMissing(key) 136 | 137 | version = config.get(VERSION) 138 | if version not in supported_versions: 139 | raise exceptions.ConfigFieldTypeMismatch(VERSION, version, 140 | "one of: {}".format(repr(supported_versions))) 141 | 142 | def install_plugins(services, integrations): 143 | for cmd, kwargs in [(service_install, {SERVICES: services}), 144 | (integration_install, {INTEGRATIONS: integrations})]: 145 | try: 146 | ctx.invoke(cmd, **kwargs) 147 | except SystemExit: 148 | # If a plugin is already installed honeycomb will exit abnormally 149 | pass 150 | 151 | def parameters_to_string(parameters_dict): 152 | return ["{}={}".format(k, v) for k, v in parameters_dict.items()] 153 | 154 | def configure_integrations(integrations): 155 | for integration in integrations: 156 | args_list = parameters_to_string(config[INTEGRATIONS][integration].get(defs.PARAMETERS, dict())) 157 | ctx.invoke(integration_configure, integration=integration, args=args_list) 158 | 159 | def run_services(services, integrations): 160 | # TODO: Enable support with multiple services as daemon, and run service.logs afterwards 161 | # tricky part is that services launched as daemon are exited with os._exit(0) so you 162 | # can't catch it. 163 | for service in services: 164 | args_list = parameters_to_string(config[SERVICES][service].get(defs.PARAMETERS, dict())) 165 | ctx.invoke(service_run, service=service, integration=integrations, args=args_list) 166 | 167 | # TODO: Silence normal stdout and follow honeycomb.debug.json instead 168 | # This would make monitoring containers and collecting logs easier 169 | with open(configfile, "rb") as fh: 170 | config = yaml.load(fh.read()) 171 | 172 | validate_yml(config) 173 | services = config.get(SERVICES).keys() 174 | integrations = config.get(INTEGRATIONS).keys() if config.get(INTEGRATIONS) else [] 175 | 176 | install_plugins(services, integrations) 177 | configure_integrations(integrations) 178 | run_services(services, integrations) 179 | -------------------------------------------------------------------------------- /honeycomb/utils/daemon.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb DaemonRunner utility.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import sys 7 | 8 | import daemon.runner 9 | import daemon.daemon 10 | 11 | 12 | class myRunner(daemon.runner.DaemonRunner): 13 | """Overriding default runner behaviour to be simpler.""" 14 | 15 | def __init__(self, app, pidfile=None, stdout=sys.stdout, stderr=sys.stderr, stdin=open("/dev/null", "rt")): 16 | """ 17 | Override init to fit honeycomb needs. 18 | 19 | We initialize app with default stdout/stderr from sys instead of file path 20 | and remove the use of parse_args() since it's not actually a standalone runner 21 | """ 22 | self.app = app 23 | self.daemon_context = daemon.daemon.DaemonContext() 24 | self.daemon_context.stdin = stdin 25 | self.daemon_context.stdout = stdout 26 | self.daemon_context.stderr = stderr 27 | self.app.pidfile_path = pidfile 28 | self.pidfile = None 29 | if self.app.pidfile_path is not None: 30 | self.pidfile = daemon.runner.make_pidlockfile(self.app.pidfile_path, 3) 31 | self.daemon_context.pidfile = self.pidfile 32 | -------------------------------------------------------------------------------- /honeycomb/utils/plugin_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb generic plugin install utils.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import os 7 | import sys 8 | import shutil 9 | import logging 10 | import zipfile 11 | import tempfile 12 | import subprocess 13 | 14 | import click 15 | import requests 16 | from requests.adapters import HTTPAdapter 17 | 18 | from honeycomb import defs, exceptions 19 | from honeycomb.utils import config_utils 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | try: 24 | O_BINARY = os.O_BINARY 25 | except Exception: 26 | O_BINARY = 0 27 | 28 | READ_FLAGS = os.O_RDONLY | O_BINARY 29 | WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | O_BINARY 30 | BUFFER_SIZE = 128 * 1024 31 | 32 | 33 | class CTError(Exception): 34 | """Copytree exception class, used to collect errors from the recursive copy_tree function.""" 35 | 36 | def __init__(self, errors): 37 | """Collect errors. 38 | 39 | :param errors: Collected errors 40 | """ 41 | self.errors = errors 42 | 43 | 44 | def get_plugin_path(home, plugin_type, plugin_name, editable=False): 45 | """Return path to plugin. 46 | 47 | :param home: Path to honeycomb home 48 | :param plugin_type: Type of plugin (:obj:`honeycomb.defs.SERVICES` pr :obj:`honeycomb.defs.INTEGRATIONS`) 49 | :param plugin_name: Name of plugin 50 | :param editable: Use plugin_name as direct path instead of loading from honeycomb home folder 51 | """ 52 | if editable: 53 | plugin_path = plugin_name 54 | else: 55 | plugin_path = os.path.join(home, plugin_type, plugin_name) 56 | 57 | return os.path.realpath(plugin_path) 58 | 59 | 60 | def install_plugin(pkgpath, plugin_type, install_path, register_func): 61 | """Install specified plugin. 62 | 63 | :param pkgpath: Name of plugin to be downloaded from online repo or path to plugin folder or zip file. 64 | :param install_path: Path where plugin will be installed. 65 | :param register_func: Method used to register and validate plugin. 66 | """ 67 | service_name = os.path.basename(pkgpath) 68 | if os.path.exists(os.path.join(install_path, service_name)): 69 | raise exceptions.PluginAlreadyInstalled(pkgpath) 70 | 71 | if os.path.exists(pkgpath): 72 | logger.debug("%s exists in filesystem", pkgpath) 73 | if os.path.isdir(pkgpath): 74 | pip_status = install_dir(pkgpath, install_path, register_func) 75 | else: # pkgpath is file 76 | pip_status = install_from_zip(pkgpath, install_path, register_func) 77 | else: 78 | logger.debug("cannot find %s locally, checking github repo", pkgpath) 79 | click.secho("Collecting {}..".format(pkgpath)) 80 | pip_status = install_from_repo(pkgpath, plugin_type, install_path, register_func) 81 | 82 | if pip_status == 0: 83 | click.secho("[+] Great success!") 84 | else: 85 | # TODO: rephrase 86 | click.secho("[-] Service installed but something was odd with dependency install, please review debug logs") 87 | 88 | 89 | def install_deps(pkgpath): 90 | """Install plugin dependencies using pip. 91 | 92 | We import pip here to reduce load time for when its not needed. 93 | """ 94 | if os.path.exists(os.path.join(pkgpath, "requirements.txt")): 95 | logger.debug("installing dependencies") 96 | click.secho("[*] Installing dependencies") 97 | pipargs = ["install", "--target", os.path.join(pkgpath, defs.DEPS_DIR), "--ignore-installed", 98 | "-r", os.path.join(pkgpath, "requirements.txt")] 99 | logger.debug("running pip %s", pipargs) 100 | return subprocess.check_call([sys.executable, "-m", "pip"] + pipargs) 101 | return 0 # pip.main returns retcode 102 | 103 | 104 | def copy_file(src, dst): 105 | """Copy a single file. 106 | 107 | :param src: Source name 108 | :param dst: Destination name 109 | """ 110 | try: 111 | fin = os.open(src, READ_FLAGS) 112 | stat = os.fstat(fin) 113 | fout = os.open(dst, WRITE_FLAGS, stat.st_mode) 114 | for x in iter(lambda: os.read(fin, BUFFER_SIZE), b""): 115 | os.write(fout, x) 116 | finally: 117 | try: 118 | os.close(fin) 119 | except Exception as exc: 120 | logger.debug("Failed to close file handle when copying: {}".format(exc)) 121 | try: 122 | os.close(fout) 123 | except Exception as exc: 124 | logger.debug("Failed to close file handle when copying: {}".format(exc)) 125 | 126 | 127 | # Due to speed issues, shutil.copytree had to be swapped out for something faster. 128 | # The solution was to copy (and slightly refactor) the code from: 129 | # https://stackoverflow.com/questions/22078621/python-how-to-copy-files-fast 130 | def copy_tree(src, dst, symlinks=False, ignore=[]): 131 | """Copy a full directory structure. 132 | 133 | :param src: Source path 134 | :param dst: Destination path 135 | :param symlinks: Copy symlinks 136 | :param ignore: Subdirs/filenames to ignore 137 | """ 138 | names = os.listdir(src) 139 | 140 | if not os.path.exists(dst): 141 | os.makedirs(dst) 142 | errors = [] 143 | for name in names: 144 | if name in ignore: 145 | continue 146 | srcname = os.path.join(src, name) 147 | dstname = os.path.join(dst, name) 148 | try: 149 | if symlinks and os.path.islink(srcname): 150 | linkto = os.readlink(srcname) 151 | os.symlink(linkto, dstname) 152 | elif os.path.isdir(srcname): 153 | copy_tree(srcname, dstname, symlinks, ignore) 154 | else: 155 | copy_file(srcname, dstname) 156 | except (IOError, os.error) as exc: 157 | errors.append((srcname, dstname, str(exc))) 158 | except CTError as exc: 159 | errors.extend(exc.errors) 160 | if errors: 161 | raise CTError(errors) 162 | 163 | 164 | def install_dir(pkgpath, install_path, register_func, delete_after_install=False): 165 | """Install plugin from specified directory. 166 | 167 | install_path and register_func are same as :func:`install_plugin`. 168 | :param delete_after_install: Delete pkgpath after install (used in :func:`install_from_zip`). 169 | """ 170 | logger.debug("%s is a directory, attempting to validate", pkgpath) 171 | plugin = register_func(pkgpath) 172 | logger.debug("%s looks good, copying to %s", pkgpath, install_path) 173 | try: 174 | copy_tree(pkgpath, os.path.join(install_path, plugin.name)) 175 | if delete_after_install: 176 | logger.debug("deleting %s", pkgpath) 177 | shutil.rmtree(pkgpath) 178 | pkgpath = os.path.join(install_path, plugin.name) 179 | except (OSError, CTError) as exc: 180 | # TODO: handle package name exists (upgrade? overwrite?) 181 | logger.debug(str(exc), exc_info=True) 182 | raise exceptions.PluginAlreadyInstalled(plugin.name) 183 | 184 | return install_deps(pkgpath) 185 | 186 | 187 | def install_from_zip(pkgpath, install_path, register_func, delete_after_install=False): 188 | """Install plugin from zipfile.""" 189 | logger.debug("%s is a file, attempting to load zip", pkgpath) 190 | pkgtempdir = tempfile.mkdtemp(prefix="honeycomb_") 191 | try: 192 | with zipfile.ZipFile(pkgpath) as pkgzip: 193 | pkgzip.extractall(pkgtempdir) 194 | except zipfile.BadZipfile as exc: 195 | logger.debug(str(exc)) 196 | raise click.ClickException(str(exc)) 197 | if delete_after_install: 198 | logger.debug("deleting %s", pkgpath) 199 | os.remove(pkgpath) 200 | logger.debug("installing from unzipped folder %s", pkgtempdir) 201 | return install_dir(pkgtempdir, install_path, register_func, delete_after_install=True) 202 | 203 | 204 | def install_from_repo(pkgname, plugin_type, install_path, register_func): 205 | """Install plugin from online repo.""" 206 | rsession = requests.Session() 207 | rsession.mount("https://", HTTPAdapter(max_retries=3)) 208 | 209 | logger.debug("trying to install %s from online repo", pkgname) 210 | pkgurl = "{}/{}s/{}.zip".format(defs.GITHUB_RAW, plugin_type, pkgname) 211 | try: 212 | logger.debug("Requesting HTTP HEAD: %s", pkgurl) 213 | r = rsession.head(pkgurl) 214 | r.raise_for_status() 215 | total_size = int(r.headers.get("content-length", 0)) 216 | pkgsize = _sizeof_fmt(total_size) 217 | with click.progressbar(length=total_size, label="Downloading {} {} ({}).." 218 | .format(plugin_type, pkgname, pkgsize)) as bar: 219 | r = rsession.get(pkgurl, stream=True) 220 | with tempfile.NamedTemporaryFile(delete=False) as f: 221 | downloaded_bytes = 0 222 | for chunk in r.iter_content(chunk_size=1): # TODO: Consider increasing to reduce cycles 223 | if chunk: 224 | f.write(chunk) 225 | downloaded_bytes += len(chunk) 226 | bar.update(downloaded_bytes) 227 | return install_from_zip(f.name, install_path, register_func, delete_after_install=True) 228 | except requests.exceptions.HTTPError as exc: 229 | logger.debug(str(exc)) 230 | raise exceptions.PluginNotFoundInOnlineRepo(pkgname) 231 | except requests.exceptions.ConnectionError as exc: 232 | logger.debug(str(exc)) 233 | raise exceptions.PluginRepoConnectionError() 234 | 235 | 236 | def _sizeof_fmt(num, suffix="B"): 237 | if not num: 238 | return "unknown size" 239 | for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: 240 | if abs(num) < 1024.0: 241 | return "%3.1f%s%s" % (num, unit, suffix) 242 | num /= 1024.0 243 | return "%.1f%s%s".format(num, "Yi", suffix) 244 | 245 | 246 | def uninstall_plugin(pkgpath, force): 247 | """Uninstall a plugin. 248 | 249 | :param pkgpath: Path to package to uninstall (delete) 250 | :param force: Force uninstall without asking 251 | """ 252 | pkgname = os.path.basename(pkgpath) 253 | if os.path.exists(pkgpath): 254 | if not force: 255 | click.confirm("[?] Are you sure you want to delete `{}` from honeycomb?".format(pkgname), 256 | abort=True) 257 | try: 258 | shutil.rmtree(pkgpath) 259 | logger.debug("successfully uninstalled {}".format(pkgname)) 260 | click.secho("[*] Uninstalled {}".format(pkgname)) 261 | except OSError as exc: 262 | logger.exception(str(exc)) 263 | else: 264 | click.secho("[-] doh! I cannot seem to find `{}`, are you sure it's installed?".format(pkgname)) 265 | 266 | 267 | def list_remote_plugins(installed_plugins, plugin_type): 268 | """List remote plugins from online repo.""" 269 | click.secho("\n[*] Additional plugins from online repository:") 270 | try: 271 | rsession = requests.Session() 272 | rsession.mount("https://", HTTPAdapter(max_retries=3)) 273 | 274 | r = rsession.get("{0}/{1}s/{1}s.txt".format(defs.GITHUB_RAW, plugin_type)) 275 | logger.debug("fetching %ss from remote repo", plugin_type) 276 | plugins = [_ for _ in r.text.splitlines() if _ not in installed_plugins] 277 | click.secho(" ".join(plugins)) 278 | 279 | except requests.exceptions.ConnectionError as exc: 280 | logger.debug(str(exc), exc_info=True) 281 | raise click.ClickException("Unable to fetch {} information from online repository".format(plugin_type)) 282 | 283 | 284 | def list_local_plugins(plugin_type, plugins_path, plugin_details): 285 | """List local plugins with details.""" 286 | installed_plugins = list() 287 | for plugin in next(os.walk(plugins_path))[1]: 288 | s = plugin_details(plugin) 289 | installed_plugins.append(plugin) 290 | click.secho(s) 291 | 292 | if not installed_plugins: 293 | click.secho("[*] You do not have any {0}s installed, " 294 | "try installing one with `honeycomb {0} install`".format(plugin_type)) 295 | 296 | return installed_plugins 297 | 298 | 299 | def parse_plugin_args(command_args, config_args): 300 | """Parse command line arguments based on the plugin's parameters config. 301 | 302 | :param command_args: Command line arguments as provided by the user in `key=value` format. 303 | :param config_args: Plugin parameters parsed from config.json. 304 | 305 | :returns: Validated dictionary of parameters that will be passed to plugin class 306 | """ 307 | parsed_args = dict() 308 | for arg in command_args: 309 | kv = arg.split("=") 310 | if len(kv) != 2: 311 | raise click.UsageError("Invalid parameter '{}', must be in key=value format".format(arg)) 312 | parsed_args[kv[0]] = config_utils.get_truetype(kv[1]) 313 | 314 | for arg in config_args: 315 | value = arg[defs.VALUE] 316 | value_type = arg[defs.TYPE] 317 | if value in parsed_args: 318 | # will raise if invalid 319 | config_utils.validate_field_matches_type(value, parsed_args[value], value_type, 320 | arg.get(defs.ITEMS), arg.get(defs.MIN), arg.get(defs.MAX)) 321 | elif defs.DEFAULT in arg: # Has a default field 322 | # return default values for unset parameters 323 | parsed_args[value] = arg[defs.DEFAULT] 324 | elif arg[defs.REQUIRED]: # requires field is true 325 | """parameter was not supplied by user, but it's required and has no default value""" 326 | raise exceptions.RequiredFieldMissing(value) 327 | return parsed_args 328 | 329 | 330 | def get_select_items(items): 331 | """Return list of possible select items.""" 332 | option_items = list() 333 | for item in items: 334 | if isinstance(item, dict) and defs.VALUE in item and defs.LABEL in item: 335 | option_items.append(item[defs.VALUE]) 336 | else: 337 | raise exceptions.ParametersFieldError(item, "a dictionary with {} and {}" 338 | .format(defs.LABEL, defs.VALUE)) 339 | return option_items 340 | 341 | 342 | def _parse_select_options(arg): 343 | options = "" 344 | if arg[defs.TYPE] == defs.SELECT_TYPE: 345 | if defs.ITEMS in arg or not isinstance(arg[defs.ITEMS], list): 346 | option_items = get_select_items(arg[defs.ITEMS]) 347 | options = " (valid options: {})".format(", ".join(option_items)) 348 | else: 349 | raise exceptions.ParametersFieldError(defs.ITEMS, "list") 350 | 351 | return options 352 | 353 | 354 | def print_plugin_args(plugin_path): 355 | """Print plugin parameters table.""" 356 | args = config_utils.get_config_parameters(plugin_path) 357 | args_format = "{:20} {:10} {:^15} {:^10} {:25}" 358 | title = args_format.format(defs.NAME.upper(), defs.TYPE.upper(), defs.DEFAULT.upper(), 359 | defs.REQUIRED.upper(), defs.DESCRIPTION.upper()) 360 | click.secho(title) 361 | click.secho("-" * len(title)) 362 | for arg in args: 363 | help_text = " ({})".format(arg[defs.HELP_TEXT]) if defs.HELP_TEXT in arg else "" 364 | options = _parse_select_options(arg) 365 | description = arg[defs.LABEL] + options + help_text 366 | click.secho(args_format.format(arg[defs.VALUE], arg[defs.TYPE], str(arg.get(defs.DEFAULT, None)), 367 | str(arg.get(defs.REQUIRED, False)), description)) 368 | -------------------------------------------------------------------------------- /honeycomb/utils/tailer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service log tailer.""" 3 | 4 | import os 5 | import sys 6 | import time 7 | import random 8 | import logging 9 | 10 | import click 11 | from attr import attrs, attrib 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | @attrs 17 | class Tailer(object): 18 | """Colorized file tailer. 19 | 20 | Print lines from a file prefixed with a colored name. Optionally continue to follow file. 21 | """ 22 | 23 | name = attrib(type=str) 24 | filepath = attrib(type=str) 25 | 26 | color = attrib("", type=str) 27 | nlines = attrib(10, type=int) 28 | follow = attrib(False, type=bool) 29 | outfile = attrib(sys.stdout) 30 | sleeptime = attrib(0.5, type=int) 31 | show_name = attrib(True, type=bool) 32 | used_colors = attrib([], type=list) 33 | 34 | colors = attrib(["red", "green", "yellow", "magenta", "blue", "cyan"], init=False) 35 | 36 | def print_log(self, line): 37 | """Print a line from a logfile.""" 38 | click.echo(line.replace("\n", ""), file=self.outfile) 39 | 40 | def print_named_log(self, line): 41 | """Print a line from a logfile prefixed with service name.""" 42 | click.echo("{}: {}".format(click.style(self.name, fg=self.color), line.replace("\n", "")), file=self.outfile) 43 | 44 | def _print(self, line): 45 | if self.show_name: 46 | self.print_named_log(line) 47 | else: 48 | self.print_log(line) 49 | 50 | def follow_file(self): 51 | """Follow a file and send every new line to a callback.""" 52 | logger.debug("following %s", self.filepath) 53 | with open(self.filepath) as fh: 54 | # Go to the end of file 55 | fh.seek(0, os.SEEK_END) 56 | 57 | while True: 58 | curr_position = fh.tell() 59 | line = fh.readline() 60 | if not line: 61 | fh.seek(curr_position) 62 | time.sleep(self.sleeptime) 63 | else: 64 | self._print(line) 65 | 66 | def __attrs_post_init__(self): 67 | """Seek file from end for nlines and call printlog on them, then follow if needed.""" 68 | logger.debug("reading %d lines from %s", self.nlines, self.filepath) 69 | 70 | if not self.color: 71 | self.color = self.colors[random.randint(0, len(self.colors) - 1)] 72 | 73 | with open(self.filepath) as fh: 74 | fh.seek(0, os.SEEK_END) 75 | end_position = curr_position = fh.tell() 76 | line_count = 0 77 | while curr_position >= 0: 78 | fh.seek(curr_position) 79 | next_char = fh.read(1) 80 | if next_char == "\n" and curr_position != end_position - 1: 81 | line_count += 1 82 | if line_count == self.nlines: 83 | break 84 | curr_position -= 1 85 | 86 | if curr_position < 0: 87 | fh.seek(0) 88 | 89 | for line in fh.readlines(): 90 | self._print(line) 91 | 92 | if self.follow: 93 | self.follow_file() 94 | 95 | def stop(self): 96 | """Stop follow.""" 97 | self.running = False 98 | -------------------------------------------------------------------------------- /honeycomb/utils/wait.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb wait utilities.""" 3 | 4 | from __future__ import unicode_literals, absolute_import 5 | 6 | import time 7 | import json 8 | import logging 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class TimeoutException(Exception): 14 | """Exception to be raised on timeout.""" 15 | 16 | pass 17 | 18 | 19 | def wait_until(func, 20 | check_return_value=True, 21 | total_timeout=60, 22 | interval=0.5, 23 | exc_list=None, 24 | error_message="", 25 | *args, 26 | **kwargs): 27 | """Run a command in a loop until desired result or timeout occurs. 28 | 29 | :param func: Function to call and wait for 30 | :param bool check_return_value: Examine return value 31 | :param int total_timeout: Wait timeout, 32 | :param float interval: Sleep interval between retries 33 | :param list exc_list: Acceptable exception list 34 | :param str error_message: Default error messages 35 | :param args: args to pass to func 36 | :param kwargs: lwargs to pass to fun 37 | """ 38 | start_function = time.time() 39 | while time.time() - start_function < total_timeout: 40 | 41 | try: 42 | logger.debug("executing {} with args {} {}".format(func, args, kwargs)) 43 | return_value = func(*args, **kwargs) 44 | if not check_return_value or (check_return_value and return_value): 45 | return return_value 46 | 47 | except Exception as exc: 48 | if exc_list and any([isinstance(exc, x) for x in exc_list]): 49 | pass 50 | else: 51 | raise 52 | 53 | time.sleep(interval) 54 | 55 | raise TimeoutException(error_message) 56 | 57 | 58 | def search_json_log(filepath, key, value): 59 | """Search json log file for a key=value pair. 60 | 61 | :param filepath: Valid path to a json file 62 | :param key: key to match 63 | :param value: value to match 64 | :returns: First matching line in json log file, parsed by :py:func:`json.loads` 65 | """ 66 | try: 67 | with open(filepath, "r") as fh: 68 | for line in fh.readlines(): 69 | log = json.loads(line) 70 | if key in log and log[key] == value: 71 | return log 72 | except IOError: 73 | pass 74 | return False 75 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | tox==3.7.0 2 | sphinx==1.8.2 3 | sphinx-click==1.4.1 4 | codecov==2.0.15 5 | pytest==4.0.1 6 | pytest-cov==2.6.1 7 | pytest-dependency==0.4.0 8 | flake8==3.6.0 9 | flake8-docstrings==1.3.0 10 | 11 | -r requirements.txt 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | six==1.12.0 2 | click==7.0 3 | attrs==18.2.0 4 | PyYAML==3.13 5 | requests==2.21.0 6 | python-daemon==2.2.3 7 | python-json-logger==0.1.10 8 | docker==3.7.0 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb is an extensible honeypot framework.""" 3 | from __future__ import absolute_import 4 | 5 | from setuptools import find_packages, setup 6 | from honeycomb import __version__ 7 | 8 | with open('requirements.txt') as fp: 9 | install_requires = fp.read() 10 | 11 | setup( 12 | name='honeycomb_framework', 13 | version=__version__, 14 | url='https://github.com/Cymmetria/honeycomb', 15 | license='MIT', 16 | author='Honeycomb - Honeypot Framework', 17 | author_email='omer.cohen@cymmetria.com', 18 | description='Honeycomb is a honeypot framework', 19 | long_description=__doc__, 20 | packages=find_packages(exclude=['tests']), 21 | include_package_data=True, 22 | zip_safe=False, 23 | platforms='any', 24 | install_requires=install_requires, 25 | entry_points={ 26 | 'console_scripts': [ 27 | 'honeycomb = honeycomb.__main__:main', 28 | ], 29 | }, 30 | classifiers=[ 31 | # As from http://pypi.python.org/pypi?%3Aaction=list_classifiers 32 | # 'Development Status :: 1 - Planning', 33 | # 'Development Status :: 2 - Pre-Alpha', 34 | # 'Development Status :: 3 - Alpha', 35 | 'Development Status :: 4 - Beta', 36 | # 'Development Status :: 5 - Production/Stable', 37 | # 'Development Status :: 6 - Mature', 38 | # 'Development Status :: 7 - Inactive', 39 | 'Environment :: Console', 40 | 'Intended Audience :: Developers', 41 | 'License :: OSI Approved :: MIT License', 42 | 'Operating System :: POSIX', 43 | 'Operating System :: MacOS', 44 | 'Operating System :: Unix', 45 | 'Operating System :: Microsoft :: Windows', 46 | 'Programming Language :: Python', 47 | 'Programming Language :: Python :: 2', 48 | 'Programming Language :: Python :: 3', 49 | 'Topic :: Software Development :: Libraries :: Python Modules', 50 | ] 51 | ) 52 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb tests.""" 3 | -------------------------------------------------------------------------------- /tests/test_honeycomb.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service tests.""" 3 | 4 | from __future__ import absolute_import 5 | 6 | import os 7 | import signal 8 | import requests 9 | import subprocess 10 | from requests.adapters import HTTPAdapter 11 | 12 | import pytest 13 | from click.testing import CliRunner 14 | 15 | from honeycomb.cli import cli 16 | from honeycomb import defs, integrationmanager 17 | from honeycomb.utils.wait import wait_until, search_json_log 18 | 19 | from tests.utils.defs import commands, args 20 | from tests.utils.syslog import runSyslogServer 21 | from tests.utils.test_utils import sanity_check, search_file_log 22 | 23 | DEMO_SERVICE = "simple_http" 24 | DEMO_SERVICE_PORT = 8888 25 | DEMO_SERVICE_ARGS = "port={}".format(DEMO_SERVICE_PORT) 26 | DEMO_SERVICE_PORTS = "Undefined" 27 | DEMO_SERVICE_ALERT = "simple_http" 28 | RUN_HONEYCOMB = "coverage run --parallel-mode --module --source=honeycomb honeycomb".split(" ") 29 | 30 | SYSLOG_PORT = 5514 31 | SYSLOG_HOST = "127.0.0.1" 32 | DEMO_INTEGRATION = "syslog" 33 | DEMO_INTEGRATION_ARGS = "protocol=udp address={} port={}".format(SYSLOG_HOST, SYSLOG_PORT) 34 | 35 | rsession = requests.Session() 36 | rsession.mount("https://", HTTPAdapter(max_retries=3)) 37 | 38 | 39 | @pytest.fixture 40 | def syslogd(tmpdir): 41 | """Run a syslog server and provide the logfile.""" 42 | logfile = str(tmpdir.join("syslog.log")) 43 | syslogd = runSyslogServer(SYSLOG_HOST, SYSLOG_PORT, logfile) 44 | yield logfile 45 | syslogd.shutdown() 46 | 47 | 48 | @pytest.fixture 49 | def service_installed(tmpdir): 50 | """Prepare honeycomb home path with DEMO_SERVICE installed.""" 51 | home = str(tmpdir) 52 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [home, defs.SERVICE, 53 | commands.INSTALL, DEMO_SERVICE]) 54 | sanity_check(result, home) 55 | assert os.path.exists(os.path.join(home, defs.SERVICES, DEMO_SERVICE, "{}_service.py".format(DEMO_SERVICE))) 56 | 57 | yield home 58 | 59 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [home, defs.SERVICE, commands.UNINSTALL, 60 | args.YES, DEMO_SERVICE]) 61 | sanity_check(result, home) 62 | assert os.path.exists(os.path.join(home, defs.SERVICES)) 63 | assert not os.path.exists(os.path.join(home, defs.SERVICES, DEMO_SERVICE)) 64 | 65 | 66 | @pytest.fixture 67 | def integration_installed(service_installed): 68 | """Prepare honeycomb home path with DEMO_INTEGRATION installed.""" 69 | home = service_installed 70 | 71 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [home, defs.INTEGRATION, 72 | commands.INSTALL, DEMO_INTEGRATION]) 73 | sanity_check(result, home) 74 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [home, defs.INTEGRATION, 75 | commands.CONFIGURE, DEMO_INTEGRATION] + DEMO_INTEGRATION_ARGS.split(" ")) 76 | sanity_check(result, home) 77 | 78 | installed_integration_path = os.path.join(home, defs.INTEGRATIONS, DEMO_INTEGRATION) 79 | assert os.path.exists(os.path.join(installed_integration_path, integrationmanager.defs.ACTIONS_FILE_NAME)) 80 | assert os.path.exists(os.path.join(installed_integration_path, defs.ARGS_JSON)) 81 | 82 | yield home 83 | 84 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [home, defs.INTEGRATION, 85 | commands.UNINSTALL, args.YES, DEMO_INTEGRATION]) 86 | sanity_check(result, home) 87 | assert os.path.exists(os.path.join(home, defs.INTEGRATIONS)) 88 | assert not os.path.exists(os.path.join(home, defs.INTEGRATIONS, DEMO_INTEGRATION)) 89 | 90 | 91 | @pytest.fixture 92 | def running_service(service_installed, request): 93 | """Provide a running instance with :func:`service_installed`.""" 94 | cmdargs = args.COMMON_ARGS + [service_installed, defs.SERVICE, commands.RUN, DEMO_SERVICE] 95 | cmd = RUN_HONEYCOMB + cmdargs + request.param 96 | p = subprocess.Popen(cmd, env=os.environ) 97 | assert wait_until(search_json_log, filepath=os.path.join(service_installed, defs.DEBUG_LOG_FILE), total_timeout=10, 98 | key="message", value="Starting Simple HTTP service on port: {}".format(DEMO_SERVICE_PORT)) 99 | sanity_check(home=service_installed) 100 | 101 | yield service_installed 102 | 103 | p.send_signal(signal.SIGINT) 104 | p.wait() 105 | 106 | try: 107 | rsession.get("http://localhost:{}".format(DEMO_SERVICE_PORT)) 108 | assert False, "Service is still available (make sure to properly kill it before repeating test)" 109 | except requests.exceptions.ConnectionError: 110 | assert True 111 | 112 | 113 | @pytest.fixture 114 | def running_service_with_integration(integration_installed, request): 115 | """Provide a running instance with :func:`integration_installed` and DEMO_INTEGRATION.""" 116 | cmdargs = args.COMMON_ARGS + [integration_installed, defs.SERVICE, commands.RUN, DEMO_SERVICE, 117 | args.INTEGRATION, DEMO_INTEGRATION] 118 | cmd = RUN_HONEYCOMB + cmdargs + request.param 119 | p = subprocess.Popen(cmd, env=os.environ) 120 | sanity_check(home=integration_installed) 121 | assert wait_until(search_json_log, filepath=os.path.join(integration_installed, defs.DEBUG_LOG_FILE), key="message", 122 | total_timeout=10, value="Starting Simple HTTP service on port: {}".format(DEMO_SERVICE_PORT)) 123 | yield integration_installed 124 | p.send_signal(signal.SIGINT) 125 | p.wait() 126 | 127 | try: 128 | rsession.get("http://localhost:{}".format(DEMO_SERVICE_PORT)) 129 | assert False, "Service is still available (make sure to properly kill it before repeating test)" 130 | except requests.exceptions.ConnectionError: 131 | assert True 132 | 133 | 134 | @pytest.fixture 135 | def running_daemon(service_installed, request): 136 | """Provide a running daemon with :func:`service_installed`.""" 137 | cmdargs = args.COMMON_ARGS + [service_installed, defs.SERVICE, commands.RUN, args.DAEMON, DEMO_SERVICE] 138 | cmd = RUN_HONEYCOMB + cmdargs + request.param 139 | p = subprocess.Popen(cmd, env=os.environ) 140 | p.wait() 141 | sanity_check(home=service_installed) 142 | assert p.returncode == 0 143 | assert wait_until(search_json_log, filepath=os.path.join(service_installed, defs.DEBUG_LOG_FILE), total_timeout=10, 144 | key="message", value="Starting Simple HTTP service on port: {}".format(DEMO_SERVICE_PORT)) 145 | 146 | assert rsession.get("http://localhost:{}".format(DEMO_SERVICE_PORT)) 147 | 148 | yield service_installed 149 | 150 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [service_installed, defs.SERVICE, 151 | commands.STOP, DEMO_SERVICE]) 152 | sanity_check(result, service_installed) 153 | assert wait_until(search_json_log, filepath=os.path.join(service_installed, defs.DEBUG_LOG_FILE), total_timeout=10, 154 | key="message", value="Simple HTTP service stopped") 155 | 156 | try: 157 | rsession.get("http://localhost:{}".format(DEMO_SERVICE_PORT)) 158 | assert False, "Service is still available (make sure to properly kill it before repeating test)" 159 | except requests.exceptions.ConnectionError: 160 | assert True 161 | 162 | 163 | def test_cli_help(): 164 | """Test honeycomb launches without an error (tests :func:`honeycomb.cli`).""" 165 | result = CliRunner().invoke(cli, args=[args.HELP]) 166 | sanity_check(result) 167 | 168 | 169 | def test_service_command(): 170 | """Test honeycomb service command.""" 171 | result = CliRunner().invoke(cli, args=[defs.SERVICE, args.HELP]) 172 | sanity_check(result) 173 | 174 | 175 | def test_integration_command(): 176 | """Test honeycomb integration command.""" 177 | result = CliRunner().invoke(cli, args=[defs.INTEGRATION, args.HELP]) 178 | sanity_check(result) 179 | 180 | 181 | def test_invalid_command(): 182 | """Test honeycomb invalid command.""" 183 | result = CliRunner().invoke(cli, args=["nosuchcommand", args.HELP]) 184 | sanity_check(result, fail=True) 185 | 186 | 187 | def test_invalid_subcommand(): 188 | """Test honeycomb invalid command.""" 189 | result = CliRunner().invoke(cli, args=[defs.SERVICE, "nosuchsubcommand", args.HELP]) 190 | sanity_check(result, fail=True) 191 | 192 | 193 | @pytest.mark.dependency(name="service_install_uninstall") 194 | def test_service_install_uninstall(service_installed): 195 | """Test the service install and uninstall commands. 196 | 197 | This is just mock test for :func:`service_installed` fixture 198 | """ 199 | assert service_installed 200 | 201 | 202 | def test_service_list_nothing_installed(tmpdir): 203 | """Test the service list command when nothing is installed.""" 204 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [str(tmpdir), defs.SERVICE, "list"]) 205 | sanity_check(result, str(tmpdir)) 206 | 207 | 208 | def test_service_list_remote(tmpdir): 209 | """Test the service list command and also show services from remote repository.""" 210 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [str(tmpdir), defs.SERVICE, "list", "--remote"]) 211 | sanity_check(result, str(tmpdir)) 212 | assert DEMO_SERVICE in result.output, result.output 213 | 214 | 215 | @pytest.mark.dependency(depends=["service_install_uninstall"]) 216 | def test_service_list_local(service_installed): 217 | """Test the service list command with a service installed.""" 218 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [service_installed, defs.SERVICE, "list"]) 219 | sanity_check(result, service_installed) 220 | assert "{} (Ports: {}) [Alerts: {}]".format(DEMO_SERVICE, DEMO_SERVICE_PORTS, 221 | DEMO_SERVICE_ALERT) in result.output, result.output 222 | 223 | 224 | def test_service_show_remote_not_installed(tmpdir): 225 | """Test the service show command to show information from remote repository.""" 226 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [str(tmpdir), defs.SERVICE, commands.SHOW, 227 | DEMO_SERVICE]) 228 | sanity_check(result, str(tmpdir)) 229 | assert "Installed: False" in result.output, result.output 230 | assert "Name: {}".format(DEMO_SERVICE) in result.output, result.output 231 | 232 | 233 | @pytest.mark.dependency(depends=["service_install_uninstall"]) 234 | def test_service_show_local_installed(service_installed): 235 | """Test the service show command to show information about locally installe service.""" 236 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [service_installed, defs.SERVICE, 237 | commands.SHOW, DEMO_SERVICE]) 238 | sanity_check(result, service_installed) 239 | assert "Installed: True" in result.output, result.output 240 | assert "Name: {}".format(DEMO_SERVICE) in result.output, result.output 241 | 242 | 243 | def test_service_show_nonexistent(tmpdir): 244 | """Test the service test command to fail on nonexistent service.""" 245 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [str(tmpdir), defs.SERVICE, commands.SHOW, 246 | "this_should_never_exist"]) 247 | sanity_check(result, str(tmpdir), fail=True) 248 | 249 | 250 | @pytest.mark.dependency(depends=["service_install_uninstall"]) 251 | def test_service_show_args(service_installed): 252 | """Test the service run command show-args.""" 253 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [service_installed, defs.SERVICE, commands.RUN, 254 | DEMO_SERVICE, "--show-args"]) 255 | sanity_check(result, service_installed, fail=True) 256 | args_format = "{:20} {:10} {:^15} {:^10} {:25}" 257 | title = args_format.format(defs.NAME.upper(), defs.TYPE.upper(), defs.DEFAULT.upper(), 258 | defs.REQUIRED.upper(), defs.DESCRIPTION.upper()) 259 | 260 | assert title in result.output, result.output 261 | 262 | 263 | @pytest.mark.dependency(name="service_arg_missing", depends=["service_install_uninstall"]) 264 | def test_service_missing_arg(service_installed): 265 | """Test the service run command with missing service parameter.""" 266 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [service_installed, defs.SERVICE, 267 | commands.RUN, DEMO_SERVICE]) 268 | sanity_check(result, service_installed, fail=True) 269 | assert "'port' is missing" in result.output, result.output 270 | 271 | 272 | @pytest.mark.dependency(name="service_arg_bad_int", depends=["service_install_uninstall"]) 273 | def test_service_arg_bad_int(service_installed): 274 | """Test the service run with invalid int.""" 275 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [service_installed, defs.SERVICE, 276 | commands.RUN, DEMO_SERVICE, "port=noint"]) 277 | sanity_check(result, service_installed, fail=True) 278 | assert "Bad value for port=noint (must be integer)" in result.output, result.output 279 | 280 | 281 | @pytest.mark.dependency(name="service_arg_bad_bool", depends=["service_install_uninstall"]) 282 | def test_service_arg_bad_bool(service_installed): 283 | """Test the service run with invalid boolean.""" 284 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [service_installed, defs.SERVICE, 285 | commands.RUN, DEMO_SERVICE, DEMO_SERVICE_ARGS, "threading=notbool"]) 286 | sanity_check(result, service_installed, fail=True) 287 | assert "Bad value for threading=notbool (must be boolean)" in result.output, result.output 288 | 289 | 290 | @pytest.mark.dependency(name="service_run", 291 | depends=["service_arg_missing", "service_arg_bad_int", "service_arg_bad_bool"]) 292 | @pytest.mark.parametrize("running_service", [[DEMO_SERVICE_ARGS]], indirect=["running_service"]) 293 | def test_service_run(running_service): 294 | """Test the service run command and validate the service started properly.""" 295 | assert wait_until(search_json_log, filepath=os.path.join(running_service, defs.DEBUG_LOG_FILE), total_timeout=10, 296 | key="message", value="Starting Simple HTTP service on port: {}".format(DEMO_SERVICE_PORT)) 297 | 298 | r = rsession.get("http://localhost:{}".format(DEMO_SERVICE_PORT)) 299 | assert "Welcome to nginx!" in r.text 300 | 301 | 302 | @pytest.mark.dependency(name="service_daemon", depends=["service_run"]) 303 | @pytest.mark.parametrize("running_daemon", [[DEMO_SERVICE_ARGS]], indirect=["running_daemon"]) 304 | def test_service_daemon(running_daemon): 305 | """Test the service run command in daemon mode.""" 306 | r = rsession.get("http://localhost:{}".format(DEMO_SERVICE_PORT)) 307 | assert "Welcome to nginx!" in r.text 308 | 309 | 310 | @pytest.mark.dependency(depends=["service_daemon"]) 311 | @pytest.mark.parametrize("running_daemon", [[DEMO_SERVICE_ARGS]], indirect=["running_daemon"]) 312 | def test_service_status(running_daemon): 313 | """Test the service status command on a running daemon.""" 314 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [running_daemon, defs.SERVICE, commands.STATUS, 315 | DEMO_SERVICE]) 316 | sanity_check(result, running_daemon) 317 | assert "{} - running".format(DEMO_SERVICE) in result.output, result.output 318 | 319 | 320 | @pytest.mark.dependency(depends=["service_daemon"]) 321 | @pytest.mark.parametrize("running_daemon", [[DEMO_SERVICE_ARGS]], indirect=["running_daemon"]) 322 | def test_service_status_all(running_daemon): 323 | """Test the service status command on all running services.""" 324 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [running_daemon, defs.SERVICE, commands.STATUS, 325 | args.SHOW_ALL]) 326 | sanity_check(result, running_daemon) 327 | assert "{} - running".format(DEMO_SERVICE) in result.output, result.output 328 | 329 | 330 | def test_service_status_nonexistent(tmpdir): 331 | """Test the service status command on a nonexistent service.""" 332 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [str(tmpdir), defs.SERVICE, commands.STATUS, 333 | "nosuchservice"]) 334 | sanity_check(result, str(tmpdir), fail=True) 335 | assert "nosuchservice - no such service" in result.output, result.output 336 | 337 | 338 | def test_service_status_no_service(tmpdir): 339 | """Test the service status command without a service name.""" 340 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [str(tmpdir), defs.SERVICE, commands.STATUS]) 341 | sanity_check(result, str(tmpdir), fail=True) 342 | assert "You must specify a service name" in result.output, result.output 343 | 344 | 345 | @pytest.mark.dependency(depends=["service_daemon"]) 346 | @pytest.mark.parametrize("running_daemon", [[DEMO_SERVICE_ARGS]], indirect=["running_daemon"]) 347 | def test_service_test(running_daemon): 348 | """Test the service test command.""" 349 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [running_daemon, defs.SERVICE, commands.TEST, 350 | DEMO_SERVICE]) 351 | sanity_check(result, running_daemon) 352 | assert "alert tested successfully" in result.output, result.output 353 | 354 | 355 | @pytest.mark.dependency(name="integration_install_uninstall", depends=["service_install_uninstall"]) 356 | def test_integration_install_uninstall(integration_installed): 357 | """Test the integration install and uninstall commands. 358 | 359 | This is just mock test for :func:`integration_installed` fixture 360 | """ 361 | assert integration_installed 362 | 363 | 364 | def test_integration_list_nothing_installed(tmpdir): 365 | """Test the integration list command when nothing is installed.""" 366 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [str(tmpdir), defs.INTEGRATION, "list"]) 367 | sanity_check(result, str(tmpdir)) 368 | 369 | 370 | def test_integration_list_remote(tmpdir): 371 | """Test the integration list command and also show integrations from remote repository.""" 372 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [str(tmpdir), defs.INTEGRATION, "list", "--remote"]) 373 | sanity_check(result, str(tmpdir)) 374 | assert DEMO_INTEGRATION in result.output, result.output 375 | 376 | 377 | @pytest.mark.dependency(depends=["integration_install_uninstall"]) 378 | def test_integration_list_local(integration_installed): 379 | """Test the integration list command with a integration installed.""" 380 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [integration_installed, defs.INTEGRATION, "list"]) 381 | sanity_check(result, integration_installed) 382 | assert DEMO_INTEGRATION in result.output, result.output 383 | 384 | 385 | def test_integration_show_remote_not_installed(tmpdir): 386 | """Test the integration show command to show information from remote repository.""" 387 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [str(tmpdir), defs.INTEGRATION, commands.SHOW, 388 | DEMO_INTEGRATION]) 389 | sanity_check(result, str(tmpdir)) 390 | assert "Installed: False" in result.output, result.output 391 | assert "Name: {}".format(DEMO_INTEGRATION.capitalize()) in result.output, result.output 392 | 393 | 394 | @pytest.mark.dependency(depends=["integration_install_uninstall"]) 395 | def test_integration_show_local_installed(integration_installed): 396 | """Test the integration show command to show information about locally installe integration.""" 397 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [integration_installed, defs.INTEGRATION, 398 | commands.SHOW, DEMO_INTEGRATION]) 399 | sanity_check(result, integration_installed) 400 | assert "Installed: True" in result.output, result.output 401 | assert "Name: {}".format(DEMO_INTEGRATION) in result.output, result.output 402 | 403 | 404 | def test_integration_show_nonexistent(tmpdir): 405 | """Test the integration test command to fail on nonexistent integration.""" 406 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [str(tmpdir), defs.INTEGRATION, commands.SHOW, 407 | "this_should_never_exist"]) 408 | sanity_check(result, str(tmpdir), fail=True) 409 | 410 | 411 | @pytest.mark.dependency(name="integration_configured", depends=["integration_install_uninstall"]) 412 | def test_integration_configure(integration_installed): 413 | """Test the integration configure command.""" 414 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [integration_installed, defs.INTEGRATION, 415 | commands.CONFIGURE, DEMO_INTEGRATION] + DEMO_INTEGRATION_ARGS.split(" ")) 416 | sanity_check(result, integration_installed) 417 | assert "has been configured, make sure to test it with" in result.output, result.output 418 | 419 | 420 | @pytest.mark.dependency(depends=["integration_configured"]) 421 | def test_integration_test(integration_installed): 422 | """Test the integration test command.""" 423 | CliRunner().invoke(cli, args=args.COMMON_ARGS + [integration_installed, defs.INTEGRATION, 424 | commands.CONFIGURE, DEMO_INTEGRATION] + DEMO_INTEGRATION_ARGS.split(" ")) 425 | result = CliRunner().invoke(cli, args=args.COMMON_ARGS + [integration_installed, defs.INTEGRATION, 426 | commands.TEST, DEMO_INTEGRATION]) 427 | sanity_check(result, integration_installed, fail=True) # TODO: consider replacing with an integration has a test 428 | # assert "alert tested successfully" in result.output, result.output 429 | 430 | 431 | @pytest.mark.dependency(name="integration_run", depends=["integration_configured", "service_run"]) 432 | @pytest.mark.parametrize("running_service_with_integration", [[DEMO_SERVICE_ARGS]], 433 | indirect=["running_service_with_integration"]) 434 | def test_integration_run(running_service_with_integration, syslogd): 435 | """Test the integration test command.""" 436 | assert running_service_with_integration 437 | rsession.get("http://localhost:{}".format(DEMO_SERVICE_PORT)) 438 | assert wait_until(search_file_log, filepath=str(syslogd), method="find", args="GET /", total_timeout=3) 439 | 440 | 441 | # @pytest.mark.dependency(depends=["service_daemon"]) 442 | # @pytest.mark.parametrize("running_daemon", [[DEMO_SERVICE_ARGS]], indirect=["running_daemon"]) 443 | # def test_service_logs(running_daemon): 444 | # """Test the service logs command.""" 445 | # teststring = "__LOGS_TEST__" 446 | # nlines = 5 447 | # 448 | # # generate lots of logs 449 | # for i in range(nlines * 2): 450 | # rsession.get("http://localhost:{}/{}".format(DEMO_SERVICE_PORT, teststring)) 451 | # 452 | # args_no_verbose = list(args.COMMON_ARGS) 453 | # args_no_verbose.remove(args.VERBOSE) 454 | # result = CliRunner().invoke(cli, args=args_no_verbose + [running_daemon, defs.SERVICE, commands.LOGS, 455 | # args.NUM, nlines, DEMO_SERVICE]) 456 | # subprocess.call(['cat', os.path.join(running_daemon, 'services', 'simple_http', 'logs', 'stdout.log')]) 457 | # sanity_check(result, running_daemon) 458 | # assert teststring in result.output, "\n{}\n{}".format(result.output, repr(result.exception)) 459 | # # when honeycomb exits after printing the logs there's an additional empty line, we exclude it 460 | # log_rows = len(result.output.split("\n")) - 1 461 | # # if we are running as root the output will have an additional line of warning 462 | # assert log_rows == nlines or log_rows == nlines + 1, "\n{}\n{}".format(result.output, repr(result.exception)) 463 | # assert False 464 | 465 | 466 | # @pytest.mark.dependency(depends=["service_daemon"]) 467 | # @pytest.mark.parametrize("running_daemon", [[DEMO_SERVICE_ARGS]], indirect=["running_daemon"]) 468 | # def test_service_logs_follow(running_daemon): 469 | # """Test the service logs command with follow.""" 470 | # # TODO: Test service logs -f 471 | # # Consider https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python 472 | # def wait_for_output(p, needle): 473 | # i = 0 474 | # output = "" 475 | # success = False 476 | # while i < 10: # wait 5 seconds 477 | # output += p.stdout.read() 478 | # if "Starting Simple HTTP service" in output: 479 | # success = True 480 | # break 481 | # time.sleep(0.5) 482 | # i += 1 483 | # assert success, output 484 | # 485 | # teststring = "__LOGS_TEST__" 486 | # args_no_verbose = list(args.COMMON_ARGS) 487 | # args_no_verbose.remove(args.VERBOSE) 488 | # cmdargs = args_no_verbose + [running_daemon, defs.SERVICE, commands.LOGS, args.FOLLOW, DEMO_SERVICE] 489 | # p = subprocess.Popen(RUN_HONEYCOMB + cmdargs, env=os.environ, stdout=subprocess.PIPE) 490 | # wait_for_output(p, "Starting Simple HTTP service") 491 | # 492 | # rsession.get("http://localhost:{}/{}".format(DEMO_SERVICE_PORT, teststring)) 493 | # wait_for_output(p, teststring) 494 | # 495 | # p.send_signal(signal.SIGINT) 496 | # assert wait_until(p.wait) 497 | 498 | 499 | @pytest.mark.dependency(depends=["service_run", "integration_run"]) 500 | def test_service_config(tmpdir): 501 | """Test honeycomb with a yml config.""" 502 | home = str(tmpdir) 503 | configfile = tmpdir.join("honeycomb.yml") 504 | 505 | sampleconfig = """ 506 | --- 507 | version: 1 508 | 509 | services: 510 | simple_http: 511 | parameters: 512 | port: 8888 513 | 514 | integrations: 515 | syslog: 516 | parameters: 517 | address: "127.0.0.1" 518 | port: 5514 519 | protocol: tcp 520 | """ 521 | 522 | configfile.write(sampleconfig) 523 | cmdargs = args.COMMON_ARGS + [home, args.CONFIG, str(configfile)] 524 | p = subprocess.Popen(RUN_HONEYCOMB + cmdargs, env=os.environ, stdout=subprocess.PIPE) 525 | assert wait_until(search_json_log, filepath=os.path.join(home, defs.DEBUG_LOG_FILE), total_timeout=20, 526 | key="message", value="Starting Simple HTTP service on port: {}".format(DEMO_SERVICE_PORT)) 527 | p.send_signal(signal.SIGINT) 528 | sanity_check(home=home) 529 | output = str(p.stdout.read()) 530 | assert "Launching simple_http" in output 531 | assert "Adding integration syslog" in output 532 | assert "syslog has been configured" in output 533 | -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb test utilities.""" 3 | -------------------------------------------------------------------------------- /tests/utils/defs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb test constants.""" 3 | 4 | 5 | class commands(): 6 | """Plugin commands.""" 7 | 8 | RUN = "run" 9 | LOGS = "logs" 10 | SHOW = "show" 11 | TEST = "test" 12 | STOP = "stop" 13 | LIST = "list" 14 | STATUS = "status" 15 | INSTALL = "install" 16 | UNINSTALL = "uninstall" 17 | CONFIGURE = "configure" 18 | 19 | 20 | class args(): 21 | """Plugin arguments.""" 22 | 23 | YES = "--yes" 24 | NUM = "--num" 25 | HOME = "--home" 26 | HELP = "--help" 27 | CONFIG = "--config" 28 | FOLLOW = "--follow" 29 | DAEMON = "--daemon" 30 | VERBOSE = "--verbose" 31 | IAMROOT = "--iamroot" 32 | SHOW_ALL = "--show-all" 33 | INTEGRATION = "--integration" 34 | COMMON_ARGS = [VERBOSE, IAMROOT, HOME] 35 | -------------------------------------------------------------------------------- /tests/utils/syslog.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Syslog utility for tests.""" 3 | 4 | import logging 5 | import threading 6 | 7 | from six.moves import socketserver 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | class SyslogUDPHandler(socketserver.BaseRequestHandler): 13 | """Syslog UDP dummy handler.""" 14 | 15 | outputHandle = None 16 | 17 | def handle(self): 18 | """Handle incoming data by logging to debug and writing to logfile.""" 19 | data = bytes.decode(self.request[0].strip()) 20 | logger.debug(data) 21 | self.outputHandle.write(data) 22 | self.outputHandle.flush() 23 | 24 | 25 | def runSyslogServer(host, port, logfile): 26 | """Run a dummy syslog server. 27 | 28 | :param host: IP address to listen 29 | :param port: Port to listen 30 | :param logfile: File handle used to write incoming logs 31 | """ 32 | logfilehandle = open(logfile, "w+") 33 | handler = SyslogUDPHandler 34 | handler.outputHandle = logfilehandle 35 | syslogd = socketserver.UDPServer((host, port), handler) 36 | 37 | def serve(): 38 | syslogd.serve_forever() 39 | 40 | thread = threading.Thread(target=serve) 41 | thread.start() 42 | return syslogd 43 | -------------------------------------------------------------------------------- /tests/utils/test_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Honeycomb service tests.""" 3 | 4 | from __future__ import absolute_import, unicode_literals 5 | 6 | import os 7 | import json 8 | 9 | from honeycomb.defs import DEBUG_LOG_FILE 10 | 11 | 12 | def sanity_check(result=None, home=None, fail=False): 13 | """Run a generic sanity check for CLI command.""" 14 | if result: 15 | assert (result.exit_code == 0 if not fail else not 0), "\n{}\n{}".format(result.output, repr(result.exception)) 16 | assert (result.exception != fail), "{}\n\n\n{}".format(result.output, repr(result.exception)) 17 | if home: 18 | assert json_log_is_valid(home) 19 | 20 | 21 | def json_log_is_valid(path): 22 | """Validate a json file. 23 | 24 | :param path: valid path to json file 25 | """ 26 | with open(os.path.join(str(path), DEBUG_LOG_FILE), "r") as fh: 27 | for line in fh.readlines(): 28 | try: 29 | json.loads(line) 30 | except json.decoder.JSONDecodeError: 31 | return False 32 | return True 33 | 34 | 35 | def search_file_log(filepath, method, args): 36 | """Search a log file by executing a string method on its lines. 37 | 38 | :param filepath: Valid path to a log file 39 | :param method: A valid :py:module:`string` method 40 | :param args: Arguments to above method 41 | :returns: First matching line in log file 42 | """ 43 | with open(filepath, "r") as fh: 44 | for line in fh.readlines(): 45 | cmd = getattr(line, method) 46 | if cmd(args): 47 | return line 48 | return False 49 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist=flake8, py27, py36 3 | 4 | [testenv] 5 | setenv = 6 | PYTHONDONTWRITEBYTECODE=1 7 | PYTHONUNBUFFERED=1 8 | passenv = 9 | CI 10 | TRAVIS 11 | TRAVIS_* 12 | deps= 13 | -r requirements-dev.txt 14 | 15 | commands = 16 | py.test -v --cov honeycomb {posargs} 17 | codecov 18 | 19 | [testenv:flake8] 20 | basepython = python3 21 | deps = 22 | -r requirements-dev.txt 23 | commands = 24 | flake8 honeycomb tests --max-line-length=120 25 | 26 | [testenv:sphinx] 27 | basepython = python3 28 | changedir=docs 29 | deps = 30 | -r requirements-dev.txt 31 | commands = 32 | make html 33 | 34 | [flake8] 35 | exclude = .*,__pycache__,build,dist 36 | max-line-length = 120 37 | ignore = D107,D105,W503,W504 38 | --------------------------------------------------------------------------------