├── .github └── workflows │ └── main.yml ├── .gitignore ├── .readthedocs.yaml ├── CHANGES.rst ├── COPYRIGHT.txt ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── docs ├── Makefile ├── conf.py ├── crashmail.rst ├── crashmailbatch.rst ├── crashsms.rst ├── development.rst ├── fatalmailbatch.rst ├── httpok.rst ├── index.rst └── memmon.rst ├── setup.cfg ├── setup.py ├── superlance ├── __init__.py ├── compat.py ├── crashmail.py ├── crashmailbatch.py ├── crashsms.py ├── fatalmailbatch.py ├── httpok.py ├── memmon.py ├── process_state_email_monitor.py ├── process_state_monitor.py ├── tests │ ├── __init__.py │ ├── dummy.py │ ├── test_crashmail.py │ ├── test_crashmailbatch.py │ ├── test_crashsms.py │ ├── test_fatalmailbatch.py │ ├── test_httpok.py │ ├── test_memmon.py │ ├── test_process_state_email_monitor.py │ └── test_process_state_monitor.py └── timeoutconn.py └── tox.ini /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Run all tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | tests: 7 | runs-on: ubuntu-20.04 8 | strategy: 9 | fail-fast: false 10 | matrix: 11 | python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", 3.11] 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | 16 | - name: Set up Python ${{ matrix.python-version }} 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: ${{ matrix.python-version }} 20 | 21 | - name: Show Python version 22 | run: python -V 23 | 24 | - name: Set TOXENV based on Python version 25 | run: python -c 'import sys; print("TOXENV=py%d%d" % (sys.version_info.major, sys.version_info.minor))' | tee -a $GITHUB_ENV 26 | 27 | - name: Install dependencies 28 | run: pip install virtualenv tox 29 | 30 | - name: Run the unit tests 31 | run: tox 32 | 33 | docs: 34 | runs-on: ubuntu-20.04 35 | 36 | steps: 37 | - uses: actions/checkout@v3 38 | 39 | - name: Set up Python ${{ matrix.python-version }} 40 | uses: actions/setup-python@v4 41 | with: 42 | python-version: "3.8" 43 | 44 | - name: Install dependencies 45 | run: pip install virtualenv tox>=4.0.0 46 | 47 | - name: Build the docs 48 | run: TOXENV=docs tox 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.egg 3 | *.egg-info 4 | *.pyc 5 | *.pyo 6 | *.swp 7 | .DS_Store 8 | .coverage 9 | .eggs/ 10 | .tox/ 11 | build/ 12 | docs/_build/ 13 | dist/ 14 | env*/ 15 | htmlcov/ 16 | tmp/ 17 | coverage.xml 18 | nosetests.xml 19 | 20 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/conf.py 17 | 18 | # We recommend specifying your dependencies to enable reproducible builds: 19 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 20 | # python: 21 | # install: 22 | # - requirements: docs/requirements.txt 23 | 24 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | 2.0.1.dev0 (Next Release) 2 | ------------------------- 3 | 4 | 2.0.0 (2021-12-26) 5 | ------------------ 6 | 7 | - Support for Python 2.6 has been dropped. On Python 2, Superlance 8 | now requires Python 2.7. 9 | 10 | - Support for Python 3.2 and 3.3 has been dropped. On Python 3, Superlance 11 | now requires Python 3.4 or later. 12 | 13 | - Fixed a bug introduced in 0.10 where if the timeout value is shorter 14 | than the time to wait between retries, the httpok check never executed. 15 | Issue #110. 16 | 17 | - Fixed a bug where ``crashmailbatch`` and ``fatalmatchbatch`` did not set 18 | the intended default subject. Patch by Joe Portela. 19 | 20 | - Added a new ``--tls`` option to ``crashmailbatch``, ``fatalmailbath``, and 21 | ``crashsms`` to use Transport Layer Security (TLS). Patch by Zhe Li. 22 | 23 | 1.0.0 (2016-10-02) 24 | ------------------ 25 | 26 | - Support for Python 2.5 has been dropped. On Python 2, Superlance 27 | now requires Python 2.6 or later. 28 | 29 | - Support for Python 3 has been added. On Python 3, Superlance 30 | requires Python 3.2 or later. 31 | 32 | - Fixed parsing of ``-n`` and ``--name`` options in ``httpok``. Patch 33 | by DenisBY. 34 | 35 | 0.14 (2016-09-24) 36 | ----------------- 37 | 38 | - Fixed docs build. 39 | 40 | 0.13 (2016-09-05) 41 | ----------------- 42 | 43 | - ``httpok`` now allows multiple expected status codes to be specified. Patch 44 | by valmiRe. 45 | 46 | - ``httpok`` now has a ``--name`` option like ``memmon``. 47 | 48 | - All commands now return exit status 0 from ``--help``. 49 | 50 | 0.12 (2016-09-03) 51 | ----------------- 52 | 53 | - Fixed ``crashmail`` parsing of ``--optionalheader``. Patch by Matt Dziuban. 54 | 55 | 0.11 (2014-08-15) 56 | ----------------- 57 | 58 | - Added support for ``memmon`` to check against cumulative RSS of a process 59 | and all its child processes. Patch by Lukas Graf. 60 | 61 | - Fixed a bug introduced in 0.9 where the ``-u`` and ``-n`` options in 62 | ``memmon`` were parsed incorrectly. Patch by Harald Friessnegger. 63 | 64 | 0.10 (2014-07-08) 65 | ----------------- 66 | 67 | - Honor timeout in httok checks even on trying the connection. 68 | Without it, processes that take make than 60 seconds to accept connections 69 | and http_ok with TICK_60 events cause a permanent restart of the process. 70 | 71 | - ``httpok`` now sends a ``User-Agent`` header of ``httpok``. 72 | 73 | - Removed ``setuptools`` from the ``requires`` list in ``setup.py`` because 74 | it caused installation issues on some systems. 75 | 76 | 0.9 (2013-09-18) 77 | ---------------- 78 | 79 | - Added license. 80 | 81 | - Fixed bug in cmd line option validator for ProcessStateEmailMonitor 82 | Bug report by Val Jordan 83 | 84 | - Added ``-u`` option to memmon the only send an email in case the restarted 85 | process' uptime (in seconds) is below this limit. This is useful to only 86 | get notified if a processes gets restarted too frequently. 87 | Patch by Harald Friessnegger. 88 | 89 | 0.8 (2013-05-26) 90 | ---------------- 91 | 92 | - Superlance will now refuse to install on an unsupported version of Python. 93 | 94 | - Allow SMTP credentials to be supplied to ProcessStateEmailMonitor 95 | Patch by Steven Davidson. 96 | 97 | - Added ``-n`` option to memmon that adds this name to the email 98 | subject to identify which memmon process restarted a process. 99 | Useful in case you run multiple supervisors that control 100 | different processes with the same name. 101 | Patch by Harald Friessnegger. 102 | 103 | - ProcessStateEmailMonitor now adds Date and Message-ID headers to emails. 104 | Patch by Andrei Vereha. 105 | 106 | 0.7 (2012-08-22) 107 | ---------------- 108 | 109 | - The ``crashmailbatch --toEmail`` option now accepts a comma-separated 110 | list of email addresses. 111 | 112 | 0.6 (2011-08-27) 113 | ---------------- 114 | 115 | - Separated unit tests into their own files 116 | 117 | - Created ``fatalmailbatch`` plugin 118 | 119 | - Created ``crashmailbatch`` plugin 120 | 121 | - Sphinxified documentation. 122 | 123 | - Fixed ``test_suite`` to use the correct module name in setup.py. 124 | 125 | - Fixed the tests for ``memmon`` to import the correct module. 126 | 127 | - Applied patch from Sam Bartlett: processes which are not autostarted 128 | have pid "0". This was crashing ``memmon``. 129 | 130 | - Add ``smtpHost`` command line flag to ``mailbatch`` processors. 131 | 132 | - Added ``crashsms`` from Juan Batiz-Benet 133 | 134 | - Converted ``crashmailbatch`` and friends from camel case to pythonic style 135 | 136 | - Fixed a bug where ``httpok`` would crash with the ``-b`` (in-body) 137 | option. Patch by Joaquin Cuenca Abela. 138 | 139 | - Fixed a bug where ``httpok`` would not handle a URL with a query string 140 | correctly. Patch by Joaquin Cuenca Abela. 141 | 142 | - Fixed a bug where ``httpok`` would not handle process names with a 143 | group ("group:process") properly. Patch by Joaquin Cuenca Abela. 144 | 145 | 146 | 0.5 (2009-05-24) 147 | ---------------- 148 | 149 | - Added the ``memmon`` plugin, originally bundled with supervisor and 150 | now moved to superlance. 151 | 152 | 153 | 0.4 (2009-02-11) 154 | ---------------- 155 | 156 | - Added ``eager`` and ``not-eager`` options to the ``httpok`` plugin. 157 | 158 | If ``not-eager`` is set, and no process being monitored is in the 159 | ``RUNNING`` state, skip the URL check / mail message. 160 | 161 | 162 | 0.3 (2008-12-10) 163 | ---------------- 164 | 165 | - Added ``gcore`` and ``coredir`` options to the ``httpok`` plugin. 166 | 167 | 168 | 0.2 (2008-11-21) 169 | ---------------- 170 | 171 | - Added the ``crashmail`` plugin. 172 | 173 | 174 | 0.1 (2008-09-18) 175 | ---------------- 176 | 177 | - Initial release 178 | -------------------------------------------------------------------------------- /COPYRIGHT.txt: -------------------------------------------------------------------------------- 1 | Superlance is Copyright (c) 2008-2013 Agendaless Consulting and Contributors. 2 | (http://www.agendaless.com), All Rights Reserved 3 | 4 | This software is subject to the provisions of the license at 5 | http://www.repoze.org/LICENSE.txt . A copy of this license should 6 | accompany this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND 7 | ANY AND ALL EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, 8 | BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, 9 | MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS FOR A PARTICULAR 10 | PURPOSE. 11 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Superlance is licensed under the following license: 2 | 3 | A copyright notice accompanies this license document that identifies 4 | the copyright holders. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | 1. Redistributions in source code must retain the accompanying 11 | copyright notice, this list of conditions, and the following 12 | disclaimer. 13 | 14 | 2. Redistributions in binary form must reproduce the accompanying 15 | copyright notice, this list of conditions, and the following 16 | disclaimer in the documentation and/or other materials provided 17 | with the distribution. 18 | 19 | 3. Names of the copyright holders must not be used to endorse or 20 | promote products derived from this software without prior 21 | written permission from the copyright holders. 22 | 23 | 4. If any files are modified, you must cause the modified files to 24 | carry prominent notices stating that you changed the files and 25 | the date of any change. 26 | 27 | Disclaimer 28 | 29 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND 30 | ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 31 | TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 32 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 33 | HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 34 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 35 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 36 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 37 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 38 | TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 39 | THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 40 | SUCH DAMAGE. 41 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGES.rst 2 | include COPYRIGHT.txt 3 | include LICENSE.txt 4 | include README.rst 5 | include docs/Makefile 6 | recursive-include docs *.py *.rst 7 | recursive-exclude docs/_build * 8 | 9 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | superlance README 2 | ================= 3 | 4 | Superlance is a package of plugin utilities for monitoring and controlling 5 | processes that run under `supervisor `_. 6 | 7 | Please see ``docs/index.rst`` for complete documentation. 8 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " pickle to make pickle files" 22 | @echo " json to make JSON files" 23 | @echo " htmlhelp to make HTML files and a HTML help project" 24 | @echo " qthelp to make HTML files and a qthelp project" 25 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 26 | @echo " changes to make an overview of all changed/added/deprecated items" 27 | @echo " linkcheck to check all external links for integrity" 28 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 29 | 30 | clean: 31 | -rm -rf $(BUILDDIR)/* 32 | 33 | html: 34 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 35 | @echo 36 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 37 | 38 | dirhtml: 39 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 40 | @echo 41 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 42 | 43 | pickle: 44 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 45 | @echo 46 | @echo "Build finished; now you can process the pickle files." 47 | 48 | json: 49 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 50 | @echo 51 | @echo "Build finished; now you can process the JSON files." 52 | 53 | htmlhelp: 54 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 55 | @echo 56 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 57 | ".hhp project file in $(BUILDDIR)/htmlhelp." 58 | 59 | qthelp: 60 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 61 | @echo 62 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 63 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 64 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/superlance.qhcp" 65 | @echo "To view the help file:" 66 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/superlance.qhc" 67 | 68 | latex: 69 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 70 | @echo 71 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 72 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 73 | "run these through (pdf)latex." 74 | 75 | changes: 76 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 77 | @echo 78 | @echo "The overview file is in $(BUILDDIR)/changes." 79 | 80 | linkcheck: 81 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 82 | @echo 83 | @echo "Link check complete; look for any errors in the above output " \ 84 | "or in $(BUILDDIR)/linkcheck/output.txt." 85 | 86 | doctest: 87 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 88 | @echo "Testing of doctests in the sources finished, look at the " \ 89 | "results in $(BUILDDIR)/doctest/output.txt." 90 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # superlance documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Jun 10 11:55:43 2010. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.append(os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # Add any Sphinx extension module names here, as strings. They can be extensions 24 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 25 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest'] 26 | 27 | # Add any paths that contain templates here, relative to this directory. 28 | templates_path = ['_templates'] 29 | 30 | # The suffix of source filenames. 31 | source_suffix = '.rst' 32 | 33 | # The encoding of source files. 34 | #source_encoding = 'utf-8' 35 | 36 | # The master toctree document. 37 | master_doc = 'index' 38 | 39 | # General information about the project. 40 | project = u'superlance' 41 | copyright = u'2010, Chris McDonough, Agendaless Consulting, Inc.' 42 | 43 | # The version info for the project you're documenting, acts as replacement for 44 | # |version| and |release|, also used in various other places throughout the 45 | # built documents. 46 | # 47 | # The short X.Y version. 48 | version = '2.0.1.dev0' 49 | # The full version, including alpha/beta/rc tags. 50 | release = version 51 | 52 | # The language for content autogenerated by Sphinx. Refer to documentation 53 | # for a list of supported languages. 54 | #language = None 55 | 56 | # There are two options for replacing |today|: either, you set today to some 57 | # non-false value, then it is used: 58 | #today = '' 59 | # Else, today_fmt is used as the format for a strftime call. 60 | #today_fmt = '%B %d, %Y' 61 | 62 | # List of documents that shouldn't be included in the build. 63 | #unused_docs = [] 64 | 65 | # List of directories, relative to source directory, that shouldn't be searched 66 | # for source files. 67 | exclude_trees = ['_build'] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'sphinx' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. Major themes that come with 93 | # Sphinx are currently 'default' and 'sphinxdoc'. 94 | html_theme = 'default' 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | #html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | #html_static_path = ['_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_use_modindex = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, an OpenSearch description file will be output, and all pages will 153 | # contain a tag referring to it. The value of this option must be the 154 | # base URL from which the finished HTML is served. 155 | #html_use_opensearch = '' 156 | 157 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). 158 | #html_file_suffix = '' 159 | 160 | # Output file base name for HTML help builder. 161 | htmlhelp_basename = 'superlancedoc' 162 | 163 | 164 | # -- Options for LaTeX output -------------------------------------------------- 165 | 166 | # The paper size ('letter' or 'a4'). 167 | #latex_paper_size = 'letter' 168 | 169 | # The font size ('10pt', '11pt' or '12pt'). 170 | #latex_font_size = '10pt' 171 | 172 | # Grouping the document tree into LaTeX files. List of tuples 173 | # (source start file, target name, title, author, documentclass [howto/manual]). 174 | latex_documents = [ 175 | ('index', 'superlance.tex', u'superlance Documentation', 176 | u'Chris McDonough, Agendaless Consulting, Inc.', 'manual'), 177 | ] 178 | 179 | # The name of an image file (relative to this directory) to place at the top of 180 | # the title page. 181 | #latex_logo = None 182 | 183 | # For "manual" documents, if this is true, then toplevel headings are parts, 184 | # not chapters. 185 | #latex_use_parts = False 186 | 187 | # Additional stuff for the LaTeX preamble. 188 | #latex_preamble = '' 189 | 190 | # Documents to append as an appendix to all manuals. 191 | #latex_appendices = [] 192 | 193 | # If false, no module index is generated. 194 | #latex_use_modindex = True 195 | -------------------------------------------------------------------------------- /docs/crashmail.rst: -------------------------------------------------------------------------------- 1 | :command:`crashmail` Documentation 2 | ================================== 3 | 4 | :command:`crashmail` is a supervisor "event listener", intended to be 5 | subscribed to ``PROCESS_STATE_EXITED`` events. When :command:`crashmail` 6 | receives that event, and the transition is "unexpected", :command:`crashmail` 7 | sends an email notification to a configured address. 8 | 9 | :command:`crashmail` is incapable of monitoring the process status of processes 10 | which are not :command:`supervisord` child processes. 11 | 12 | :command:`crashmail` is a "console script" installed when you install 13 | :mod:`superlance`. Although :command:`crashmail` is an executable program, it 14 | isn't useful as a general-purpose script: it must be run as a 15 | :command:`supervisor` event listener to do anything useful. 16 | 17 | Command-Line Syntax 18 | ------------------- 19 | 20 | .. code-block:: sh 21 | 22 | $ crashmail [-p processname] [-a] [-o string] [-m mail_address] \ 23 | [-s sendmail] 24 | 25 | .. program:: crashmail 26 | 27 | .. cmdoption:: -p , --program= 28 | 29 | Send mail when the specified :command:`supervisord` child process 30 | transitions unexpectedly to the ``EXITED`` state. 31 | 32 | This option can be provided more than once to have :command:`crashmail` 33 | monitor more than one program. 34 | 35 | To monitor a process which is part of a :command:`supervisord` group, 36 | specify its name as ``group_name:process_name``. 37 | 38 | .. cmdoption:: -a, --any 39 | 40 | Send mail when any :command:`supervisord` child process transitions 41 | unexpectedly to the ``EXITED`` state. 42 | 43 | Overrides any ``-p`` parameters passed in the same :command:`crashmail` 44 | process invocation. 45 | 46 | .. cmdoption:: -o , --optionalheader= 47 | 48 | Specify a parameter used as a prefix in the mail :mailheader:`Subject` 49 | header. 50 | 51 | .. cmdoption:: -s , --sendmail_program= 52 | 53 | Specify the sendmail command to use to send email. 54 | 55 | Must be a command which accepts header and message data on stdin and 56 | sends mail. Default is ``/usr/sbin/sendmail -t -i``. 57 | 58 | .. cmdoption:: -m , --email= 59 | 60 | Specify an email address to which crash notification messages are sent. 61 | If no email address is specified, email will not be sent. 62 | 63 | 64 | Configuring :command:`crashmail` Into the Supervisor Config 65 | ----------------------------------------------------------- 66 | 67 | An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` 68 | in order for :command:`crashmail` to do its work. See the "Events" chapter in 69 | the Supervisor manual for more information about event listeners. 70 | 71 | The following example assumes that :command:`crashmail` is on your system 72 | :envvar:`PATH`. 73 | 74 | .. code-block:: ini 75 | 76 | [eventlistener:crashmail] 77 | command=crashmail -p program1 -p group1:program2 -m dev@example.com 78 | events=PROCESS_STATE_EXITED 79 | -------------------------------------------------------------------------------- /docs/crashmailbatch.rst: -------------------------------------------------------------------------------- 1 | :command:`crashmailbatch` Documentation 2 | ======================================= 3 | 4 | :command:`crashmailbatch` is a supervisor "event listener", intended to be 5 | subscribed to ``PROCESS_STATE`` and ``TICK_60`` events. It monitors 6 | all processes running under a given supervisord instance. 7 | 8 | Similar to :command:`crashmail`, :command:`crashmailbatch` sends email 9 | alerts when processes die unexpectedly. The difference is that all alerts 10 | generated within the configured time interval are batched together to avoid 11 | sending too many emails. 12 | 13 | :command:`crashmailbatch` is a "console script" installed when you install 14 | :mod:`superlance`. Although :command:`crashmailbatch` is an executable 15 | program, it isn't useful as a general-purpose script: it must be run as a 16 | :command:`supervisor` event listener to do anything useful. 17 | 18 | Command-Line Syntax 19 | ------------------- 20 | 21 | .. code-block:: sh 22 | 23 | $ crashmailbatch --toEmail= --fromEmail= \ 24 | [--interval=] [--subject=] \ 25 | [--tickEvent=] [--smtpHost=] \ 26 | [--userName=] [--password=] \ 27 | [--tls] 28 | 29 | .. program:: crashmailbatch 30 | 31 | .. cmdoption:: -t , --toEmail= 32 | 33 | Specify comma separated email addresses to which crash notification messages are sent. 34 | 35 | .. cmdoption:: -f , --fromEmail= 36 | 37 | Specify an email address from which crash notification messages are sent. 38 | 39 | .. cmdoption:: -i , --interval= 40 | 41 | Specify the time interval in minutes to use for batching notifcations. 42 | Defaults to 1.0 minute. 43 | 44 | .. cmdoption:: -s , --subject= 45 | 46 | Override the email subject line. Defaults to "Crash alert from supervisord" 47 | 48 | .. cmdoption:: -e , --tickEvent= 49 | 50 | Override the TICK event name. Defaults to "TICK_60" 51 | 52 | .. cmdoption:: -H --smtpHost 53 | 54 | Specify STMP server for sending email 55 | 56 | .. cmdoption:: -u --userName 57 | 58 | Specify STMP username 59 | 60 | .. cmdoption:: -p --password 61 | 62 | Specify STMP password 63 | 64 | .. cmdoption:: --tls 65 | 66 | Use Transport Layer Security (TLS) 67 | 68 | Configuring :command:`crashmailbatch` Into the Supervisor Config 69 | ---------------------------------------------------------------- 70 | 71 | An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` 72 | in order for :command:`crashmailbatch` to do its work. See the "Events" chapter in 73 | the Supervisor manual for more information about event listeners. 74 | 75 | The following example assumes that :command:`crashmailbatch` is on your system 76 | :envvar:`PATH`. 77 | 78 | .. code-block:: ini 79 | 80 | [eventlistener:crashmailbatch] 81 | command=crashmailbatch --toEmail="alertme@fubar.com" --fromEmail="supervisord@fubar.com" 82 | events=PROCESS_STATE,TICK_60 83 | -------------------------------------------------------------------------------- /docs/crashsms.rst: -------------------------------------------------------------------------------- 1 | :command:`crashsms` Documentation 2 | ================================== 3 | 4 | :command:`crashsms` is a supervisor "event listener", intended to be 5 | subscribed to ``PROCESS_STATE`` events and ``TICK`` events such as ``TICK_60``. It monitors 6 | all processes running under a given supervisord instance. 7 | 8 | Similar to :command:`crashmailbatch`, :command:`crashsms` sends SMS alerts 9 | through an email gateway. Messages are formatted to fit in SMS 10 | 11 | :command:`crashsms` is a "console script" installed when you install 12 | :mod:`superlance`. Although :command:`crashsms` is an executable 13 | program, it isn't useful as a general-purpose script: it must be run as a 14 | :command:`supervisor` event listener to do anything useful. 15 | 16 | Command-Line Syntax 17 | ------------------- 18 | 19 | .. code-block:: sh 20 | 21 | $ crashsms --toEmail= --fromEmail= \ 22 | [--interval=] [--subject=] \ 23 | [--tickEvent=] 24 | 25 | .. program:: crashsms 26 | 27 | .. cmdoption:: -t , --toEmail= 28 | 29 | Specify comma separated email addresses to which crash notification messages are sent. 30 | 31 | .. cmdoption:: -f , --fromEmail= 32 | 33 | Specify an email address from which crash notification messages are sent. 34 | 35 | .. cmdoption:: -i , --interval= 36 | 37 | Specify the time interval in minutes to use for batching notifcations. 38 | Defaults to 1.0 minute. 39 | 40 | .. cmdoption:: -s , --subject= 41 | 42 | Set the email subject line. Default is None 43 | 44 | .. cmdoption:: -e , --tickEvent= 45 | 46 | Override the TICK event name. Defaults to "TICK_60" 47 | 48 | Configuring :command:`crashsms` Into the Supervisor Config 49 | ----------------------------------------------------------- 50 | 51 | An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` 52 | in order for :command:`crashsms` to do its work. See the "Events" chapter in 53 | the Supervisor manual for more information about event listeners. 54 | 55 | The following example assumes that :command:`crashsms` is on your system 56 | :envvar:`PATH`. 57 | 58 | .. code-block:: ini 59 | 60 | [eventlistener:crashsms] 61 | command=crashsms --toEmail="@" --fromEmail="supervisord@fubar.com" 62 | events=PROCESS_STATE,TICK_60 63 | -------------------------------------------------------------------------------- /docs/development.rst: -------------------------------------------------------------------------------- 1 | Resources and Development 2 | ========================= 3 | 4 | Bug Tracker 5 | ----------- 6 | 7 | Superlance has a bug tracker where you may report any bugs or other 8 | errors you find. Please report bugs to the `GitHub issues page 9 | `_. 10 | 11 | Version Control Repository 12 | -------------------------- 13 | 14 | You can also view the `Superlance version control repository 15 | `_. 16 | 17 | Contributing 18 | ------------ 19 | 20 | `Pull requests `_ 21 | can be submitted to the Superlance repository on GitHub. 22 | 23 | In the time since Superlance was created, 24 | there are now many `third party plugins for Supervisor `_. 25 | Most new plugins should be in their own package rather than added to Superlance. 26 | 27 | Author Information 28 | ------------------ 29 | 30 | The following people are responsible for creating Superlance. 31 | 32 | Original Author 33 | ~~~~~~~~~~~~~~~ 34 | 35 | `Chris McDonough `_ is the original author of Superlance. 36 | 37 | Contributors 38 | ~~~~~~~~~~~~ 39 | 40 | Contributors are tracked on the `GitHub contributions page 41 | `_. 42 | 43 | The list below is included for historical reasons. It records contributors who 44 | signed a legal agreement. The legal agreement was 45 | `introduced `_ 46 | in January 2014 but later 47 | `withdrawn `_ 48 | in April 2014. This list is being preserved in case it is useful 49 | later (e.g. if at some point there was a desire to donate the project 50 | to a foundation that required such agreements). 51 | 52 | - Chris McDonough, 2008-09-18 53 | 54 | - Tres Seaver, 2009-02-11 55 | 56 | - Roger Hoover, 2010-07-30 57 | 58 | - Joaquín Cuenca Abela, 2011-06-23 59 | 60 | - Harald Friessnegger, 2012-11-01 61 | 62 | - Mikhail Lukyanchenko, 2013-12-23 63 | 64 | - Patrick Gerken, 2014-01-27 65 | -------------------------------------------------------------------------------- /docs/fatalmailbatch.rst: -------------------------------------------------------------------------------- 1 | :command:`fatalmailbatch` Documentation 2 | ======================================= 3 | 4 | :command:`fatalmailbatch` is a supervisor "event listener", intended to be 5 | subscribed to ``PROCESS_STATE`` and ``TICK_60`` events. It monitors 6 | all processes running under a given supervisord instance. 7 | 8 | :command:`fatalmailbatch` sends email alerts when processes fail to start 9 | too many times such that supervisord gives up retrying. All of the fatal 10 | start events generated within the configured time interval are batched 11 | together to avoid sending too many emails. 12 | 13 | :command:`fatalmailbatch` is a "console script" installed when you install 14 | :mod:`superlance`. Although :command:`fatalmailbatch` is an executable 15 | program, it isn't useful as a general-purpose script: it must be run as a 16 | :command:`supervisor` event listener to do anything useful. 17 | 18 | Command-Line Syntax 19 | ------------------- 20 | 21 | .. code-block:: sh 22 | 23 | $ fatalmailbatch --toEmail= --fromEmail= \ 24 | [--interval=] [--subject=] 25 | 26 | .. program:: fatalmailbatch 27 | 28 | .. cmdoption:: -t , --toEmail= 29 | 30 | Specify comma separated email addresses to which fatal start notification messages are sent. 31 | 32 | .. cmdoption:: -f , --fromEmail= 33 | 34 | Specify an email address from which fatal start notification messages 35 | are sent. 36 | 37 | .. cmdoption:: -i , --interval= 38 | 39 | Specify the time interval in minutes to use for batching notifcations. 40 | Defaults to 1 minute. 41 | 42 | .. cmdoption:: -s , --subject= 43 | 44 | Override the email subject line. Defaults to "Fatal start alert from 45 | supervisord" 46 | 47 | Configuring :command:`fatalmailbatch` Into the Supervisor Config 48 | ---------------------------------------------------------------- 49 | 50 | An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` 51 | in order for :command:`fatalmailbatch` to do its work. See the "Events" chapter in 52 | the Supervisor manual for more information about event listeners. 53 | 54 | The following example assumes that :command:`fatalmailbatch` is on your system 55 | :envvar:`PATH`. 56 | 57 | .. code-block:: ini 58 | 59 | [eventlistener:fatalmailbatch] 60 | command=fatalmailbatch --toEmail="alertme@fubar.com" --fromEmail="supervisord@fubar.com" 61 | events=PROCESS_STATE,TICK_60 62 | -------------------------------------------------------------------------------- /docs/httpok.rst: -------------------------------------------------------------------------------- 1 | :command:`httpok` Documentation 2 | ================================== 3 | 4 | :command:`httpok` is a supervisor "event listener" which may be subscribed to 5 | a concrete ``TICK_5``, ``TICK_60`` or ``TICK_3600`` event. 6 | When :command:`httpok` receives a ``TICK_x`` 7 | event (``TICK_60`` is recommended, indicating activity every 60 seconds), 8 | :command:`httpok` makes an HTTP GET request to a confgured URL. If the request 9 | fails or times out, :command:`httpok` will restart the "hung" child 10 | process(es). :command:`httpok` can be configured to send an email notification 11 | when it restarts a process. 12 | 13 | :command:`httpok` can only monitor the process status of processes 14 | which are :command:`supervisord` child processes. 15 | 16 | :command:`httpok` is a "console script" installed when you install 17 | :mod:`superlance`. Although :command:`httpok` is an executable program, it 18 | isn't useful as a general-purpose script: it must be run as a 19 | :command:`supervisor` event listener to do anything useful. 20 | 21 | Command-Line Syntax 22 | ------------------- 23 | 24 | .. code-block:: sh 25 | 26 | $ httpok [-p processname] [-a] [-g] [-t timeout] [-c status_code] \ 27 | [-b inbody] [-m mail_address] [-s sendmail] URL 28 | 29 | .. program:: httpok 30 | 31 | .. cmdoption:: -p , --program= 32 | 33 | Restart the :command:`supervisord` child process named ``process_name`` 34 | if it is in the ``RUNNING`` state when the URL returns an unexpected 35 | result or times out. 36 | 37 | This option can be provided more than once to have :command:`httpok` 38 | monitor more than one process. 39 | 40 | To monitor a process which is part of a :command:`supervisord` group, 41 | specify its name as ``group_name:process_name``. 42 | 43 | .. cmdoption:: -a, --any 44 | 45 | Restart any child of :command:`supervisord` in the ``RUNNING`` state 46 | if the URL returns an unexpected result or times out. 47 | 48 | Overrides any ``-p`` parameters passed in the same :command:`httpok` 49 | process invocation. 50 | 51 | .. cmdoption:: -g , --gcore= 52 | 53 | Use the specifed program to ``gcore`` the :command:`supervisord` child 54 | process. The program should accept two arguments on the command line: 55 | a filename and a pid. Defaults to ``/usr/bin/gcore -o``. 56 | 57 | .. cmdoption:: -d , --coredir= 58 | 59 | If a core directory is specified, :command:`httpok` will try to use the 60 | ``gcore`` program (see ``-g``) to write a core file into this directory 61 | for each hung process before restarting it. It will then append any gcore 62 | stdout output to the email message, if mail is configured (see the ``-m`` 63 | option below). 64 | 65 | .. cmdoption:: -t , --timeout= 66 | 67 | The number of seconds that :command:`httpok` should wait for a response 68 | to the HTTP request before timing out. 69 | 70 | If this timeout is exceeded, :command:`httpok` will attempt to restart 71 | child processes which are in the ``RUNNING`` state, and specified by 72 | ``-p`` or ``-a``. 73 | 74 | Defaults to 10 seconds. 75 | 76 | .. cmdoption:: -c , --code= 77 | 78 | Specify the expected HTTP status code for the configured URL. 79 | 80 | If this status code is not the status code provided by the response, 81 | :command:`httpok` will attempt to restart child processes which are 82 | in the ``RUNNING`` state, and specified by ``-p`` or ``-a``. 83 | 84 | Defaults to 200. 85 | 86 | .. cmdoption:: -b , --body= 87 | 88 | Specify a string which should be present in the body resulting 89 | from the GET request. 90 | 91 | If this string is not present in the response, :command:`httpok` will 92 | attempt to restart child processes which are in the RUNNING state, 93 | and specified by ``-p`` or ``-a``. 94 | 95 | The default is to ignore the body. 96 | 97 | .. cmdoption:: -s , --sendmail_program= 98 | 99 | Specify the sendmail command to use to send email. 100 | 101 | Must be a command which accepts header and message data on stdin and 102 | sends mail. Default is ``/usr/sbin/sendmail -t -i``. 103 | 104 | .. cmdoption:: -m , --email= 105 | 106 | Specify an email address to which notification messages are sent. 107 | If no email address is specified, email will not be sent. 108 | 109 | .. cmdoption:: -e, --eager 110 | 111 | Enable "eager" monitoring: check the URL and emit mail even if no 112 | monitored child process is in the ``RUNNING`` state. 113 | 114 | Enabled by default. 115 | 116 | .. cmdoption:: -E, --not-eager 117 | 118 | Disable "eager" monitoring: do not check the URL or emit mail if no 119 | monitored process is in the RUNNING state. 120 | 121 | .. cmdoption:: URL 122 | 123 | The URL to which to issue a GET request. 124 | 125 | .. cmdoption:: -n , --name= 126 | 127 | An optional name that identifies this httpok process. If given, the 128 | email subject will start with ``httpok []:`` instead 129 | of ``httpok:`` 130 | In case you run multiple supervisors on a single host that control 131 | different processes with the same name (eg `zopeinstance1`) you can 132 | use this option to indicate which project the restarted instance 133 | belongs to. 134 | 135 | 136 | Configuring :command:`httpok` Into the Supervisor Config 137 | ----------------------------------------------------------- 138 | 139 | An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` 140 | in order for :command:`httpok` to do its work. 141 | See the "Events" chapter in the 142 | Supervisor manual for more information about event listeners. 143 | 144 | The following example assumes that :command:`httpok` is on your system 145 | :envvar:`PATH`. 146 | 147 | .. code-block:: ini 148 | 149 | [eventlistener:httpok] 150 | command=httpok -p program1 -p group1:program2 http://localhost:8080/tasty 151 | events=TICK_60 152 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | superlance plugins for supervisor 2 | ================================= 3 | 4 | Superlance is a package of plugin utilities for monitoring and 5 | controlling processes that run under `Supervisor 6 | `_. It provides these plugins: 7 | 8 | :command:`httpok` 9 | This plugin is meant to be used as a supervisor event listener, 10 | subscribed to ``TICK_*`` events. It tests that a given child process 11 | which must in the ``RUNNING`` state, is viable via an HTTP ``GET`` 12 | request to a configured URL. If the request fails or times out, 13 | :command:`httpok` will restart the "hung" child process. 14 | 15 | :command:`crashmail` 16 | This plugin is meant to be used as a supervisor event listener, 17 | subscribed to ``PROCESS_STATE_EXITED`` events. It email a user when 18 | a process enters the ``EXITED`` state unexpectedly. 19 | 20 | :command:`memmon` 21 | This plugin is meant to be used as a supervisor event listener, 22 | subscribed to ``TICK_*`` events. It monitors memory usage for configured 23 | child processes, and restarts them when they exceed a configured 24 | maximum size. 25 | 26 | :command:`crashmailbatch` 27 | Similar to :command:`crashmail`, :command:`crashmailbatch` sends email 28 | alerts when processes die unexpectedly. The difference is that all alerts 29 | generated within the configured time interval are batched together to avoid 30 | sending too many emails. 31 | 32 | :command:`fatalmailbatch` 33 | This plugin sends email alerts when processes fail to start 34 | too many times such that supervisord gives up retrying. All of the fatal 35 | start events generated within the configured time interval are batched 36 | together to avoid sending too many emails. 37 | 38 | :command:`crashsms` 39 | Similar to :command:`crashmailbatch` except it sends SMS alerts 40 | through an email gateway. Messages are formatted to fit in SMS. 41 | 42 | Contents: 43 | 44 | .. toctree:: 45 | :maxdepth: 2 46 | 47 | httpok 48 | crashmail 49 | memmon 50 | crashmailbatch 51 | fatalmailbatch 52 | crashsms 53 | development 54 | 55 | Indices and tables 56 | ================== 57 | 58 | * :ref:`genindex` 59 | * :ref:`modindex` 60 | * :ref:`search` 61 | 62 | -------------------------------------------------------------------------------- /docs/memmon.rst: -------------------------------------------------------------------------------- 1 | :command:`memmon` Overview 2 | ========================== 3 | 4 | :command:`memmon` is a supervisor "event listener" which may be subscribed to 5 | a concrete ``TICK_x`` event. When :command:`memmon` receives a ``TICK_x`` 6 | event (``TICK_60`` is recommended, indicating activity every 60 seconds), 7 | :command:`memmon` checks that a configurable list of programs (or all 8 | programs running under supervisor) are not exceeding a configurable amount of 9 | memory (resident segment size, or RSS). If one or more of these processes is 10 | consuming more than the amount of memory that :command:`memmon` believes it 11 | should, :command:`memmon` will restart the process. :command:`memmon` can be 12 | configured to send an email notification when it restarts a process. 13 | 14 | :command:`memmon` is known to work on Linux and Mac OS X, but has not been 15 | tested on other operating systems (it relies on :command:`ps` output and 16 | command-line switches). 17 | 18 | :command:`memmon` is incapable of monitoring the process status of processes 19 | which are not :command:`supervisord` child processes. Without the 20 | `--cumulative` option, only the RSS of immediate children of the 21 | :command:`supervisord` process will be considered. 22 | 23 | :command:`memmon` is a "console script" installed when you install 24 | :mod:`superlance`. Although :command:`memmon` is an executable program, it 25 | isn't useful as a general-purpose script: it must be run as a 26 | :command:`supervisor` event listener to do anything useful. 27 | 28 | :command:`memmon` uses Supervisor's XML-RPC interface. Your ``supervisord.conf`` 29 | file must have a valid `[unix_http_server] 30 | `_ 31 | or `[inet_http_server] 32 | `_ 33 | section, and must have an `[rpcinterface:supervisor] 34 | `_ 35 | section. If you are able to control your ``supervisord`` instance with 36 | ``supervisorctl``, you have already met these requirements. 37 | 38 | Command-Line Syntax 39 | ------------------- 40 | 41 | .. code-block:: sh 42 | 43 | $ memmon [-c] [-p processname=byte_size] [-g groupname=byte_size] \ 44 | [-a byte_size] [-s sendmail] [-m email_address] \ 45 | [-u email_uptime_limit] [-n memmon_name] 46 | 47 | .. program:: memmon 48 | 49 | .. cmdoption:: -h, --help 50 | 51 | Show program help. 52 | 53 | .. cmdoption:: -c, --cumulative 54 | 55 | Check against cumulative RSS. When calculating a process' RSS, also 56 | consider its child processes. With this option `memmon` will sum up 57 | the RSS of the process to be monitored and all its children. 58 | 59 | .. cmdoption:: -p , --program= 60 | 61 | A name/size pair, e.g. "foo=1MB". The name represents the supervisor 62 | program name that you would like :command:`memmon` to monitor; the size 63 | represents the number of bytes (suffix-multiplied using "KB", "MB" or "GB") 64 | that should be considered "too much". 65 | 66 | This option can be provided more than once to have :command:`memmon` 67 | monitor more than one program. 68 | 69 | Programs can be specified using a "namespec", to disambiguate same-named 70 | programs in different groups, e.g. ``foo:bar`` represents the program 71 | ``bar`` in the ``foo`` group. 72 | 73 | .. cmdoption:: -g , --groupname= 74 | 75 | A groupname/size pair, e.g. "group=1MB". The name represents the supervisor 76 | group name that you would like :command:`memmon` to monitor; the size 77 | represents the number of bytes (suffix-multiplied using "KB", "MB" or "GB") 78 | that should be considered "too much". 79 | 80 | Multiple ``-g`` options can be provided to have :command:`memmon` monitor 81 | more than one group. If any process in this group exceeds the maximum, 82 | it will be restarted. 83 | 84 | .. cmdoption:: -a , --any= 85 | 86 | A size (suffix-multiplied using "KB", "MB" or "GB") that should be 87 | considered "too much". If any program running as a child of supervisor 88 | exceeds this maximum, it will be restarted. E.g. 100MB. 89 | 90 | .. cmdoption:: -s , --sendmail= 91 | 92 | A command that will send mail if passed the email body (including the 93 | headers). Defaults to ``/usr/sbin/sendmail -t -i``. 94 | 95 | .. note:: 96 | 97 | Specifying this option doesn't cause memmon to send mail by itself: 98 | see the ``-m`` / ``--email`` option. 99 | 100 | .. cmdoption:: -m , --email= 101 | 102 | An email address to which to send email when a process is restarted. 103 | By default, memmon will not send any mail unless an email address is 104 | specified. 105 | 106 | .. cmdoption:: -u , --uptime= 107 | 108 | Only send an email in case the restarted process' uptime (in seconds) 109 | is below this limit. 110 | (Useful to only get notified if a processes gets restarted too frequently) 111 | 112 | Uptime is given in seconds (suffix-multiplied using "m" for minutes, 113 | "h" for hours or "d" for days) 114 | 115 | .. cmdoption:: -n , --name= 116 | 117 | An optional name that identifies this memmon process. If given, the 118 | email subject will start with ``memmon []:`` instead 119 | of ``memmon:`` 120 | In case you run multiple supervisors on a single host that control 121 | different processes with the same name (eg `zopeinstance1`) you can 122 | use this option to indicate which project the restarted instance 123 | belongs to. 124 | 125 | 126 | 127 | Configuring :command:`memmon` Into the Supervisor Config 128 | -------------------------------------------------------- 129 | 130 | An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` 131 | in order for :command:`memmon` to do its work. See the "Events" chapter in the 132 | Supervisor manual for more information about event listeners. 133 | 134 | If the `[unix_http_server] 135 | `_ 136 | or `[inet_http_server] 137 | `_ 138 | has been configured to use authentication, add the environment variables 139 | ``SUPERVISOR_USERNAME`` and ``SUPERVISOR_PASSWORD`` in the ``[eventlistener:x]`` 140 | section as shown in Example Configuration 5. 141 | 142 | The following examples assume that :command:`memmon` is on your system 143 | :envvar:`PATH`. 144 | 145 | Example Configuration 1 146 | ####################### 147 | 148 | This configuration causes :command:`memmon` to restart any process which is 149 | a child of :command:`supervisord` consuming more than 200MB of RSS, and will 150 | send mail to ``bob@example.com`` when it restarts a process using the 151 | default :command:`sendmail` command. 152 | 153 | .. code-block:: ini 154 | 155 | [eventlistener:memmon] 156 | command=memmon -a 200MB -m bob@example.com 157 | events=TICK_60 158 | 159 | 160 | Example Configuration 2 161 | ####################### 162 | 163 | This configuration causes :command:`memmon` to restart any process with the 164 | supervisor program name ``foo`` consuming more than 200MB of RSS, and 165 | will send mail to ``bob@example.com`` when it restarts a process using 166 | the default sendmail command. 167 | 168 | .. code-block:: ini 169 | 170 | [eventlistener:memmon] 171 | command=memmon -p foo=200MB -m bob@example.com 172 | events=TICK_60 173 | 174 | 175 | Example Configuration 3 176 | ####################### 177 | 178 | This configuration causes :command:`memmon` to restart any process in the 179 | process group "bar" consuming more than 200MB of RSS, and will send mail to 180 | ``bob@example.com`` when it restarts a process using the default 181 | :command:`sendmail` command. 182 | 183 | .. code-block:: ini 184 | 185 | [eventlistener:memmon] 186 | command=memmon -g bar=200MB -m bob@example.com 187 | events=TICK_60 188 | 189 | 190 | Example Configuration 4 191 | ####################### 192 | 193 | This configuration causes :command:`memmon` to restart any process meeting 194 | the same requirements as in `Example Configuration 2`_ with one difference: 195 | 196 | The email will only be sent if the process' uptime is less or equal than 197 | 2 days (172800 seconds) 198 | 199 | .. code-block:: ini 200 | 201 | [eventlistener:memmon] 202 | command=memmon -p foo=200MB -m bob@example.com -u 2d 203 | events=TICK_60 204 | 205 | 206 | Example Configuration 5 (Authentication) 207 | ######################################## 208 | 209 | This configuration is the same as the one in `Example Configuration 1`_ with 210 | the only difference being that the `[unix_http_server] 211 | `_ 212 | or `[inet_http_server] 213 | `_ 214 | has been configured to use authentication. 215 | 216 | .. code-block:: ini 217 | 218 | [eventlistener:memmon] 219 | command=memmon -a 200MB -m bob@example.com 220 | environment=SUPERVISOR_USERNAME="",SUPERVISOR_PASSWORD="" 221 | events=TICK_60 222 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2008-2013 Agendaless Consulting and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the BSD-like license at 7 | # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany 8 | # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL 9 | # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, 10 | # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND 11 | # FITNESS FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | import os 16 | import sys 17 | 18 | py_version = sys.version_info[:2] 19 | 20 | if py_version < (2, 7): 21 | raise RuntimeError('On Python 2, superlance requires Python 2.7 or later') 22 | elif (3, 0) < py_version < (3, 4): 23 | raise RuntimeError('On Python 3, superlance requires Python 3.4 or later') 24 | 25 | from setuptools import setup, find_packages 26 | 27 | here = os.path.abspath(os.path.dirname(__file__)) 28 | try: 29 | README = open(os.path.join(here, 'README.rst')).read() 30 | except (IOError, OSError): 31 | README = '' 32 | try: 33 | CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() 34 | except (IOError, OSError): 35 | CHANGES = '' 36 | 37 | tests_require = ['supervisor'] 38 | if py_version < (3, 3): 39 | tests_require.append('mock<4.0.0.dev0') 40 | 41 | setup(name='superlance', 42 | version='2.0.1.dev0', 43 | license='BSD-derived (http://www.repoze.org/LICENSE.txt)', 44 | description='superlance plugins for supervisord', 45 | long_description=README + '\n\n' + CHANGES, 46 | classifiers=[ 47 | "Development Status :: 5 - Production/Stable", 48 | 'Environment :: No Input/Output (Daemon)', 49 | 'Intended Audience :: System Administrators', 50 | 'Natural Language :: English', 51 | 'Operating System :: POSIX', 52 | 'Programming Language :: Python :: 2', 53 | 'Programming Language :: Python :: 2.7', 54 | 'Programming Language :: Python :: 3', 55 | 'Programming Language :: Python :: 3.4', 56 | 'Programming Language :: Python :: 3.5', 57 | 'Programming Language :: Python :: 3.6', 58 | 'Programming Language :: Python :: 3.7', 59 | 'Programming Language :: Python :: 3.8', 60 | 'Programming Language :: Python :: 3.9', 61 | 'Programming Language :: Python :: 3.10', 62 | 'Topic :: System :: Boot', 63 | 'Topic :: System :: Monitoring', 64 | 'Topic :: System :: Systems Administration', 65 | ], 66 | author='Chris McDonough', 67 | author_email='chrism@plope.com', 68 | url='https://github.com/Supervisor/superlance', 69 | keywords = 'supervisor monitoring', 70 | packages = find_packages(), 71 | include_package_data=True, 72 | zip_safe=False, 73 | install_requires=[ 74 | 'supervisor', 75 | ], 76 | tests_require=tests_require, 77 | test_suite='superlance.tests', 78 | entry_points = """\ 79 | [console_scripts] 80 | httpok = superlance.httpok:main 81 | crashsms = superlance.crashsms:main 82 | crashmail = superlance.crashmail:main 83 | crashmailbatch = superlance.crashmailbatch:main 84 | fatalmailbatch = superlance.fatalmailbatch:main 85 | memmon = superlance.memmon:main 86 | """ 87 | ) 88 | -------------------------------------------------------------------------------- /superlance/__init__.py: -------------------------------------------------------------------------------- 1 | # superlance package 2 | -------------------------------------------------------------------------------- /superlance/compat.py: -------------------------------------------------------------------------------- 1 | try: 2 | import http.client as httplib 3 | except ImportError: 4 | import httplib 5 | 6 | try: 7 | from StringIO import StringIO 8 | except ImportError: 9 | from io import StringIO 10 | 11 | try: 12 | from sys import maxsize as maxint 13 | except ImportError: 14 | from sys import maxint 15 | 16 | try: 17 | import urllib.parse as urlparse 18 | import urllib.parse as urllib 19 | except ImportError: 20 | import urlparse 21 | import urllib 22 | 23 | try: 24 | import xmlrpc.client as xmlrpclib 25 | except ImportError: 26 | import xmlrpclib 27 | -------------------------------------------------------------------------------- /superlance/crashmail.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python -u 2 | ############################################################################## 3 | # 4 | # Copyright (c) 2007 Agendaless Consulting and Contributors. 5 | # All Rights Reserved. 6 | # 7 | # This software is subject to the provisions of the BSD-like license at 8 | # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany 9 | # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL 10 | # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, 11 | # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND 12 | # FITNESS FOR A PARTICULAR PURPOSE 13 | # 14 | ############################################################################## 15 | 16 | # A event listener meant to be subscribed to PROCESS_STATE_CHANGE 17 | # events. It will send mail when processes that are children of 18 | # supervisord transition unexpectedly to the EXITED state. 19 | 20 | # A supervisor config snippet that tells supervisor to use this script 21 | # as a listener is below. 22 | # 23 | # [eventlistener:crashmail] 24 | # command = 25 | # /usr/bin/crashmail 26 | # -o hostname -a -m notify-on-crash@domain.com 27 | # -s '/usr/sbin/sendmail -t -i -f crash-notifier@domain.com' 28 | # events=PROCESS_STATE 29 | # 30 | # Sendmail is used explicitly here so that we can specify the 'from' address. 31 | 32 | doc = """\ 33 | crashmail.py [-p processname] [-a] [-o string] [-m mail_address] 34 | [-s sendmail] URL 35 | 36 | Options: 37 | 38 | -p -- specify a supervisor process_name. Send mail when this process 39 | transitions to the EXITED state unexpectedly. If this process is 40 | part of a group, it can be specified using the 41 | 'group_name:process_name' syntax. 42 | 43 | -a -- Send mail when any child of the supervisord transitions 44 | unexpectedly to the EXITED state unexpectedly. Overrides any -p 45 | parameters passed in the same crashmail process invocation. 46 | 47 | -o -- Specify a parameter used as a prefix in the mail subject header. 48 | 49 | -s -- the sendmail command to use to send email 50 | (e.g. "/usr/sbin/sendmail -t -i"). Must be a command which accepts 51 | header and message data on stdin and sends mail. Default is 52 | "/usr/sbin/sendmail -t -i". 53 | 54 | -m -- specify an email address. The script will send mail to this 55 | address when crashmail detects a process crash. If no email 56 | address is specified, email will not be sent. 57 | 58 | The -p option may be specified more than once, allowing for 59 | specification of multiple processes. Specifying -a overrides any 60 | selection of -p. 61 | 62 | A sample invocation: 63 | 64 | crashmail.py -p program1 -p group1:program2 -m dev@example.com 65 | 66 | """ 67 | 68 | import getopt 69 | import os 70 | import sys 71 | 72 | from supervisor import childutils 73 | 74 | 75 | def usage(exitstatus=255): 76 | print(doc) 77 | sys.exit(exitstatus) 78 | 79 | 80 | class CrashMail: 81 | 82 | def __init__(self, programs, any, email, sendmail, optionalheader): 83 | 84 | self.programs = programs 85 | self.any = any 86 | self.email = email 87 | self.sendmail = sendmail 88 | self.optionalheader = optionalheader 89 | self.stdin = sys.stdin 90 | self.stdout = sys.stdout 91 | self.stderr = sys.stderr 92 | 93 | def runforever(self, test=False): 94 | while 1: 95 | # we explicitly use self.stdin, self.stdout, and self.stderr 96 | # instead of sys.* so we can unit test this code 97 | headers, payload = childutils.listener.wait( 98 | self.stdin, self.stdout) 99 | 100 | if not headers['eventname'] == 'PROCESS_STATE_EXITED': 101 | # do nothing with non-TICK events 102 | childutils.listener.ok(self.stdout) 103 | if test: 104 | self.stderr.write('non-exited event\n') 105 | self.stderr.flush() 106 | break 107 | continue 108 | 109 | pheaders, pdata = childutils.eventdata(payload+'\n') 110 | 111 | if int(pheaders['expected']): 112 | childutils.listener.ok(self.stdout) 113 | if test: 114 | self.stderr.write('expected exit\n') 115 | self.stderr.flush() 116 | break 117 | continue 118 | 119 | msg = ('Process %(processname)s in group %(groupname)s exited ' 120 | 'unexpectedly (pid %(pid)s) from state %(from_state)s' % 121 | pheaders) 122 | 123 | subject = ' %s crashed at %s' % (pheaders['processname'], 124 | childutils.get_asctime()) 125 | if self.optionalheader: 126 | subject = self.optionalheader + ':' + subject 127 | 128 | self.stderr.write('unexpected exit, mailing\n') 129 | self.stderr.flush() 130 | 131 | self.mail(self.email, subject, msg) 132 | 133 | childutils.listener.ok(self.stdout) 134 | if test: 135 | break 136 | 137 | def mail(self, email, subject, msg): 138 | body = 'To: %s\n' % self.email 139 | body += 'Subject: %s\n' % subject 140 | body += '\n' 141 | body += msg 142 | with os.popen(self.sendmail, 'w') as m: 143 | m.write(body) 144 | self.stderr.write('Mailed:\n\n%s' % body) 145 | self.mailed = body 146 | 147 | 148 | def main(argv=sys.argv): 149 | short_args = "hp:ao:s:m:" 150 | long_args = [ 151 | "help", 152 | "program=", 153 | "any", 154 | "optionalheader=", 155 | "sendmail_program=", 156 | "email=", 157 | ] 158 | arguments = argv[1:] 159 | try: 160 | opts, args = getopt.getopt(arguments, short_args, long_args) 161 | except: 162 | usage() 163 | 164 | programs = [] 165 | any = False 166 | sendmail = '/usr/sbin/sendmail -t -i' 167 | email = None 168 | optionalheader = None 169 | 170 | for option, value in opts: 171 | 172 | if option in ('-h', '--help'): 173 | usage(exitstatus=0) 174 | 175 | if option in ('-p', '--program'): 176 | programs.append(value) 177 | 178 | if option in ('-a', '--any'): 179 | any = True 180 | 181 | if option in ('-s', '--sendmail_program'): 182 | sendmail = value 183 | 184 | if option in ('-m', '--email'): 185 | email = value 186 | 187 | if option in ('-o', '--optionalheader'): 188 | optionalheader = value 189 | 190 | if not 'SUPERVISOR_SERVER_URL' in os.environ: 191 | sys.stderr.write('crashmail must be run as a supervisor event ' 192 | 'listener\n') 193 | sys.stderr.flush() 194 | return 195 | 196 | prog = CrashMail(programs, any, email, sendmail, optionalheader) 197 | prog.runforever() 198 | 199 | 200 | if __name__ == '__main__': 201 | main() 202 | -------------------------------------------------------------------------------- /superlance/crashmailbatch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python -u 2 | ############################################################################## 3 | # 4 | # Copyright (c) 2007 Agendaless Consulting and Contributors. 5 | # All Rights Reserved. 6 | # 7 | # This software is subject to the provisions of the BSD-like license at 8 | # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany 9 | # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL 10 | # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, 11 | # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND 12 | # FITNESS FOR A PARTICULAR PURPOSE 13 | # 14 | ############################################################################## 15 | 16 | # A event listener meant to be subscribed to PROCESS_STATE_CHANGE 17 | # events. It will send mail when processes that are children of 18 | # supervisord transition unexpectedly to the EXITED state. 19 | 20 | # A supervisor config snippet that tells supervisor to use this script 21 | # as a listener is below. 22 | # 23 | # [eventlistener:crashmailbatch] 24 | # command=python crashmailbatch --toEmail=you@bar.com --fromEmail=me@bar.com 25 | # events=PROCESS_STATE,TICK_60 26 | 27 | doc = """\ 28 | crashmailbatch.py [--interval=] 29 | [--toEmail=] 30 | [--fromEmail=] 31 | [--subject=] 32 | [--smtpHost=] 33 | 34 | Options: 35 | 36 | --interval - batch cycle length (in minutes). The default is 1.0 minute. 37 | This means that all events in each cycle are batched together 38 | and sent as a single email 39 | 40 | --toEmail - the email address(es) to send alerts to - comma separated 41 | 42 | --fromEmail - the email address to send alerts from 43 | 44 | --subject - the email subject line 45 | 46 | --smtpHost - the SMTP server's hostname or address (defaults to 'localhost') 47 | 48 | A sample invocation: 49 | 50 | crashmailbatch.py --toEmail="you@bar.com" --fromEmail="me@bar.com" 51 | 52 | """ 53 | 54 | from supervisor import childutils 55 | from superlance.process_state_email_monitor import ProcessStateEmailMonitor 56 | 57 | 58 | class CrashMailBatch(ProcessStateEmailMonitor): 59 | 60 | process_state_events = ['PROCESS_STATE_EXITED'] 61 | 62 | def __init__(self, **kwargs): 63 | if kwargs.get('subject') is None: 64 | kwargs['subject'] = 'Crash alert from supervisord' 65 | ProcessStateEmailMonitor.__init__(self, **kwargs) 66 | self.now = kwargs.get('now', None) 67 | 68 | def get_process_state_change_msg(self, headers, payload): 69 | pheaders, pdata = childutils.eventdata(payload+'\n') 70 | 71 | if int(pheaders['expected']): 72 | return None 73 | 74 | txt = 'Process %(groupname)s:%(processname)s (pid %(pid)s) died \ 75 | unexpectedly' % pheaders 76 | return '%s -- %s' % (childutils.get_asctime(self.now), txt) 77 | 78 | 79 | def main(): 80 | crash = CrashMailBatch.create_from_cmd_line() 81 | crash.run() 82 | 83 | 84 | if __name__ == '__main__': 85 | main() 86 | -------------------------------------------------------------------------------- /superlance/crashsms.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python -u 2 | ############################################################################## 3 | # 4 | # Copyright (c) 2007 Agendaless Consulting and Contributors. 5 | # All Rights Reserved. 6 | # 7 | # This software is subject to the provisions of the BSD-like license at 8 | # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany 9 | # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL 10 | # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, 11 | # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND 12 | # FITNESS FOR A PARTICULAR PURPOSE 13 | # 14 | ############################################################################## 15 | 16 | ############################################################################## 17 | # crashsms 18 | # author: Juan Batiz-Benet (http://github.com/jbenet) 19 | # based on crashmailbatch.py 20 | ############################################################################## 21 | 22 | 23 | # A event listener meant to be subscribed to PROCESS_STATE_CHANGE 24 | # events. It will send mail when processes that are children of 25 | # supervisord transition unexpectedly to the EXITED state. 26 | 27 | # A supervisor config snippet that tells supervisor to use this script 28 | # as a listener is below. 29 | # 30 | # [eventlistener:crashsms] 31 | # command = 32 | # python crashsms 33 | # -t @ -f me@bar.com -e TICK_5 34 | # events=PROCESS_STATE,TICK_5 35 | 36 | doc = """\ 37 | crashsms.py [--interval=] 38 | [--toEmail=] 39 | [--fromEmail=] 40 | [--subject=] 41 | 42 | Options: 43 | 44 | -i,--interval - batch cycle length (in minutes). The default is 1 minute. 45 | This means that all events in each cycle are batched together 46 | and sent as a single email 47 | 48 | -t,--toEmail - the comma separated email addresses to send alerts to. Mobile providers 49 | tend to allow sms messages to be sent to their phone numbers 50 | via an email address (e.g.: 1234567890@txt.att.net) 51 | 52 | -f,--fromEmail - the email address to send alerts from 53 | 54 | -s,--subject - the email subject line 55 | 56 | -e, --tickEvent - specify which TICK event to use (e.g. TICK_5, TICK_60, 57 | TICK_3600) 58 | 59 | A sample invocation: 60 | 61 | crashsms.py -t @ -f me@bar.com -e TICK_5 62 | 63 | """ 64 | 65 | from supervisor import childutils 66 | from superlance.process_state_email_monitor import ProcessStateEmailMonitor 67 | 68 | 69 | class CrashSMS(ProcessStateEmailMonitor): 70 | process_state_events = ['PROCESS_STATE_EXITED'] 71 | 72 | def __init__(self, **kwargs): 73 | ProcessStateEmailMonitor.__init__(self, **kwargs) 74 | self.now = kwargs.get('now', None) 75 | 76 | def get_process_state_change_msg(self, headers, payload): 77 | pheaders, pdata = childutils.eventdata(payload+'\n') 78 | 79 | if int(pheaders['expected']): 80 | return None 81 | 82 | txt = '[%(groupname)s:%(processname)s](%(pid)s) exited unexpectedly' \ 83 | % pheaders 84 | return '%s %s' % (txt, childutils.get_asctime(self.now)) 85 | 86 | 87 | def main(): 88 | crash = CrashSMS.create_from_cmd_line() 89 | crash.run() 90 | 91 | 92 | if __name__ == '__main__': 93 | main() 94 | -------------------------------------------------------------------------------- /superlance/fatalmailbatch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python -u 2 | ############################################################################## 3 | # 4 | # Copyright (c) 2007 Agendaless Consulting and Contributors. 5 | # All Rights Reserved. 6 | # 7 | # This software is subject to the provisions of the BSD-like license at 8 | # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany 9 | # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL 10 | # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, 11 | # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND 12 | # FITNESS FOR A PARTICULAR PURPOSE 13 | # 14 | ############################################################################## 15 | 16 | # A event listener meant to be subscribed to PROCESS_STATE_CHANGE 17 | # events. It will send mail when processes that are children of 18 | # supervisord transition unexpectedly to the EXITED state. 19 | 20 | # A supervisor config snippet that tells supervisor to use this script 21 | # as a listener is below. 22 | # 23 | # [eventlistener:fatalmailbatch] 24 | # command=python fatalmailbatch 25 | # events=PROCESS_STATE,TICK_60 26 | 27 | doc = """\ 28 | fatalmailbatch.py [--interval=] 29 | [--toEmail=] 30 | [--fromEmail=] 31 | [--subject=] 32 | [--smtpHost=] 33 | [--userName=] 34 | [--password= 0: 184 | 185 | try: 186 | # build a loop value that is guaranteed to execute at least 187 | # once and at most until the timeout is reached and that 188 | # has 0 as the last value (to allow raising an exception 189 | # in the last iteration) 190 | for will_retry in range( 191 | (self.timeout - 1) // (self.retry_time or 1), 192 | -1, -1): 193 | try: 194 | headers = {'User-Agent': 'httpok'} 195 | conn.request('GET', path, headers=headers) 196 | break 197 | except socket.error as e: 198 | if e.errno == 111 and will_retry: 199 | time.sleep(self.retry_time) 200 | else: 201 | raise 202 | 203 | res = conn.getresponse() 204 | body = res.read() 205 | status = res.status 206 | msg = 'status contacting %s: %s %s' % (self.url, 207 | res.status, 208 | res.reason) 209 | except Exception as e: 210 | body = '' 211 | status = None 212 | msg = 'error contacting %s:\n\n %s' % (self.url, e) 213 | 214 | if status not in self.statuses: 215 | subject = self.format_subject( 216 | '%s: bad status returned' % self.url 217 | ) 218 | self.act(subject, msg) 219 | elif self.inbody and self.inbody not in body: 220 | subject = self.format_subject( 221 | '%s: bad body returned' % self.url 222 | ) 223 | self.act(subject, msg) 224 | 225 | childutils.listener.ok(self.stdout) 226 | if test: 227 | break 228 | 229 | def format_subject(self, subject): 230 | if self.name is None: 231 | return 'httpok: %s' % subject 232 | else: 233 | return 'httpok [%s]: %s' % (self.name, subject) 234 | 235 | def act(self, subject, msg): 236 | messages = [msg] 237 | 238 | def write(msg): 239 | self.stderr.write('%s\n' % msg) 240 | self.stderr.flush() 241 | messages.append(msg) 242 | 243 | try: 244 | specs = self.rpc.supervisor.getAllProcessInfo() 245 | except Exception as e: 246 | write('Exception retrieving process info %s, not acting' % e) 247 | return 248 | 249 | waiting = list(self.programs) 250 | 251 | if self.any: 252 | write('Restarting all running processes') 253 | for spec in specs: 254 | name = spec['name'] 255 | group = spec['group'] 256 | self.restart(spec, write) 257 | namespec = make_namespec(group, name) 258 | if name in waiting: 259 | waiting.remove(name) 260 | if namespec in waiting: 261 | waiting.remove(namespec) 262 | else: 263 | write('Restarting selected processes %s' % self.programs) 264 | for spec in specs: 265 | name = spec['name'] 266 | group = spec['group'] 267 | namespec = make_namespec(group, name) 268 | if (name in self.programs) or (namespec in self.programs): 269 | self.restart(spec, write) 270 | if name in waiting: 271 | waiting.remove(name) 272 | if namespec in waiting: 273 | waiting.remove(namespec) 274 | 275 | if waiting: 276 | write( 277 | 'Programs not restarted because they did not exist: %s' % 278 | waiting) 279 | 280 | if self.email: 281 | message = '\n'.join(messages) 282 | self.mail(self.email, subject, message) 283 | 284 | def mail(self, email, subject, msg): 285 | body = 'To: %s\n' % self.email 286 | body += 'Subject: %s\n' % subject 287 | body += '\n' 288 | body += msg 289 | with os.popen(self.sendmail, 'w') as m: 290 | m.write(body) 291 | self.stderr.write('Mailed:\n\n%s' % body) 292 | self.mailed = body 293 | 294 | def restart(self, spec, write): 295 | namespec = make_namespec(spec['group'], spec['name']) 296 | if spec['state'] is ProcessStates.RUNNING: 297 | if self.coredir and self.gcore: 298 | corename = os.path.join(self.coredir, namespec) 299 | cmd = self.gcore + ' "%s" %s' % (corename, spec['pid']) 300 | with os.popen(cmd) as m: 301 | write('gcore output for %s:\n\n %s' % ( 302 | namespec, m.read())) 303 | write('%s is in RUNNING state, restarting' % namespec) 304 | try: 305 | self.rpc.supervisor.stopProcess(namespec) 306 | except xmlrpclib.Fault as e: 307 | write('Failed to stop process %s: %s' % ( 308 | namespec, e)) 309 | 310 | try: 311 | self.rpc.supervisor.startProcess(namespec) 312 | except xmlrpclib.Fault as e: 313 | write('Failed to start process %s: %s' % ( 314 | namespec, e)) 315 | else: 316 | write('%s restarted' % namespec) 317 | 318 | else: 319 | write('%s not in RUNNING state, NOT restarting' % namespec) 320 | 321 | 322 | def main(argv=sys.argv): 323 | short_args="hp:at:c:b:s:m:g:d:eEn:" 324 | long_args=[ 325 | "help", 326 | "program=", 327 | "any", 328 | "timeout=", 329 | "code=", 330 | "body=", 331 | "sendmail_program=", 332 | "email=", 333 | "gcore=", 334 | "coredir=", 335 | "eager", 336 | "not-eager", 337 | "name=", 338 | ] 339 | arguments = argv[1:] 340 | try: 341 | opts, args = getopt.getopt(arguments, short_args, long_args) 342 | except: 343 | usage() 344 | 345 | # check for -h must be done before positional args check 346 | for option, value in opts: 347 | if option in ('-h', '--help'): 348 | usage(exitstatus=0) 349 | 350 | if not args: 351 | usage() 352 | if len(args) > 1: 353 | usage() 354 | 355 | programs = [] 356 | any = False 357 | sendmail = '/usr/sbin/sendmail -t -i' 358 | gcore = '/usr/bin/gcore -o' 359 | coredir = None 360 | eager = True 361 | email = None 362 | timeout = 10 363 | retry_time = 10 364 | statuses = [] 365 | inbody = None 366 | name = None 367 | 368 | for option, value in opts: 369 | 370 | if option in ('-p', '--program'): 371 | programs.append(value) 372 | 373 | if option in ('-a', '--any'): 374 | any = True 375 | 376 | if option in ('-s', '--sendmail_program'): 377 | sendmail = value 378 | 379 | if option in ('-m', '--email'): 380 | email = value 381 | 382 | if option in ('-t', '--timeout'): 383 | timeout = int(value) 384 | 385 | if option in ('-c', '--code'): 386 | statuses.append(int(value)) 387 | 388 | if option in ('-b', '--body'): 389 | inbody = value 390 | 391 | if option in ('-g', '--gcore'): 392 | gcore = value 393 | 394 | if option in ('-d', '--coredir'): 395 | coredir = value 396 | 397 | if option in ('-e', '--eager'): 398 | eager = True 399 | 400 | if option in ('-E', '--not-eager'): 401 | eager = False 402 | 403 | if option in ('-n', '--name'): 404 | name = value 405 | 406 | if not statuses: 407 | statuses = [200] 408 | 409 | url = arguments[-1] 410 | 411 | try: 412 | rpc = childutils.getRPCInterface(os.environ) 413 | except KeyError as e: 414 | if e.args[0] != 'SUPERVISOR_SERVER_URL': 415 | raise 416 | sys.stderr.write('httpok must be run as a supervisor event ' 417 | 'listener\n') 418 | sys.stderr.flush() 419 | return 420 | 421 | prog = HTTPOk(rpc, programs, any, url, timeout, statuses, inbody, email, 422 | sendmail, coredir, gcore, eager, retry_time, name) 423 | prog.runforever() 424 | 425 | if __name__ == '__main__': 426 | main() 427 | -------------------------------------------------------------------------------- /superlance/memmon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ############################################################################## 3 | # 4 | # Copyright (c) 2007 Agendaless Consulting and Contributors. 5 | # All Rights Reserved. 6 | # 7 | # This software is subject to the provisions of the BSD-like license at 8 | # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany 9 | # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL 10 | # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, 11 | # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND 12 | # FITNESS FOR A PARTICULAR PURPOSE 13 | # 14 | ############################################################################## 15 | 16 | # A event listener meant to be subscribed to TICK_60 (or TICK_5) 17 | # events, which restarts any processes that are children of 18 | # supervisord that consume "too much" memory. Performs horrendous 19 | # screenscrapes of ps output. Works on Linux and OS X (Tiger/Leopard) 20 | # as far as I know. 21 | 22 | # A supervisor config snippet that tells supervisor to use this script 23 | # as a listener is below. 24 | # 25 | # [eventlistener:memmon] 26 | # command=python memmon.py [options] 27 | # events=TICK_60 28 | 29 | doc = """\ 30 | memmon.py [-c] [-p processname=byte_size] [-g groupname=byte_size] 31 | [-a byte_size] [-s sendmail] [-m email_address] 32 | [-u uptime] [-n memmon_name] 33 | 34 | Options: 35 | 36 | -c -- Check against cumulative RSS. When calculating a process' RSS, also 37 | consider its child processes. With this option `memmon` will sum up 38 | the RSS of the process to be monitored and all its children. 39 | 40 | -p -- specify a process_name=byte_size pair. Restart the supervisor 41 | process named 'process_name' when it uses more than byte_size 42 | RSS. If this process is in a group, it can be specified using 43 | the 'group_name:process_name' syntax. 44 | 45 | -g -- specify a group_name=byte_size pair. Restart any process in this group 46 | when it uses more than byte_size RSS. 47 | 48 | -a -- specify a global byte_size. Restart any child of the supervisord 49 | under which this runs if it uses more than byte_size RSS. 50 | 51 | -s -- the sendmail command to use to send email 52 | (e.g. "/usr/sbin/sendmail -t -i"). Must be a command which accepts 53 | header and message data on stdin and sends mail. 54 | Default is "/usr/sbin/sendmail -t -i". 55 | 56 | -m -- specify an email address. The script will send mail to this 57 | address when any process is restarted. If no email address is 58 | specified, email will not be sent. 59 | 60 | -u -- optionally specify the minimum uptime in seconds for the process. 61 | if the process uptime is longer than this value, no email is sent 62 | (useful to only be notified if processes are restarted too often/early) 63 | 64 | seconds can be specified as plain integer values or a suffix-multiplied 65 | integer (e.g. 1m). Valid suffixes are m (minute), h (hour) and d (day). 66 | 67 | -n -- optionally specify the name of the memmon process. This name will 68 | be used in the email subject to identify which memmon process 69 | restarted the process. 70 | 71 | The -p and -g options may be specified more than once, allowing for 72 | specification of multiple groups and processes. 73 | 74 | Any byte_size can be specified as a plain integer (10000) or a 75 | suffix-multiplied integer (e.g. 1GB). Valid suffixes are 'KB', 'MB' 76 | and 'GB'. 77 | 78 | A sample invocation: 79 | 80 | memmon.py -p program1=200MB -p theprog:thegroup=100MB -g thegroup=100MB -a 1GB -s "/usr/sbin/sendmail -t -i" -m chrism@plope.com -n "Project 1" 81 | """ 82 | 83 | import getopt 84 | import os 85 | import sys 86 | import time 87 | from collections import namedtuple 88 | from superlance.compat import maxint 89 | from superlance.compat import xmlrpclib 90 | 91 | from supervisor import childutils 92 | from supervisor.datatypes import byte_size, SuffixMultiplier 93 | 94 | def usage(exitstatus=255): 95 | print(doc) 96 | sys.exit(exitstatus) 97 | 98 | def shell(cmd): 99 | with os.popen(cmd) as f: 100 | return f.read() 101 | 102 | class Memmon: 103 | def __init__(self, cumulative, programs, groups, any, sendmail, email, email_uptime_limit, name, rpc=None): 104 | self.cumulative = cumulative 105 | self.programs = programs 106 | self.groups = groups 107 | self.any = any 108 | self.sendmail = sendmail 109 | self.email = email 110 | self.email_uptime_limit = email_uptime_limit 111 | self.name = name 112 | self.rpc = rpc 113 | self.stdin = sys.stdin 114 | self.stdout = sys.stdout 115 | self.stderr = sys.stderr 116 | self.pscommand = 'ps -orss= -p %s' 117 | self.pstreecommand = 'ps ax -o "pid= ppid= rss="' 118 | self.mailed = False # for unit tests 119 | 120 | def runforever(self, test=False): 121 | while 1: 122 | # we explicitly use self.stdin, self.stdout, and self.stderr 123 | # instead of sys.* so we can unit test this code 124 | headers, payload = childutils.listener.wait(self.stdin, self.stdout) 125 | 126 | if not headers['eventname'].startswith('TICK'): 127 | # do nothing with non-TICK events 128 | childutils.listener.ok(self.stdout) 129 | if test: 130 | break 131 | continue 132 | 133 | status = [] 134 | if self.programs: 135 | keys = sorted(self.programs.keys()) 136 | status.append( 137 | 'Checking programs %s' % ', '.join( 138 | [ '%s=%s' % (k, self.programs[k]) for k in keys ]) 139 | ) 140 | 141 | if self.groups: 142 | keys = sorted(self.groups.keys()) 143 | status.append( 144 | 'Checking groups %s' % ', '.join( 145 | [ '%s=%s' % (k, self.groups[k]) for k in keys ]) 146 | ) 147 | if self.any is not None: 148 | status.append('Checking any=%s' % self.any) 149 | 150 | self.stderr.write('\n'.join(status) + '\n') 151 | 152 | infos = self.rpc.supervisor.getAllProcessInfo() 153 | 154 | for info in infos: 155 | pid = info['pid'] 156 | name = info['name'] 157 | group = info['group'] 158 | pname = '%s:%s' % (group, name) 159 | 160 | if not pid: 161 | # ps throws an error in this case (for processes 162 | # in standby mode, non-auto-started). 163 | continue 164 | 165 | rss = self.calc_rss(pid) 166 | if rss is None: 167 | # no such pid (deal with race conditions) or 168 | # rss couldn't be calculated for other reasons 169 | continue 170 | 171 | for n in name, pname: 172 | if n in self.programs: 173 | self.stderr.write('RSS of %s is %s\n' % (pname, rss)) 174 | if rss > self.programs[name]: 175 | self.restart(pname, rss) 176 | continue 177 | 178 | if group in self.groups: 179 | self.stderr.write('RSS of %s is %s\n' % (pname, rss)) 180 | if rss > self.groups[group]: 181 | self.restart(pname, rss) 182 | continue 183 | 184 | if self.any is not None: 185 | self.stderr.write('RSS of %s is %s\n' % (pname, rss)) 186 | if rss > self.any: 187 | self.restart(pname, rss) 188 | continue 189 | 190 | self.stderr.flush() 191 | childutils.listener.ok(self.stdout) 192 | if test: 193 | break 194 | 195 | def restart(self, name, rss): 196 | info = self.rpc.supervisor.getProcessInfo(name) 197 | uptime = info['now'] - info['start'] #uptime in seconds 198 | self.stderr.write('Restarting %s\n' % name) 199 | try: 200 | self.rpc.supervisor.stopProcess(name) 201 | except xmlrpclib.Fault as e: 202 | msg = ('Failed to stop process %s (RSS %s), exiting: %s' % 203 | (name, rss, e)) 204 | self.stderr.write(str(msg)) 205 | if self.email: 206 | subject = self.format_subject( 207 | 'failed to stop process %s, exiting' % name 208 | ) 209 | self.mail(self.email, subject, msg) 210 | raise 211 | 212 | try: 213 | self.rpc.supervisor.startProcess(name) 214 | except xmlrpclib.Fault as e: 215 | msg = ('Failed to start process %s after stopping it, ' 216 | 'exiting: %s' % (name, e)) 217 | self.stderr.write(str(msg)) 218 | if self.email: 219 | subject = self.format_subject( 220 | 'failed to start process %s, exiting' % name 221 | ) 222 | self.mail(self.email, subject, msg) 223 | raise 224 | 225 | if self.email and uptime <= self.email_uptime_limit: 226 | now = time.asctime() 227 | timezone = time.strftime('%Z') 228 | msg = ( 229 | 'memmon.py restarted the process named %s at %s %s because ' 230 | 'it was consuming too much memory (%s bytes RSS)' % ( 231 | name, now, timezone, rss) 232 | ) 233 | subject = self.format_subject( 234 | 'process %s restarted' % name 235 | ) 236 | self.mail(self.email, subject, msg) 237 | 238 | def format_subject(self, subject): 239 | if self.name is None: 240 | return 'memmon: %s' % subject 241 | else: 242 | return 'memmon [%s]: %s' % (self.name, subject) 243 | 244 | def calc_rss(self, pid): 245 | ProcInfo = namedtuple('ProcInfo', ['pid', 'ppid', 'rss']) 246 | 247 | def find_children(parent_pid, procs): 248 | children = [] 249 | for proc in procs: 250 | pid, ppid, rss = proc 251 | if ppid == parent_pid: 252 | children.append(proc) 253 | children.extend(find_children(pid, procs)) 254 | return children 255 | 256 | def cum_rss(pid, procs): 257 | parent_proc = [p for p in procs if p.pid == pid][0] 258 | children = find_children(pid, procs) 259 | tree = [parent_proc] + children 260 | total_rss = sum(map(int, [p.rss for p in tree])) 261 | return total_rss 262 | 263 | def get_all_process_infos(data): 264 | data = data.strip() 265 | procs = [] 266 | for line in data.splitlines(): 267 | pid, ppid, rss = map(int, line.split()) 268 | procs.append(ProcInfo(pid=pid, ppid=ppid, rss=rss)) 269 | return procs 270 | 271 | if self.cumulative: 272 | data = shell(self.pstreecommand) 273 | procs = get_all_process_infos(data) 274 | 275 | try: 276 | rss = cum_rss(pid, procs) 277 | except (ValueError, IndexError): 278 | # Could not determine cumulative RSS 279 | return None 280 | 281 | else: 282 | data = shell(self.pscommand % pid) 283 | if not data: 284 | # no such pid (deal with race conditions) 285 | return None 286 | 287 | try: 288 | rss = data.lstrip().rstrip() 289 | rss = int(rss) 290 | except ValueError: 291 | # line doesn't contain any data, or rss cant be intified 292 | return None 293 | 294 | rss = rss * 1024 # rss is in KB 295 | return rss 296 | 297 | def mail(self, email, subject, msg): 298 | body = 'To: %s\n' % self.email 299 | body += 'Subject: %s\n' % subject 300 | body += '\n' 301 | body += msg 302 | with os.popen(self.sendmail, 'w') as m: 303 | m.write(body) 304 | self.mailed = body 305 | 306 | def parse_namesize(option, value): 307 | try: 308 | name, size = value.split('=') 309 | except ValueError: 310 | print('Unparseable value %r for %r' % (value, option)) 311 | usage() 312 | size = parse_size(option, size) 313 | return name, size 314 | 315 | def parse_size(option, value): 316 | try: 317 | size = byte_size(value) 318 | except: 319 | print('Unparseable byte_size in %r for %r' % (value, option)) 320 | usage() 321 | 322 | return size 323 | 324 | seconds_size = SuffixMultiplier({'s': 1, 325 | 'm': 60, 326 | 'h': 60 * 60, 327 | 'd': 60 * 60 * 24 328 | }) 329 | 330 | def parse_seconds(option, value): 331 | try: 332 | seconds = seconds_size(value) 333 | except: 334 | print('Unparseable value for time in %r for %s' % (value, option)) 335 | usage() 336 | return seconds 337 | 338 | help_request = object() # returned from memmon_from_args to indicate --help 339 | 340 | def memmon_from_args(arguments): 341 | short_args = "hcp:g:a:s:m:n:u:" 342 | long_args = [ 343 | "help", 344 | "cumulative", 345 | "program=", 346 | "group=", 347 | "any=", 348 | "sendmail_program=", 349 | "email=", 350 | "uptime=", 351 | "name=", 352 | ] 353 | 354 | if not arguments: 355 | return None 356 | try: 357 | opts, args = getopt.getopt(arguments, short_args, long_args) 358 | except: 359 | return None 360 | 361 | cumulative = False 362 | programs = {} 363 | groups = {} 364 | any = None 365 | sendmail = '/usr/sbin/sendmail -t -i' 366 | email = None 367 | uptime_limit = maxint 368 | name = None 369 | 370 | for option, value in opts: 371 | 372 | if option in ('-h', '--help'): 373 | return help_request 374 | 375 | if option in ('-c', '--cumulative'): 376 | cumulative = True 377 | 378 | if option in ('-p', '--program'): 379 | name, size = parse_namesize(option, value) 380 | programs[name] = size 381 | 382 | if option in ('-g', '--group'): 383 | name, size = parse_namesize(option, value) 384 | groups[name] = size 385 | 386 | if option in ('-a', '--any'): 387 | size = parse_size(option, value) 388 | any = size 389 | 390 | if option in ('-s', '--sendmail_program'): 391 | sendmail = value 392 | 393 | if option in ('-m', '--email'): 394 | email = value 395 | 396 | if option in ('-u', '--uptime'): 397 | uptime_limit = parse_seconds(option, value) 398 | 399 | if option in ('-n', '--name'): 400 | name = value 401 | 402 | memmon = Memmon(cumulative=cumulative, 403 | programs=programs, 404 | groups=groups, 405 | any=any, 406 | sendmail=sendmail, 407 | email=email, 408 | email_uptime_limit=uptime_limit, 409 | name=name) 410 | return memmon 411 | 412 | def main(): 413 | memmon = memmon_from_args(sys.argv[1:]) 414 | if memmon is help_request: # --help 415 | usage(exitstatus=0) 416 | elif memmon is None: # something went wrong 417 | usage() 418 | memmon.rpc = childutils.getRPCInterface(os.environ) 419 | memmon.runforever() 420 | 421 | if __name__ == '__main__': 422 | main() 423 | -------------------------------------------------------------------------------- /superlance/process_state_email_monitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python -u 2 | ############################################################################## 3 | # 4 | # Copyright (c) 2007 Agendaless Consulting and Contributors. 5 | # All Rights Reserved. 6 | # 7 | # This software is subject to the provisions of the BSD-like license at 8 | # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany 9 | # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL 10 | # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, 11 | # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND 12 | # FITNESS FOR A PARTICULAR PURPOSE 13 | # 14 | ############################################################################## 15 | import copy 16 | import optparse 17 | import os 18 | import smtplib 19 | import sys 20 | 21 | from email.mime.text import MIMEText 22 | from email.utils import formatdate, make_msgid 23 | from superlance.process_state_monitor import ProcessStateMonitor 24 | 25 | doc = """\ 26 | Base class for common functionality when monitoring process state changes 27 | and sending email notification 28 | """ 29 | 30 | class ProcessStateEmailMonitor(ProcessStateMonitor): 31 | COMMASPACE = ', ' 32 | 33 | @classmethod 34 | def _get_opt_parser(cls): 35 | parser = optparse.OptionParser() 36 | parser.add_option("-i", "--interval", dest="interval", type="float", default=1.0, 37 | help="batch interval in minutes (defaults to 1 minute)") 38 | parser.add_option("-t", "--toEmail", dest="to_emails", 39 | help="destination email address(es) - comma separated") 40 | parser.add_option("-f", "--fromEmail", dest="from_email", 41 | help="source email address") 42 | parser.add_option("-s", "--subject", dest="subject", 43 | help="email subject") 44 | parser.add_option("-H", "--smtpHost", dest="smtp_host", default="localhost", 45 | help="SMTP server hostname or address") 46 | parser.add_option("-e", "--tickEvent", dest="eventname", default="TICK_60", 47 | help="TICK event name (defaults to TICK_60)") 48 | parser.add_option("-u", "--userName", dest="smtp_user", default="", 49 | help="SMTP server user name (defaults to nothing)") 50 | parser.add_option("-p", "--password", dest="smtp_password", default="", 51 | help="SMTP server password (defaults to nothing)") 52 | parser.add_option("--tls", dest="use_tls", action="store_true", default=False, 53 | help="Use Transport Layer Security (TLS), default to False") 54 | return parser 55 | 56 | @classmethod 57 | def parse_cmd_line_options(cls): 58 | parser = cls._get_opt_parser() 59 | (options, args) = parser.parse_args() 60 | return options 61 | 62 | @classmethod 63 | def validate_cmd_line_options(cls, options): 64 | parser = cls._get_opt_parser() 65 | if not options.to_emails: 66 | parser.print_help() 67 | sys.exit(1) 68 | if not options.from_email: 69 | parser.print_help() 70 | sys.exit(1) 71 | 72 | validated = copy.copy(options) 73 | validated.to_emails = [x.strip() for x in options.to_emails.split(",")] 74 | return validated 75 | 76 | @classmethod 77 | def get_cmd_line_options(cls): 78 | return cls.validate_cmd_line_options(cls.parse_cmd_line_options()) 79 | 80 | @classmethod 81 | def create_from_cmd_line(cls): 82 | options = cls.get_cmd_line_options() 83 | 84 | if not 'SUPERVISOR_SERVER_URL' in os.environ: 85 | sys.stderr.write('Must run as a supervisor event listener\n') 86 | sys.exit(1) 87 | 88 | return cls(**options.__dict__) 89 | 90 | def __init__(self, **kwargs): 91 | ProcessStateMonitor.__init__(self, **kwargs) 92 | 93 | self.from_email = kwargs['from_email'] 94 | self.to_emails = kwargs['to_emails'] 95 | self.subject = kwargs.get('subject') 96 | self.smtp_host = kwargs.get('smtp_host', 'localhost') 97 | self.smtp_user = kwargs.get('smtp_user') 98 | self.smtp_password = kwargs.get('smtp_password') 99 | self.use_tls = kwargs.get('use_tls') 100 | self.digest_len = 76 101 | 102 | def send_batch_notification(self): 103 | email = self.get_batch_email() 104 | if email: 105 | self.send_email(email) 106 | self.log_email(email) 107 | 108 | def log_email(self, email): 109 | email_for_log = copy.copy(email) 110 | email_for_log['to'] = self.COMMASPACE.join(email['to']) 111 | if len(email_for_log['body']) > self.digest_len: 112 | email_for_log['body'] = '%s...' % email_for_log['body'][:self.digest_len] 113 | self.write_stderr("Sending notification email:\nTo: %(to)s\n\ 114 | From: %(from)s\nSubject: %(subject)s\nBody:\n%(body)s\n" % email_for_log) 115 | 116 | def get_batch_email(self): 117 | if len(self.batchmsgs): 118 | return { 119 | 'to': self.to_emails, 120 | 'from': self.from_email, 121 | 'subject': self.subject, 122 | 'body': '\n'.join(self.get_batch_msgs()), 123 | } 124 | return None 125 | 126 | def send_email(self, email): 127 | msg = MIMEText(email['body']) 128 | if self.subject: 129 | msg['Subject'] = email['subject'] 130 | msg['From'] = email['from'] 131 | msg['To'] = self.COMMASPACE.join(email['to']) 132 | msg['Date'] = formatdate() 133 | msg['Message-ID'] = make_msgid() 134 | 135 | try: 136 | self.send_smtp(msg, email['to']) 137 | except Exception as e: 138 | self.write_stderr("Error sending email: %s\n" % e) 139 | 140 | def send_smtp(self, mime_msg, to_emails): 141 | s = smtplib.SMTP(self.smtp_host) 142 | try: 143 | if self.smtp_user and self.smtp_password: 144 | if self.use_tls: 145 | s.starttls() 146 | s.login(self.smtp_user, self.smtp_password) 147 | s.sendmail(mime_msg['From'], to_emails, mime_msg.as_string()) 148 | except: 149 | s.quit() 150 | raise 151 | s.quit() 152 | 153 | -------------------------------------------------------------------------------- /superlance/process_state_monitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python -u 2 | ############################################################################## 3 | # 4 | # Copyright (c) 2007 Agendaless Consulting and Contributors. 5 | # All Rights Reserved. 6 | # 7 | # This software is subject to the provisions of the BSD-like license at 8 | # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany 9 | # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL 10 | # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, 11 | # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND 12 | # FITNESS FOR A PARTICULAR PURPOSE 13 | # 14 | ############################################################################## 15 | doc = """\ 16 | Base class for common functionality when monitoring process state changes 17 | """ 18 | 19 | import sys 20 | 21 | from supervisor import childutils 22 | 23 | class ProcessStateMonitor: 24 | 25 | # In child class, define a list of events to monitor 26 | process_state_events = [] 27 | 28 | def __init__(self, **kwargs): 29 | self.interval = kwargs.get('interval', 1.0) 30 | 31 | self.debug = kwargs.get('debug', False) 32 | self.stdin = kwargs.get('stdin', sys.stdin) 33 | self.stdout = kwargs.get('stdout', sys.stdout) 34 | self.stderr = kwargs.get('stderr', sys.stderr) 35 | self.eventname = kwargs.get('eventname', 'TICK_60') 36 | self.tickmins = self._get_tick_mins(self.eventname) 37 | 38 | self.batchmsgs = [] 39 | self.batchmins = 0.0 40 | 41 | def _get_tick_mins(self, eventname): 42 | return float(self._get_tick_secs(eventname))/60.0 43 | 44 | def _get_tick_secs(self, eventname): 45 | self._validate_tick_name(eventname) 46 | return int(eventname.split('_')[1]) 47 | 48 | def _validate_tick_name(self, eventname): 49 | if not eventname.startswith('TICK_'): 50 | raise ValueError("Invalid TICK event name: %s" % eventname) 51 | 52 | def run(self): 53 | while 1: 54 | hdrs, payload = childutils.listener.wait(self.stdin, self.stdout) 55 | self.handle_event(hdrs, payload) 56 | childutils.listener.ok(self.stdout) 57 | 58 | def handle_event(self, headers, payload): 59 | if headers['eventname'] in self.process_state_events: 60 | self.handle_process_state_change_event(headers, payload) 61 | elif headers['eventname'] == self.eventname: 62 | self.handle_tick_event(headers, payload) 63 | 64 | def handle_process_state_change_event(self, headers, payload): 65 | msg = self.get_process_state_change_msg(headers, payload) 66 | if msg: 67 | self.write_stderr('%s\n' % msg) 68 | self.batchmsgs.append(msg) 69 | 70 | """ 71 | Override this method in child classes to customize messaging 72 | """ 73 | def get_process_state_change_msg(self, headers, payload): 74 | return None 75 | 76 | def handle_tick_event(self, headers, payload): 77 | self.batchmins += self.tickmins 78 | if self.batchmins >= self.interval: 79 | self.send_batch_notification() 80 | self.clear_batch() 81 | 82 | """ 83 | Override this method in child classes to send notification 84 | """ 85 | def send_batch_notification(self): 86 | pass 87 | 88 | def get_batch_minutes(self): 89 | return self.batchmins 90 | 91 | def get_batch_msgs(self): 92 | return self.batchmsgs 93 | 94 | def clear_batch(self): 95 | self.batchmins = 0.0 96 | self.batchmsgs = [] 97 | 98 | def write_stderr(self, msg): 99 | self.stderr.write(msg) 100 | self.stderr.flush() 101 | -------------------------------------------------------------------------------- /superlance/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Supervisor/superlance/49f94596cbe22a981cb59ef9ac955ee5b0b49bc7/superlance/tests/__init__.py -------------------------------------------------------------------------------- /superlance/tests/dummy.py: -------------------------------------------------------------------------------- 1 | import time 2 | from supervisor.process import ProcessStates 3 | 4 | _NOW = time.time() 5 | 6 | class DummyRPCServer: 7 | def __init__(self): 8 | self.supervisor = DummySupervisorRPCNamespace() 9 | self.system = DummySystemRPCNamespace() 10 | 11 | class DummyResponse: 12 | status = 200 13 | reason = 'OK' 14 | body = 'OK' 15 | def read(self): 16 | return self.body 17 | 18 | class DummySystemRPCNamespace: 19 | pass 20 | 21 | class DummySupervisorRPCNamespace: 22 | _restartable = True 23 | _restarted = False 24 | _shutdown = False 25 | _readlog_error = False 26 | 27 | 28 | all_process_info = [ 29 | { 30 | 'name':'foo', 31 | 'group':'foo', 32 | 'pid':11, 33 | 'state':ProcessStates.RUNNING, 34 | 'statename':'RUNNING', 35 | 'start':_NOW - 100, 36 | 'stop':0, 37 | 'spawnerr':'', 38 | 'now':_NOW, 39 | 'description':'foo description', 40 | }, 41 | { 42 | 'name':'bar', 43 | 'group':'bar', 44 | 'pid':12, 45 | 'state':ProcessStates.FATAL, 46 | 'statename':'FATAL', 47 | 'start':_NOW - 100, 48 | 'stop':_NOW - 50, 49 | 'spawnerr':'screwed', 50 | 'now':_NOW, 51 | 'description':'bar description', 52 | }, 53 | { 54 | 'name':'baz_01', 55 | 'group':'baz', 56 | 'pid':12, 57 | 'state':ProcessStates.STOPPED, 58 | 'statename':'STOPPED', 59 | 'start':_NOW - 100, 60 | 'stop':_NOW - 25, 61 | 'spawnerr':'', 62 | 'now':_NOW, 63 | 'description':'baz description', 64 | }, 65 | ] 66 | 67 | def getAllProcessInfo(self): 68 | return self.all_process_info 69 | 70 | def getProcessInfo(self, name): 71 | for info in self.all_process_info: 72 | if info['name'] == name or name == '%s:%s' %(info['group'], info['name']): 73 | return info 74 | return None 75 | 76 | def startProcess(self, name): 77 | from supervisor import xmlrpc 78 | from superlance.compat import xmlrpclib 79 | if name.endswith('SPAWN_ERROR'): 80 | raise xmlrpclib.Fault(xmlrpc.Faults.SPAWN_ERROR, 'SPAWN_ERROR') 81 | return True 82 | 83 | def stopProcess(self, name): 84 | from supervisor import xmlrpc 85 | from superlance.compat import xmlrpclib 86 | if name == 'BAD_NAME:BAD_NAME': 87 | raise xmlrpclib.Fault(xmlrpc.Faults.BAD_NAME, 'BAD_NAME:BAD_NAME') 88 | if name.endswith('FAILED'): 89 | raise xmlrpclib.Fault(xmlrpc.Faults.FAILED, 'FAILED') 90 | return True 91 | 92 | -------------------------------------------------------------------------------- /superlance/tests/test_crashmail.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from superlance.compat import StringIO 3 | 4 | class CrashMailTests(unittest.TestCase): 5 | def _getTargetClass(self): 6 | from superlance.crashmail import CrashMail 7 | return CrashMail 8 | 9 | def _makeOne(self, *opts): 10 | return self._getTargetClass()(*opts) 11 | 12 | def setUp(self): 13 | import tempfile 14 | self.tempdir = tempfile.mkdtemp() 15 | 16 | def tearDown(self): 17 | import shutil 18 | shutil.rmtree(self.tempdir) 19 | 20 | def _makeOnePopulated(self, programs, any, response=None): 21 | import os 22 | sendmail = 'cat - > %s' % os.path.join(self.tempdir, 'email.log') 23 | email = 'chrism@plope.com' 24 | header = '[foo]' 25 | prog = self._makeOne(programs, any, email, sendmail, header) 26 | prog.stdin = StringIO() 27 | prog.stdout = StringIO() 28 | prog.stderr = StringIO() 29 | return prog 30 | 31 | def test_runforever_not_process_state_exited(self): 32 | programs = {'foo':0, 'bar':0, 'baz_01':0 } 33 | any = None 34 | prog = self._makeOnePopulated(programs, any) 35 | prog.stdin.write('eventname:PROCESS_STATE len:0\n') 36 | prog.stdin.seek(0) 37 | prog.runforever(test=True) 38 | self.assertEqual(prog.stderr.getvalue(), 'non-exited event\n') 39 | 40 | def test_runforever_expected_exit(self): 41 | programs = ['foo'] 42 | any = None 43 | prog = self._makeOnePopulated(programs, any) 44 | payload=('expected:1 processname:foo groupname:bar ' 45 | 'from_state:RUNNING pid:1') 46 | prog.stdin.write( 47 | 'eventname:PROCESS_STATE_EXITED len:%s\n' % len(payload)) 48 | prog.stdin.write(payload) 49 | prog.stdin.seek(0) 50 | prog.runforever(test=True) 51 | self.assertEqual(prog.stderr.getvalue(), 'expected exit\n') 52 | 53 | def test_runforever_unexpected_exit(self): 54 | programs = ['foo'] 55 | any = None 56 | prog = self._makeOnePopulated(programs, any) 57 | payload=('expected:0 processname:foo groupname:bar ' 58 | 'from_state:RUNNING pid:1') 59 | prog.stdin.write( 60 | 'eventname:PROCESS_STATE_EXITED len:%s\n' % len(payload)) 61 | prog.stdin.write(payload) 62 | prog.stdin.seek(0) 63 | prog.runforever(test=True) 64 | output = prog.stderr.getvalue() 65 | lines = output.split('\n') 66 | self.assertEqual(lines[0], 'unexpected exit, mailing') 67 | self.assertEqual(lines[1], 'Mailed:') 68 | self.assertEqual(lines[2], '') 69 | self.assertEqual(lines[3], 'To: chrism@plope.com') 70 | self.assertTrue('Subject: [foo]: foo crashed at' in lines[4]) 71 | self.assertEqual(lines[5], '') 72 | self.assertTrue( 73 | 'Process foo in group bar exited unexpectedly' in lines[6]) 74 | import os 75 | f = open(os.path.join(self.tempdir, 'email.log'), 'r') 76 | mail = f.read() 77 | f.close() 78 | self.assertTrue( 79 | 'Process foo in group bar exited unexpectedly' in mail) 80 | 81 | if __name__ == '__main__': 82 | unittest.main() 83 | -------------------------------------------------------------------------------- /superlance/tests/test_crashmailbatch.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | try: # pragma: no cover 3 | from unittest.mock import Mock 4 | except ImportError: # pragma: no cover 5 | from mock import Mock 6 | from superlance.compat import StringIO 7 | 8 | class CrashMailBatchTests(unittest.TestCase): 9 | from_email = 'testFrom@blah.com' 10 | to_emails = ('testTo@blah.com') 11 | subject = 'Test Alert' 12 | unexpected_err_msg = 'Process bar:foo (pid 58597) died unexpectedly' 13 | 14 | def _get_target_class(self): 15 | from superlance.crashmailbatch import CrashMailBatch 16 | return CrashMailBatch 17 | 18 | def _make_one_mocked(self, **kwargs): 19 | kwargs['stdin'] = StringIO() 20 | kwargs['stdout'] = StringIO() 21 | kwargs['stderr'] = StringIO() 22 | kwargs['from_email'] = kwargs.get('from_email', self.from_email) 23 | kwargs['to_emails'] = kwargs.get('to_emails', self.to_emails) 24 | kwargs['subject'] = kwargs.get('subject', self.subject) 25 | 26 | obj = self._get_target_class()(**kwargs) 27 | obj.send_email = Mock() 28 | return obj 29 | 30 | def get_process_exited_event(self, pname, gname, expected): 31 | headers = { 32 | 'ver': '3.0', 'poolserial': '7', 'len': '71', 33 | 'server': 'supervisor', 'eventname': 'PROCESS_STATE_EXITED', 34 | 'serial': '7', 'pool': 'checkmailbatch', 35 | } 36 | payload = 'processname:%s groupname:%s from_state:RUNNING expected:%d \ 37 | pid:58597' % (pname, gname, expected) 38 | return (headers, payload) 39 | 40 | def test_get_process_state_change_msg_expected(self): 41 | crash = self._make_one_mocked() 42 | hdrs, payload = self.get_process_exited_event('foo', 'bar', 1) 43 | self.assertEqual(None, crash.get_process_state_change_msg(hdrs, payload)) 44 | 45 | def test_get_process_state_change_msg_unexpected(self): 46 | crash = self._make_one_mocked() 47 | hdrs, payload = self.get_process_exited_event('foo', 'bar', 0) 48 | msg = crash.get_process_state_change_msg(hdrs, payload) 49 | self.assertTrue(self.unexpected_err_msg in msg) 50 | 51 | def test_handle_event_exit_expected(self): 52 | crash = self._make_one_mocked() 53 | hdrs, payload = self.get_process_exited_event('foo', 'bar', 1) 54 | crash.handle_event(hdrs, payload) 55 | self.assertEqual([], crash.get_batch_msgs()) 56 | self.assertEqual('', crash.stderr.getvalue()) 57 | 58 | def test_handle_event_exit_unexpected(self): 59 | crash = self._make_one_mocked() 60 | hdrs, payload = self.get_process_exited_event('foo', 'bar', 0) 61 | crash.handle_event(hdrs, payload) 62 | msgs = crash.get_batch_msgs() 63 | self.assertEqual(1, len(msgs)) 64 | self.assertTrue(self.unexpected_err_msg in msgs[0]) 65 | self.assertTrue(self.unexpected_err_msg in crash.stderr.getvalue()) 66 | 67 | def test_sets_default_subject_when_None(self): 68 | crash = self._make_one_mocked(subject=None) # see issue #109 69 | self.assertEqual(crash.subject, "Crash alert from supervisord") 70 | 71 | if __name__ == '__main__': 72 | unittest.main() 73 | -------------------------------------------------------------------------------- /superlance/tests/test_crashsms.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from .test_crashmailbatch import CrashMailBatchTests 4 | 5 | class CrashSMSTests(CrashMailBatchTests): 6 | subject = None 7 | unexpected_err_msg = '[bar:foo](58597) exited unexpectedly' 8 | 9 | def _get_target_class(self): 10 | from superlance.crashsms import CrashSMS 11 | return CrashSMS 12 | 13 | def test_sets_default_subject_when_None(self): 14 | crash = self._make_one_mocked(subject=None) 15 | self.assertEqual(crash.subject, self.subject) 16 | 17 | if __name__ == '__main__': 18 | unittest.main() 19 | -------------------------------------------------------------------------------- /superlance/tests/test_fatalmailbatch.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | try: # pragma: no cover 3 | from unittest.mock import Mock 4 | except ImportError: # pragma: no cover 5 | from mock import Mock 6 | from superlance.compat import StringIO 7 | 8 | class FatalMailBatchTests(unittest.TestCase): 9 | from_email = 'testFrom@blah.com' 10 | to_emails = ('testTo@blah.com') 11 | subject = 'Test Alert' 12 | unexpected_err_msg = 'Process bar:foo failed to start too many times' 13 | 14 | def _get_target_class(self): 15 | from superlance.fatalmailbatch import FatalMailBatch 16 | return FatalMailBatch 17 | 18 | def _make_one_mocked(self, **kwargs): 19 | kwargs['stdin'] = StringIO() 20 | kwargs['stdout'] = StringIO() 21 | kwargs['stderr'] = StringIO() 22 | kwargs['from_email'] = kwargs.get('from_email', self.from_email) 23 | kwargs['to_emails'] = kwargs.get('to_emails', self.to_emails) 24 | kwargs['subject'] = kwargs.get('subject', self.subject) 25 | 26 | obj = self._get_target_class()(**kwargs) 27 | obj.send_email = Mock() 28 | return obj 29 | 30 | def get_process_fatal_event(self, pname, gname): 31 | headers = { 32 | 'ver': '3.0', 'poolserial': '7', 'len': '71', 33 | 'server': 'supervisor', 'eventname': 'PROCESS_STATE_FATAL', 34 | 'serial': '7', 'pool': 'checkmailbatch', 35 | } 36 | payload = 'processname:%s groupname:%s from_state:BACKOFF' \ 37 | % (pname, gname) 38 | return (headers, payload) 39 | 40 | def test_get_process_state_change_msg(self): 41 | crash = self._make_one_mocked() 42 | hdrs, payload = self.get_process_fatal_event('foo', 'bar') 43 | msg = crash.get_process_state_change_msg(hdrs, payload) 44 | self.assertTrue(self.unexpected_err_msg in msg) 45 | 46 | def test_sets_default_subject_when_None(self): 47 | crash = self._make_one_mocked(subject=None) # see issue #109 48 | self.assertEqual(crash.subject, "Fatal start alert from supervisord") 49 | 50 | if __name__ == '__main__': 51 | unittest.main() 52 | -------------------------------------------------------------------------------- /superlance/tests/test_httpok.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import time 3 | import unittest 4 | from superlance.compat import StringIO 5 | from supervisor.process import ProcessStates 6 | from superlance.tests.dummy import DummyResponse 7 | from superlance.tests.dummy import DummyRPCServer 8 | from superlance.tests.dummy import DummySupervisorRPCNamespace 9 | 10 | _NOW = time.time() 11 | 12 | _FAIL = [ { 13 | 'name':'FAILED', 14 | 'group':'foo', 15 | 'pid':11, 16 | 'state':ProcessStates.RUNNING, 17 | 'statename':'RUNNING', 18 | 'start':_NOW - 100, 19 | 'stop':0, 20 | 'spawnerr':'', 21 | 'now':_NOW, 22 | 'description':'foo description', 23 | }, 24 | { 25 | 'name':'SPAWN_ERROR', 26 | 'group':'foo', 27 | 'pid':11, 28 | 'state':ProcessStates.RUNNING, 29 | 'statename':'RUNNING', 30 | 'start':_NOW - 100, 31 | 'stop':0, 32 | 'spawnerr':'', 33 | 'now':_NOW, 34 | 'description':'foo description', 35 | },] 36 | 37 | def make_connection(response, exc=None): 38 | class TestConnection: 39 | def __init__(self, hostport): 40 | self.hostport = hostport 41 | 42 | def request(self, method, path, headers): 43 | if exc: 44 | if exc is True: 45 | raise ValueError('foo') 46 | else: 47 | raise exc.pop() 48 | self.method = method 49 | self.path = path 50 | self.headers = headers 51 | 52 | def getresponse(self): 53 | return response 54 | 55 | return TestConnection 56 | 57 | class HTTPOkTests(unittest.TestCase): 58 | def _getTargetClass(self): 59 | from superlance.httpok import HTTPOk 60 | return HTTPOk 61 | 62 | def _makeOne(self, *args, **kwargs): 63 | return self._getTargetClass()(*args, **kwargs) 64 | 65 | def _makeOnePopulated(self, programs, any=None, statuses=None, inbody=None, 66 | eager=True, gcore=None, coredir=None, 67 | response=None, exc=None, name=None, 68 | timeout=10, retry_time=0): 69 | if statuses is None: 70 | statuses = [200] 71 | if response is None: 72 | response = DummyResponse() 73 | httpok = self._makeOne( 74 | programs=programs, 75 | any=any, 76 | statuses=statuses, 77 | inbody=inbody, 78 | eager=eager, 79 | coredir=coredir, 80 | gcore=gcore, 81 | name=name, 82 | rpc=DummyRPCServer(), 83 | url='http://foo/bar', 84 | timeout=timeout, 85 | email='chrism@plope.com', 86 | sendmail='cat - > /dev/null', 87 | retry_time=retry_time, 88 | ) 89 | httpok.stdin = StringIO() 90 | httpok.stdout = StringIO() 91 | httpok.stderr = StringIO() 92 | httpok.connclass = make_connection(response, exc=exc) 93 | return httpok 94 | 95 | def test_listProcesses_no_programs(self): 96 | programs = [] 97 | any = None 98 | prog = self._makeOnePopulated(programs, any) 99 | specs = list(prog.listProcesses()) 100 | self.assertEqual(len(specs), 0) 101 | 102 | def test_listProcesses_w_RUNNING_programs_default_state(self): 103 | programs = ['foo'] 104 | any = None 105 | prog = self._makeOnePopulated(programs, any) 106 | specs = list(prog.listProcesses()) 107 | self.assertEqual(len(specs), 1) 108 | self.assertEqual(specs[0], 109 | DummySupervisorRPCNamespace.all_process_info[0]) 110 | 111 | def test_listProcesses_w_nonRUNNING_programs_default_state(self): 112 | programs = ['bar'] 113 | any = None 114 | prog = self._makeOnePopulated(programs, any) 115 | specs = list(prog.listProcesses()) 116 | self.assertEqual(len(specs), 1) 117 | self.assertEqual(specs[0], 118 | DummySupervisorRPCNamespace.all_process_info[1]) 119 | 120 | def test_listProcesses_w_nonRUNNING_programs_RUNNING_state(self): 121 | programs = ['bar'] 122 | any = None 123 | prog = self._makeOnePopulated(programs, any) 124 | specs = list(prog.listProcesses(ProcessStates.RUNNING)) 125 | self.assertEqual(len(specs), 0, (prog.programs, specs)) 126 | 127 | def test_runforever_eager_notatick(self): 128 | programs = {'foo':0, 'bar':0, 'baz_01':0 } 129 | any = None 130 | prog = self._makeOnePopulated(programs, any) 131 | prog.stdin.write('eventname:NOTATICK len:0\n') 132 | prog.stdin.seek(0) 133 | prog.runforever(test=True) 134 | self.assertEqual(prog.stderr.getvalue(), '') 135 | 136 | def test_runforever_doesnt_act_if_status_is_expected(self): 137 | statuses = [200, 201] 138 | for status in statuses: 139 | response = DummyResponse() 140 | response.status = status # expected 141 | prog = self._makeOnePopulated( 142 | programs=['foo'], 143 | statuses=statuses, 144 | response=response, 145 | ) 146 | prog.stdin.write('eventname:TICK len:0\n') 147 | prog.stdin.seek(0) 148 | prog.runforever(test=True) 149 | # status is expected so there should be no output 150 | self.assertEqual('', prog.stderr.getvalue()) 151 | 152 | def test_runforever_acts_if_status_is_unexpected(self): 153 | statuses = [200, 201] 154 | response = DummyResponse() 155 | response.status = 500 # unexpected 156 | response.reason = 'Internal Server Error' 157 | prog = self._makeOnePopulated( 158 | programs=['foo'], 159 | statuses=[statuses], 160 | response=response, 161 | ) 162 | prog.stdin.write('eventname:TICK len:0\n') 163 | prog.stdin.seek(0) 164 | prog.runforever(test=True) 165 | lines = prog.stderr.getvalue().split('\n') 166 | self.assertTrue('Subject: httpok: http://foo/bar: ' 167 | 'bad status returned' in lines) 168 | self.assertTrue('status contacting http://foo/bar: ' 169 | '500 Internal Server Error' in lines) 170 | 171 | def test_runforever_doesnt_act_if_inbody_is_present(self): 172 | response = DummyResponse() 173 | response.body = 'It works' 174 | prog = self._makeOnePopulated( 175 | programs=['foo'], 176 | statuses=[response.status], 177 | response=response, 178 | inbody='works', 179 | ) 180 | prog.stdin.write('eventname:TICK len:0\n') 181 | prog.stdin.seek(0) 182 | prog.runforever(test=True) 183 | # body is expected so there should be no output 184 | self.assertEqual('', prog.stderr.getvalue()) 185 | 186 | def test_runforever_acts_if_inbody_isnt_present(self): 187 | response = DummyResponse() 188 | response.body = 'Some kind of error' 189 | prog = self._makeOnePopulated( 190 | programs=['foo'], 191 | statuses=[response.status], 192 | response=response, 193 | inbody="works", 194 | ) 195 | prog.stdin.write('eventname:TICK len:0\n') 196 | prog.stdin.seek(0) 197 | prog.runforever(test=True) 198 | lines = prog.stderr.getvalue().split('\n') 199 | self.assertTrue('Subject: httpok: http://foo/bar: ' 200 | 'bad body returned' in lines) 201 | 202 | def test_runforever_eager_error_on_request_some(self): 203 | programs = ['foo', 'bar', 'baz_01', 'notexisting'] 204 | any = None 205 | prog = self._makeOnePopulated(programs, any, exc=True) 206 | prog.stdin.write('eventname:TICK len:0\n') 207 | prog.stdin.seek(0) 208 | prog.runforever(test=True) 209 | lines = prog.stderr.getvalue().split('\n') 210 | self.assertEqual(lines[0], 211 | ("Restarting selected processes ['foo', 'bar', " 212 | "'baz_01', 'notexisting']") 213 | ) 214 | self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') 215 | self.assertEqual(lines[2], 'foo restarted') 216 | self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') 217 | self.assertEqual(lines[4], 218 | 'baz:baz_01 not in RUNNING state, NOT restarting') 219 | self.assertEqual(lines[5], 220 | "Programs not restarted because they did not exist: ['notexisting']") 221 | mailed = prog.mailed.split('\n') 222 | self.assertEqual(len(mailed), 12) 223 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 224 | self.assertEqual(mailed[1], 225 | 'Subject: httpok: http://foo/bar: bad status returned') 226 | 227 | def test_runforever_eager_error_on_request_any(self): 228 | programs = [] 229 | any = True 230 | prog = self._makeOnePopulated(programs, any, exc=True) 231 | prog.stdin.write('eventname:TICK len:0\n') 232 | prog.stdin.seek(0) 233 | prog.runforever(test=True) 234 | lines = prog.stderr.getvalue().split('\n') 235 | self.assertEqual(lines[0], 'Restarting all running processes') 236 | self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') 237 | self.assertEqual(lines[2], 'foo restarted') 238 | self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') 239 | self.assertEqual(lines[4], 240 | 'baz:baz_01 not in RUNNING state, NOT restarting') 241 | mailed = prog.mailed.split('\n') 242 | self.assertEqual(len(mailed), 11) 243 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 244 | self.assertEqual(mailed[1], 245 | 'Subject: httpok: http://foo/bar: bad status returned') 246 | 247 | def test_runforever_eager_error_on_process_stop(self): 248 | programs = ['FAILED'] 249 | any = False 250 | prog = self._makeOnePopulated(programs, any, exc=True) 251 | prog.rpc.supervisor.all_process_info = _FAIL 252 | prog.stdin.write('eventname:TICK len:0\n') 253 | prog.stdin.seek(0) 254 | prog.runforever(test=True) 255 | lines = prog.stderr.getvalue().split('\n') 256 | self.assertEqual(lines[0], "Restarting selected processes ['FAILED']") 257 | self.assertEqual(lines[1], 'foo:FAILED is in RUNNING state, restarting') 258 | self.assertEqual(lines[2], 259 | "Failed to stop process foo:FAILED: ") 260 | self.assertEqual(lines[3], 'foo:FAILED restarted') 261 | mailed = prog.mailed.split('\n') 262 | self.assertEqual(len(mailed), 10) 263 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 264 | self.assertEqual(mailed[1], 265 | 'Subject: httpok: http://foo/bar: bad status returned') 266 | 267 | def test_runforever_eager_error_on_process_start(self): 268 | programs = ['SPAWN_ERROR'] 269 | any = False 270 | prog = self._makeOnePopulated(programs, any, exc=True) 271 | prog.rpc.supervisor.all_process_info = _FAIL 272 | prog.stdin.write('eventname:TICK len:0\n') 273 | prog.stdin.seek(0) 274 | prog.runforever(test=True) 275 | lines = prog.stderr.getvalue().split('\n') 276 | self.assertEqual(lines[0], 277 | "Restarting selected processes ['SPAWN_ERROR']") 278 | self.assertEqual(lines[1], 279 | 'foo:SPAWN_ERROR is in RUNNING state, restarting') 280 | self.assertEqual(lines[2], 281 | "Failed to start process foo:SPAWN_ERROR: ") 282 | mailed = prog.mailed.split('\n') 283 | self.assertEqual(len(mailed), 9) 284 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 285 | self.assertEqual(mailed[1], 286 | 'Subject: httpok: http://foo/bar: bad status returned') 287 | 288 | def test_runforever_eager_gcore(self): 289 | programs = ['foo', 'bar', 'baz_01', 'notexisting'] 290 | any = None 291 | prog = self._makeOnePopulated(programs, any, exc=True, gcore="true", 292 | coredir="/tmp") 293 | prog.stdin.write('eventname:TICK len:0\n') 294 | prog.stdin.seek(0) 295 | prog.runforever(test=True) 296 | lines = prog.stderr.getvalue().split('\n') 297 | self.assertEqual(lines[0], 298 | ("Restarting selected processes ['foo', 'bar', " 299 | "'baz_01', 'notexisting']") 300 | ) 301 | self.assertEqual(lines[1], 'gcore output for foo:') 302 | self.assertEqual(lines[2], '') 303 | self.assertEqual(lines[3], ' ') 304 | self.assertEqual(lines[4], 'foo is in RUNNING state, restarting') 305 | self.assertEqual(lines[5], 'foo restarted') 306 | self.assertEqual(lines[6], 'bar not in RUNNING state, NOT restarting') 307 | self.assertEqual(lines[7], 308 | 'baz:baz_01 not in RUNNING state, NOT restarting') 309 | self.assertEqual(lines[8], 310 | "Programs not restarted because they did not exist: ['notexisting']") 311 | mailed = prog.mailed.split('\n') 312 | self.assertEqual(len(mailed), 15) 313 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 314 | self.assertEqual(mailed[1], 315 | 'Subject: httpok: http://foo/bar: bad status returned') 316 | 317 | def test_runforever_not_eager_none_running(self): 318 | programs = ['bar', 'baz_01'] 319 | any = None 320 | prog = self._makeOnePopulated(programs, any, exc=True, gcore="true", 321 | coredir="/tmp", eager=False) 322 | prog.stdin.write('eventname:TICK len:0\n') 323 | prog.stdin.seek(0) 324 | prog.runforever(test=True) 325 | lines = [x for x in prog.stderr.getvalue().split('\n') if x] 326 | self.assertEqual(len(lines), 0, lines) 327 | self.assertFalse('mailed' in prog.__dict__) 328 | 329 | def test_runforever_not_eager_running(self): 330 | programs = ['foo', 'bar'] 331 | any = None 332 | prog = self._makeOnePopulated(programs, any, exc=True, eager=False) 333 | prog.stdin.write('eventname:TICK len:0\n') 334 | prog.stdin.seek(0) 335 | prog.runforever(test=True) 336 | lines = [x for x in prog.stderr.getvalue().split('\n') if x] 337 | self.assertEqual(lines[0], 338 | ("Restarting selected processes ['foo', 'bar']") 339 | ) 340 | self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') 341 | self.assertEqual(lines[2], 'foo restarted') 342 | self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') 343 | mailed = prog.mailed.split('\n') 344 | self.assertEqual(len(mailed), 10) 345 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 346 | self.assertEqual(mailed[1], 347 | 'Subject: httpok: http://foo/bar: bad status returned') 348 | 349 | def test_runforever_honor_timeout_on_connrefused(self): 350 | programs = ['foo', 'bar'] 351 | any = None 352 | error = socket.error() 353 | error.errno = 111 354 | prog = self._makeOnePopulated(programs, any, exc=[error], eager=False) 355 | prog.stdin.write('eventname:TICK len:0\n') 356 | prog.stdin.seek(0) 357 | prog.runforever(test=True) 358 | self.assertEqual(prog.stderr.getvalue(), '') 359 | self.assertEqual(prog.stdout.getvalue(), 'READY\nRESULT 2\nOK') 360 | 361 | def test_runforever_connrefused_error(self): 362 | programs = ['foo', 'bar'] 363 | any = None 364 | error = socket.error() 365 | error.errno = 111 366 | prog = self._makeOnePopulated(programs, any, 367 | exc=[error for x in range(100)], eager=False) 368 | prog.stdin.write('eventname:TICK len:0\n') 369 | prog.stdin.seek(0) 370 | prog.runforever(test=True) 371 | lines = [x for x in prog.stderr.getvalue().split('\n') if x] 372 | self.assertEqual(lines[0], 373 | ("Restarting selected processes ['foo', 'bar']") 374 | ) 375 | self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') 376 | self.assertEqual(lines[2], 'foo restarted') 377 | self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') 378 | mailed = prog.mailed.split('\n') 379 | self.assertEqual(len(mailed), 10) 380 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 381 | self.assertEqual(mailed[1], 382 | 'Subject: httpok: http://foo/bar: bad status returned') 383 | 384 | def test_bug_110(self): 385 | error = socket.error() 386 | error.errno = 111 387 | prog = self._makeOnePopulated(programs=['foo'], any=None, 388 | exc=[error for x in range(100)], eager=False, 389 | timeout=1, retry_time=10) 390 | prog.stdin.write('eventname:TICK len:0\n') 391 | prog.stdin.seek(0) 392 | prog.runforever(test=True) 393 | lines = [x for x in prog.stderr.getvalue().split('\n') if x] 394 | self.assertEqual(lines[0], 395 | ("Restarting selected processes ['foo']") 396 | ) 397 | self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') 398 | self.assertEqual(lines[2], 'foo restarted') 399 | 400 | def test_subject_no_name(self): 401 | """set the name to None to check if subject formats to: 402 | httpok: %(subject)s 403 | """ 404 | prog = self._makeOnePopulated( 405 | programs=['foo', 'bar'], 406 | any=None, 407 | eager=False, 408 | exc=[ValueError('this causes status to be None')], 409 | name=None, 410 | ) 411 | prog.stdin.write('eventname:TICK len:0\n') 412 | prog.stdin.seek(0) 413 | prog.runforever(test=True) 414 | mailed = prog.mailed.split('\n') 415 | self.assertEqual(mailed[1], 416 | 'Subject: httpok: http://foo/bar: bad status returned') 417 | 418 | def test_subject_with_name(self): 419 | """set the name to a string to check if subject formats to: 420 | httpok [%(name)s]: %(subject)s 421 | """ 422 | prog = self._makeOnePopulated( 423 | programs=['foo', 'bar'], 424 | any=None, 425 | eager=False, 426 | exc=[ValueError('this causes status to be None')], 427 | name='thinko', 428 | ) 429 | prog.stdin.write('eventname:TICK len:0\n') 430 | prog.stdin.seek(0) 431 | prog.runforever(test=True) 432 | mailed = prog.mailed.split('\n') 433 | self.assertEqual(mailed[1], 434 | 'Subject: httpok [thinko]: http://foo/bar: bad status returned') 435 | 436 | if __name__ == '__main__': 437 | unittest.main() 438 | -------------------------------------------------------------------------------- /superlance/tests/test_memmon.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import unittest 3 | from superlance.compat import StringIO 4 | from superlance.compat import maxint 5 | from superlance.memmon import ( 6 | help_request, 7 | memmon_from_args, 8 | seconds_size 9 | ) 10 | from superlance.tests.dummy import DummyRPCServer 11 | 12 | class MemmonTests(unittest.TestCase): 13 | def _getTargetClass(self): 14 | from superlance.memmon import Memmon 15 | return Memmon 16 | 17 | def _makeOne(self, *args, **kwargs): 18 | return self._getTargetClass()(*args, **kwargs) 19 | 20 | def _makeOnePopulated(self, programs, groups, any, name=None): 21 | memmon = self._makeOne( 22 | programs=programs, 23 | groups=groups, 24 | any=any, 25 | name=name, 26 | rpc=DummyRPCServer(), 27 | cumulative=False, 28 | sendmail='cat - > /dev/null', 29 | email='chrism@plope.com', 30 | email_uptime_limit=2000, 31 | ) 32 | memmon.stdin = StringIO() 33 | memmon.stdout = StringIO() 34 | memmon.stderr = StringIO() 35 | memmon.pscommand = 'echo 22%s' 36 | return memmon 37 | 38 | def test_runforever_notatick(self): 39 | programs = {'foo':0, 'bar':0, 'baz_01':0 } 40 | groups = {} 41 | any = None 42 | memmon = self._makeOnePopulated(programs, groups, any) 43 | memmon.stdin.write('eventname:NOTATICK len:0\n') 44 | memmon.stdin.seek(0) 45 | memmon.runforever(test=True) 46 | self.assertEqual(memmon.stderr.getvalue(), '') 47 | 48 | def test_runforever_tick_programs(self): 49 | programs = {'foo':0, 'bar':0, 'baz_01':0 } 50 | groups = {} 51 | any = None 52 | memmon = self._makeOnePopulated(programs, groups, any) 53 | memmon.stdin.write('eventname:TICK len:0\n') 54 | memmon.stdin.seek(0) 55 | memmon.runforever(test=True) 56 | lines = memmon.stderr.getvalue().split('\n') 57 | self.assertEqual(len(lines), 8) 58 | self.assertEqual(lines[0], 'Checking programs bar=0, baz_01=0, foo=0') 59 | self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') 60 | self.assertEqual(lines[2], 'Restarting foo:foo') 61 | self.assertEqual(lines[3], 'RSS of bar:bar is 2265088') 62 | self.assertEqual(lines[4], 'Restarting bar:bar') 63 | self.assertEqual(lines[5], 'RSS of baz:baz_01 is 2265088') 64 | self.assertEqual(lines[6], 'Restarting baz:baz_01') 65 | self.assertEqual(lines[7], '') 66 | mailed = memmon.mailed.split('\n') 67 | self.assertEqual(len(mailed), 4) 68 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 69 | self.assertEqual(mailed[1], 70 | 'Subject: memmon: process baz:baz_01 restarted') 71 | self.assertEqual(mailed[2], '') 72 | self.assertTrue(mailed[3].startswith('memmon.py restarted')) 73 | 74 | def test_runforever_tick_groups(self): 75 | programs = {} 76 | groups = {'foo':0} 77 | any = None 78 | memmon = self._makeOnePopulated(programs, groups, any) 79 | memmon.stdin.write('eventname:TICK len:0\n') 80 | memmon.stdin.seek(0) 81 | memmon.runforever(test=True) 82 | lines = memmon.stderr.getvalue().split('\n') 83 | self.assertEqual(len(lines), 4) 84 | self.assertEqual(lines[0], 'Checking groups foo=0') 85 | self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') 86 | self.assertEqual(lines[2], 'Restarting foo:foo') 87 | self.assertEqual(lines[3], '') 88 | mailed = memmon.mailed.split('\n') 89 | self.assertEqual(len(mailed), 4) 90 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 91 | self.assertEqual(mailed[1], 92 | 'Subject: memmon: process foo:foo restarted') 93 | self.assertEqual(mailed[2], '') 94 | self.assertTrue(mailed[3].startswith('memmon.py restarted')) 95 | 96 | def test_runforever_tick_any(self): 97 | programs = {} 98 | groups = {} 99 | any = 0 100 | memmon = self._makeOnePopulated(programs, groups, any) 101 | memmon.stdin.write('eventname:TICK len:0\n') 102 | memmon.stdin.seek(0) 103 | memmon.runforever(test=True) 104 | lines = memmon.stderr.getvalue().split('\n') 105 | self.assertEqual(len(lines), 8) 106 | self.assertEqual(lines[0], 'Checking any=0') 107 | self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') 108 | self.assertEqual(lines[2], 'Restarting foo:foo') 109 | self.assertEqual(lines[3], 'RSS of bar:bar is 2265088') 110 | self.assertEqual(lines[4], 'Restarting bar:bar') 111 | self.assertEqual(lines[5], 'RSS of baz:baz_01 is 2265088') 112 | self.assertEqual(lines[6], 'Restarting baz:baz_01') 113 | self.assertEqual(lines[7], '') 114 | mailed = memmon.mailed.split('\n') 115 | self.assertEqual(len(mailed), 4) 116 | 117 | def test_runforever_tick_programs_and_groups(self): 118 | programs = {'baz_01':0} 119 | groups = {'foo':0} 120 | any = None 121 | memmon = self._makeOnePopulated(programs, groups, any) 122 | memmon.stdin.write('eventname:TICK len:0\n') 123 | memmon.stdin.seek(0) 124 | memmon.runforever(test=True) 125 | lines = memmon.stderr.getvalue().split('\n') 126 | self.assertEqual(len(lines), 7) 127 | self.assertEqual(lines[0], 'Checking programs baz_01=0') 128 | self.assertEqual(lines[1], 'Checking groups foo=0') 129 | self.assertEqual(lines[2], 'RSS of foo:foo is 2264064') 130 | self.assertEqual(lines[3], 'Restarting foo:foo') 131 | self.assertEqual(lines[4], 'RSS of baz:baz_01 is 2265088') 132 | self.assertEqual(lines[5], 'Restarting baz:baz_01') 133 | self.assertEqual(lines[6], '') 134 | mailed = memmon.mailed.split('\n') 135 | self.assertEqual(len(mailed), 4) 136 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 137 | self.assertEqual(mailed[1], 138 | 'Subject: memmon: process baz:baz_01 restarted') 139 | self.assertEqual(mailed[2], '') 140 | self.assertTrue(mailed[3].startswith('memmon.py restarted')) 141 | 142 | def test_runforever_tick_programs_norestart(self): 143 | programs = {'foo': maxint} 144 | groups = {} 145 | any = None 146 | memmon = self._makeOnePopulated(programs, groups, any) 147 | memmon.stdin.write('eventname:TICK len:0\n') 148 | memmon.stdin.seek(0) 149 | memmon.runforever(test=True) 150 | lines = memmon.stderr.getvalue().split('\n') 151 | self.assertEqual(len(lines), 3) 152 | self.assertEqual(lines[0], 'Checking programs foo=%s' % maxint) 153 | self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') 154 | self.assertEqual(lines[2], '') 155 | self.assertEqual(memmon.mailed, False) 156 | 157 | def test_stopprocess_fault_tick_programs_norestart(self): 158 | programs = {'foo': maxint} 159 | groups = {} 160 | any = None 161 | memmon = self._makeOnePopulated(programs, groups, any) 162 | memmon.stdin.write('eventname:TICK len:0\n') 163 | memmon.stdin.seek(0) 164 | memmon.runforever(test=True) 165 | lines = memmon.stderr.getvalue().split('\n') 166 | self.assertEqual(len(lines), 3) 167 | self.assertEqual(lines[0], 'Checking programs foo=%s' % maxint) 168 | self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') 169 | self.assertEqual(lines[2], '') 170 | self.assertEqual(memmon.mailed, False) 171 | 172 | def test_stopprocess_fails_to_stop(self): 173 | programs = {'BAD_NAME': 0} 174 | groups = {} 175 | any = None 176 | memmon = self._makeOnePopulated(programs, groups, any) 177 | memmon.stdin.write('eventname:TICK len:0\n') 178 | memmon.stdin.seek(0) 179 | from supervisor.process import ProcessStates 180 | memmon.rpc.supervisor.all_process_info = [ { 181 | 'name':'BAD_NAME', 182 | 'group':'BAD_NAME', 183 | 'pid':11, 184 | 'state':ProcessStates.RUNNING, 185 | 'statename':'RUNNING', 186 | 'start':0, 187 | 'stop':0, 188 | 'spawnerr':'', 189 | 'now':0, 190 | 'description':'BAD_NAME description', 191 | } ] 192 | from superlance.compat import xmlrpclib 193 | self.assertRaises(xmlrpclib.Fault, memmon.runforever, True) 194 | lines = memmon.stderr.getvalue().split('\n') 195 | self.assertEqual(len(lines), 4) 196 | self.assertEqual(lines[0], 'Checking programs BAD_NAME=%s' % 0) 197 | self.assertEqual(lines[1], 'RSS of BAD_NAME:BAD_NAME is 2264064') 198 | self.assertEqual(lines[2], 'Restarting BAD_NAME:BAD_NAME') 199 | self.assertTrue(lines[3].startswith('Failed')) 200 | mailed = memmon.mailed.split('\n') 201 | self.assertEqual(len(mailed), 4) 202 | self.assertEqual(mailed[0], 'To: chrism@plope.com') 203 | self.assertEqual(mailed[1], 204 | 'Subject: memmon: failed to stop process BAD_NAME:BAD_NAME, exiting') 205 | self.assertEqual(mailed[2], '') 206 | self.assertTrue(mailed[3].startswith('Failed')) 207 | 208 | def test_subject_no_name(self): 209 | """set the name to None to check if subject formats to: 210 | memmon: %(subject)s 211 | """ 212 | memmon = self._makeOnePopulated( 213 | programs={}, 214 | groups={}, 215 | any=0, 216 | name=None, 217 | ) 218 | memmon.stdin.write('eventname:TICK len:0\n') 219 | memmon.stdin.seek(0) 220 | memmon.runforever(test=True) 221 | 222 | mailed = memmon.mailed.split('\n') 223 | self.assertEqual(mailed[1], 224 | 'Subject: memmon: process baz:baz_01 restarted') 225 | 226 | def test_subject_with_name(self): 227 | """set the name to a string to check if subject formats to: 228 | memmon [%(name)s]: %(subject)s 229 | """ 230 | memmon = self._makeOnePopulated( 231 | programs={}, 232 | groups={}, 233 | any=0, 234 | name='thinko', 235 | ) 236 | memmon.stdin.write('eventname:TICK len:0\n') 237 | memmon.stdin.seek(0) 238 | memmon.runforever(test=True) 239 | 240 | mailed = memmon.mailed.split('\n') 241 | self.assertEqual(mailed[1], 242 | 'Subject: memmon [thinko]: process baz:baz_01 restarted') 243 | 244 | def test_parse_uptime(self): 245 | """test parsing of time parameter for uptime 246 | """ 247 | self.assertEqual(seconds_size('1'), 1, 'default is seconds') 248 | self.assertEqual(seconds_size('1s'), 1, 'seconds suffix is allowed, too') 249 | self.assertEqual(seconds_size('2m'), 120) 250 | self.assertEqual(seconds_size('3h'), 10800) 251 | self.assertEqual(seconds_size('1d'), 86400) 252 | self.assertRaises(ValueError, seconds_size, '1y') 253 | 254 | def test_uptime_short_email(self): 255 | """in case an email is provided and the restarted process' uptime 256 | is shorter than our uptime_limit we do send an email 257 | """ 258 | programs = {'foo':0} 259 | groups = {} 260 | any = None 261 | memmon = self._makeOnePopulated(programs, groups, any) 262 | memmon.email_uptime_limit = 101 263 | 264 | memmon.stdin.write('eventname:TICK len:0\n') 265 | memmon.stdin.seek(0) 266 | memmon.runforever(test=True) 267 | self.assertTrue(memmon.mailed, 'email has been sent') 268 | 269 | #in case uptime == limit, we send an email too 270 | memmon = self._makeOnePopulated(programs, groups, any) 271 | memmon.email_uptime_limit = 100 272 | memmon.stdin.write('eventname:TICK len:0\n') 273 | memmon.stdin.seek(0) 274 | memmon.runforever(test=True) 275 | self.assertTrue(memmon.mailed, 'email has been sent') 276 | 277 | 278 | 279 | def test_uptime_long_no_email(self): 280 | """in case an email is provided and the restarted process' uptime 281 | is longer than our uptime_limit we do not send an email 282 | """ 283 | programs = {'foo':0} 284 | groups = {} 285 | any = None 286 | memmon = self._makeOnePopulated(programs, groups, any) 287 | memmon.email_uptime_limit = 99 288 | 289 | memmon.stdin.write('eventname:TICK len:0\n') 290 | memmon.stdin.seek(0) 291 | memmon.runforever(test=True) 292 | self.assertFalse(memmon.mailed, 'no email should be sent because uptime is above limit') 293 | 294 | def test_calc_rss_not_cumulative(self): 295 | programs = {} 296 | groups = {} 297 | any = None 298 | memmon = self._makeOnePopulated(programs, groups, any) 299 | 300 | noop = '_=%s; ' 301 | pid = 1 302 | 303 | memmon.pscommand = noop + 'echo 16' 304 | rss = memmon.calc_rss(pid) 305 | self.assertEqual(16 * 1024, rss) 306 | 307 | memmon.pscommand = noop + 'echo not_an_int' 308 | rss = memmon.calc_rss(pid) 309 | self.assertEqual( 310 | None, rss, 'Failure to parse an integer RSS value from the ps ' 311 | 'output should result in calc_rss() returning None.') 312 | 313 | def test_calc_rss_cumulative(self): 314 | """Let calc_rss() do its work on a fake process tree: 315 | 316 | ├─┬= 99 317 | │ └─┬= 1 318 | │ └─┬= 2 319 | │ ├─── 3 320 | │ └─── 4 321 | 322 | (Where the process with PID 1 is the one being monitored) 323 | """ 324 | programs = {} 325 | groups = {} 326 | any = None 327 | memmon = self._makeOnePopulated(programs, groups, any) 328 | memmon.cumulative = True 329 | 330 | # output of ps ax -o "pid= ppid= rss=" representing the process 331 | # tree described above, including extraneous whitespace and 332 | # unrelated processes. 333 | ps_output = """ 334 | 11111 22222 333 335 | 1 99 100 336 | 2 1 200 337 | 3 2 300 338 | 4 2 400 339 | 11111 22222 333 340 | """ 341 | 342 | memmon.pstreecommand = 'echo "%s"' % ps_output 343 | rss = memmon.calc_rss(1) 344 | self.assertEqual( 345 | 1000 * 1024, rss, 346 | 'Cumulative RSS of the test process and its three children ' 347 | 'should add up to 1000 kb.') 348 | 349 | def test_argparser(self): 350 | """test if arguments are parsed correctly 351 | """ 352 | # help 353 | arguments = ['-h', ] 354 | memmon = memmon_from_args(arguments) 355 | self.assertTrue(memmon is help_request, 356 | '-h returns help_request to make main() script print usage' 357 | ) 358 | 359 | #all arguments 360 | arguments = ['-c', 361 | '-p', 'foo=50MB', 362 | '-g', 'bar=10kB', 363 | '--any', '250', 364 | '-s', 'mutt', 365 | '-m', 'me@you.com', 366 | '-u', '1d', 367 | '-n', 'myproject'] 368 | memmon = memmon_from_args(arguments) 369 | self.assertEqual(memmon.cumulative, True) 370 | self.assertEqual(memmon.programs['foo'], 50 * 1024 * 1024) 371 | self.assertEqual(memmon.groups['bar'], 10 * 1024) 372 | self.assertEqual(memmon.any, 250) 373 | self.assertEqual(memmon.sendmail, 'mutt') 374 | self.assertEqual(memmon.email, 'me@you.com') 375 | self.assertEqual(memmon.email_uptime_limit, 1 * 24 * 60 * 60) 376 | self.assertEqual(memmon.name, 'myproject') 377 | 378 | 379 | #default arguments 380 | arguments = ['-m', 'me@you.com'] 381 | memmon = memmon_from_args(arguments) 382 | self.assertEqual(memmon.cumulative, False) 383 | self.assertEqual(memmon.programs, {}) 384 | self.assertEqual(memmon.groups, {}) 385 | self.assertEqual(memmon.any, None) 386 | self.assertTrue('sendmail' in memmon.sendmail, 'not using sendmail as default') 387 | self.assertEqual(memmon.email_uptime_limit, maxint) 388 | self.assertEqual(memmon.name, None) 389 | 390 | arguments = ['-p', 'foo=50MB'] 391 | memmon = memmon_from_args(arguments) 392 | self.assertEqual(memmon.email, None) 393 | 394 | if __name__ == '__main__': 395 | unittest.main() 396 | 397 | -------------------------------------------------------------------------------- /superlance/tests/test_process_state_email_monitor.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | try: # pragma: no cover 3 | from unittest.mock import Mock 4 | except ImportError: # pragma: no cover 5 | from mock import Mock 6 | from superlance.compat import StringIO 7 | 8 | class ProcessStateEmailMonitorTestException(Exception): 9 | pass 10 | 11 | class ProcessStateEmailMonitorTests(unittest.TestCase): 12 | from_email = 'testFrom@blah.com' 13 | to_emails = ('testTo@blah.com', 'testTo2@blah.com') 14 | to_str = 'testTo@blah.com, testTo2@blah.com' 15 | subject = 'Test Alert' 16 | 17 | def _get_target_class(self): 18 | from superlance.process_state_email_monitor \ 19 | import ProcessStateEmailMonitor 20 | return ProcessStateEmailMonitor 21 | 22 | def _make_one(self, **kwargs): 23 | kwargs['stdin'] = StringIO() 24 | kwargs['stdout'] = StringIO() 25 | kwargs['stderr'] = StringIO() 26 | kwargs['from_email'] = kwargs.get('from_email', self.from_email) 27 | kwargs['to_emails'] = kwargs.get('to_emails', self.to_emails) 28 | kwargs['subject'] = kwargs.get('subject', self.subject) 29 | 30 | obj = self._get_target_class()(**kwargs) 31 | return obj 32 | 33 | def _make_one_mock_send_email(self, **kwargs): 34 | obj = self._make_one(**kwargs) 35 | obj.send_email = Mock() 36 | return obj 37 | 38 | def _make_one_mock_send_smtp(self, **kwargs): 39 | obj = self._make_one(**kwargs) 40 | obj.send_smtp = Mock() 41 | return obj 42 | 43 | def test_validate_cmd_line_options_single_to_email_ok(self): 44 | klass = self._get_target_class() 45 | 46 | options = Mock() 47 | options.from_email = 'blah' 48 | options.to_emails = 'frog' 49 | 50 | validated = klass.validate_cmd_line_options(options) 51 | self.assertEqual(['frog'], validated.to_emails) 52 | 53 | def test_validate_cmd_line_options_multi_to_emails_ok(self): 54 | klass = self._get_target_class() 55 | 56 | options = Mock() 57 | options.from_email = 'blah' 58 | options.to_emails = 'frog, log,dog' 59 | 60 | validated = klass.validate_cmd_line_options(options) 61 | self.assertEqual(['frog', 'log', 'dog'], validated.to_emails) 62 | 63 | def test_send_email_ok(self): 64 | email = { 65 | 'body': 'msg1\nmsg2', 66 | 'to': self.to_emails, 67 | 'from': 'testFrom@blah.com', 68 | 'subject': 'Test Alert', 69 | } 70 | monitor = self._make_one_mock_send_smtp() 71 | monitor.send_email(email) 72 | 73 | # Test that email was sent 74 | self.assertEqual(1, monitor.send_smtp.call_count) 75 | smtpCallArgs = monitor.send_smtp.call_args[0] 76 | mimeMsg = smtpCallArgs[0] 77 | self.assertEqual(self.to_str, mimeMsg['To']) 78 | self.assertEqual(email['from'], mimeMsg['From']) 79 | self.assertEqual(email['subject'], mimeMsg['Subject']) 80 | self.assertEqual(email['body'], mimeMsg.get_payload()) 81 | 82 | def _raiseSTMPException(self, mime, to_emails): 83 | raise ProcessStateEmailMonitorTestException('test') 84 | 85 | def test_send_email_exception(self): 86 | email = { 87 | 'body': 'msg1\nmsg2', 88 | 'to': self.to_emails, 89 | 'from': 'testFrom@blah.com', 90 | 'subject': 'Test Alert', 91 | } 92 | monitor = self._make_one_mock_send_smtp() 93 | monitor.send_smtp.side_effect = self._raiseSTMPException 94 | monitor.send_email(email) 95 | 96 | # Test that error was logged to stderr 97 | self.assertEqual("Error sending email: test\n", monitor.stderr.getvalue()) 98 | 99 | def test_send_batch_notification(self): 100 | test_msgs = ['msg1', 'msg2'] 101 | monitor = self._make_one_mock_send_email() 102 | monitor.batchmsgs = test_msgs 103 | monitor.send_batch_notification() 104 | 105 | # Test that email was sent 106 | expected = { 107 | 'body': 'msg1\nmsg2', 108 | 'to': self.to_emails, 109 | 'from': 'testFrom@blah.com', 110 | 'subject': 'Test Alert', 111 | } 112 | self.assertEqual(1, monitor.send_email.call_count) 113 | monitor.send_email.assert_called_with(expected) 114 | 115 | # Test that email was logged 116 | self.assertEqual("""Sending notification email: 117 | To: %s 118 | From: testFrom@blah.com 119 | Subject: Test Alert 120 | Body: 121 | msg1 122 | msg2 123 | """ % (self.to_str), monitor.stderr.getvalue()) 124 | 125 | def test_log_email_with_body_digest(self): 126 | bodyLen = 80 127 | monitor = self._make_one_mock_send_email() 128 | email = { 129 | 'to': ['you@fubar.com'], 130 | 'from': 'me@fubar.com', 131 | 'subject': 'yo yo', 132 | 'body': 'a' * bodyLen, 133 | } 134 | monitor.log_email(email) 135 | self.assertEqual("""Sending notification email: 136 | To: you@fubar.com 137 | From: me@fubar.com 138 | Subject: yo yo 139 | Body: 140 | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa... 141 | """, monitor.stderr.getvalue()) 142 | self.assertEqual('a' * bodyLen, email['body']) 143 | 144 | def test_log_email_without_body_digest(self): 145 | monitor = self._make_one_mock_send_email() 146 | email = { 147 | 'to': ['you@fubar.com'], 148 | 'from': 'me@fubar.com', 149 | 'subject': 'yo yo', 150 | 'body': 'a' * 20, 151 | } 152 | monitor.log_email(email) 153 | self.assertEqual("""Sending notification email: 154 | To: you@fubar.com 155 | From: me@fubar.com 156 | Subject: yo yo 157 | Body: 158 | aaaaaaaaaaaaaaaaaaaa 159 | """, monitor.stderr.getvalue()) 160 | 161 | if __name__ == '__main__': 162 | unittest.main() -------------------------------------------------------------------------------- /superlance/tests/test_process_state_monitor.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | try: # pragma: no cover 3 | from unittest.mock import Mock 4 | except ImportError: # pragma: no cover 5 | from mock import Mock 6 | from superlance.compat import StringIO 7 | from superlance.process_state_monitor import ProcessStateMonitor 8 | 9 | class TestProcessStateMonitor(ProcessStateMonitor): 10 | 11 | process_state_events = ['PROCESS_STATE_EXITED'] 12 | 13 | def get_process_state_change_msg(self, headers, payload): 14 | return repr(payload) 15 | 16 | class ProcessStateMonitorTests(unittest.TestCase): 17 | 18 | def _get_target_class(self): 19 | return TestProcessStateMonitor 20 | 21 | def _make_one_mocked(self, **kwargs): 22 | kwargs['stdin'] = StringIO() 23 | kwargs['stdout'] = StringIO() 24 | kwargs['stderr'] = StringIO() 25 | 26 | obj = self._get_target_class()(**kwargs) 27 | obj.send_batch_notification = Mock() 28 | return obj 29 | 30 | def get_process_exited_event(self, pname, gname, expected, 31 | eventname='PROCESS_STATE_EXITED'): 32 | headers = { 33 | 'ver': '3.0', 'poolserial': '7', 'len': '71', 34 | 'server': 'supervisor', 'eventname': eventname, 35 | 'serial': '7', 'pool': 'checkmailbatch', 36 | } 37 | payload = 'processname:%s groupname:%s from_state:RUNNING expected:%d \ 38 | pid:58597' % (pname, gname, expected) 39 | return (headers, payload) 40 | 41 | def get_tick60_event(self): 42 | headers = { 43 | 'ver': '3.0', 'poolserial': '5', 'len': '15', 44 | 'server': 'supervisor', 'eventname': 'TICK_60', 45 | 'serial': '5', 'pool': 'checkmailbatch', 46 | } 47 | payload = 'when:1279665240' 48 | return (headers, payload) 49 | 50 | def test__get_tick_secs(self): 51 | monitor = self._make_one_mocked() 52 | self.assertEqual(5, monitor._get_tick_secs('TICK_5')) 53 | self.assertEqual(60, monitor._get_tick_secs('TICK_60')) 54 | self.assertEqual(3600, monitor._get_tick_secs('TICK_3600')) 55 | self.assertRaises(ValueError, monitor._get_tick_secs, 'JUNK_60') 56 | 57 | def test__get_tick_mins(self): 58 | monitor = self._make_one_mocked() 59 | self.assertEqual(5.0/60.0, monitor._get_tick_mins('TICK_5')) 60 | 61 | def test_handle_event_exit(self): 62 | monitor = self._make_one_mocked() 63 | hdrs, payload = self.get_process_exited_event('foo', 'bar', 0) 64 | monitor.handle_event(hdrs, payload) 65 | unexpected_err_msg = repr(payload) 66 | self.assertEqual([unexpected_err_msg], monitor.get_batch_msgs()) 67 | self.assertEqual('%s\n' % unexpected_err_msg, monitor.stderr.getvalue()) 68 | 69 | def test_handle_event_non_exit(self): 70 | monitor = self._make_one_mocked() 71 | hdrs, payload = self.get_process_exited_event('foo', 'bar', 0, 72 | eventname='PROCESS_STATE_FATAL') 73 | monitor.handle_event(hdrs, payload) 74 | self.assertEqual([], monitor.get_batch_msgs()) 75 | self.assertEqual('', monitor.stderr.getvalue()) 76 | 77 | def test_handle_event_tick_interval_expired(self): 78 | monitor = self._make_one_mocked() 79 | #Put msgs in batch 80 | hdrs, payload = self.get_process_exited_event('foo', 'bar', 0) 81 | monitor.handle_event(hdrs, payload) 82 | hdrs, payload = self.get_process_exited_event('bark', 'dog', 0) 83 | monitor.handle_event(hdrs, payload) 84 | self.assertEqual(2, len(monitor.get_batch_msgs())) 85 | #Time expired 86 | hdrs, payload = self.get_tick60_event() 87 | monitor.handle_event(hdrs, payload) 88 | 89 | # Test that batch messages are now gone 90 | self.assertEqual([], monitor.get_batch_msgs()) 91 | # Test that email was sent 92 | self.assertEqual(1, monitor.send_batch_notification.call_count) 93 | 94 | def test_handle_event_tick_interval_not_expired(self): 95 | monitor = self._make_one_mocked(interval=3) 96 | hdrs, payload = self.get_tick60_event() 97 | monitor.handle_event(hdrs, payload) 98 | self.assertEqual(1.0, monitor.get_batch_minutes()) 99 | monitor.handle_event(hdrs, payload) 100 | self.assertEqual(2.0, monitor.get_batch_minutes()) 101 | 102 | if __name__ == '__main__': 103 | unittest.main() -------------------------------------------------------------------------------- /superlance/timeoutconn.py: -------------------------------------------------------------------------------- 1 | from superlance.compat import httplib 2 | import socket 3 | import ssl 4 | 5 | 6 | class TimeoutHTTPConnection(httplib.HTTPConnection): 7 | """A customised HTTPConnection allowing a per-connection 8 | timeout, specified at construction.""" 9 | timeout = None 10 | 11 | def connect(self): 12 | """Override HTTPConnection.connect to connect to 13 | host/port specified in __init__.""" 14 | 15 | e = "getaddrinfo returns an empty list" 16 | for res in socket.getaddrinfo(self.host, self.port, 17 | 0, socket.SOCK_STREAM): 18 | af, socktype, proto, canonname, sa = res 19 | try: 20 | self.sock = socket.socket(af, socktype, proto) 21 | if self.timeout: # this is the new bit 22 | self.sock.settimeout(self.timeout) 23 | self.sock.connect(sa) 24 | except socket.error: 25 | if self.sock: 26 | self.sock.close() 27 | self.sock = None 28 | continue 29 | break 30 | if not self.sock: 31 | raise socket.error(e) 32 | 33 | 34 | class TimeoutHTTPSConnection(httplib.HTTPSConnection): 35 | timeout = None 36 | 37 | def connect(self): 38 | "Connect to a host on a given (SSL) port." 39 | 40 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 41 | if self.timeout: 42 | sock.settimeout(self.timeout) 43 | sock.connect((self.host, self.port)) 44 | self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) 45 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | docs,py27,py34,py35,py36,py37,py38,py39,py310 4 | 5 | [testenv] 6 | commands = 7 | python setup.py test -q 8 | 9 | [testenv:docs] 10 | deps = 11 | Sphinx 12 | readme 13 | setuptools >= 18.5 14 | allowlist_externals = make 15 | commands = 16 | make -C docs html BUILDDIR={envtmpdir} "SPHINXOPTS=-W -E" 17 | python setup.py check -m -r -s 18 | --------------------------------------------------------------------------------