├── .gitignore
├── LICENSE.txt
├── MANIFEST.in
├── README.rst
├── doc
    ├── Makefile
    └── source
    │   ├── conf.py
    │   ├── example.rst
    │   ├── gmtasks.rst
    │   ├── index.rst
    │   └── jsonclass.rst
├── gmtasks
    ├── __init__.py
    └── jsonclass.py
├── setup.cfg
└── setup.py


/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | *.egg-info
3 | dist
4 | doc/build
5 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
 1 | See:  http://en.wikipedia.org/wiki/Bsd_license#3-clause_license_.28.22New_BSD_License.22_or_.22Modified_BSD_License.22.29
 2 | 
 3 | Copyright (C) 2012 Chris Petersen
 4 | 
 5 | All rights reserved.
 6 | 
 7 | Redistribution and use in source and binary forms, with or without
 8 | modification, are permitted provided that the following conditions are met:
 9 |     * Redistributions of source code must retain the above copyright
10 |       notice, this list of conditions and the following disclaimer.
11 |     * Redistributions in binary form must reproduce the above copyright
12 |       notice, this list of conditions and the following disclaimer in the
13 |       documentation and/or other materials provided with the distribution.
14 |     * Neither the name of the <organization> nor the
15 |       names of its contributors may be used to endorse or promote products
16 |       derived from this software without specific prior written permission.
17 | 
18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 | DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
22 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 
29 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include *.txt *.rst
2 | 
3 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
 1 | ===================
 2 | Gearman Task Server
 3 | ===================
 4 | 
 5 | ``gmtasks`` contains a simple multiprocessing server for Gearman workers,
 6 | designed for ease of configuration and maximum availability.  It includes a
 7 | task wrapper class that traps any interrupts or exceptions that might be thrown
 8 | by your task methods, and attempts to make sure that the affected job is
 9 | re-inserted into the queue (which is not the default behavior if the worker
10 | script exits abnormally)
11 | 
12 | Please see the `full documentation <http://packages.python.org/gmtasks/>`_ for
13 | more detailed information.
14 | 
15 | Sample usage
16 | ~~~~~~~~~~~~
17 | 
18 | ::
19 | 
20 |     from multiprocessing   import freeze_support
21 |     from gmtasks.jsonclass import GearmanWorker
22 |     from gmtasks           import GearmanTaskServer, Task
23 | 
24 |     # Jobs
25 |     def job1(worker, job):
26 |         return job.data['string1']
27 |     def job2(worker, job):
28 |         return job.data['string2']
29 |     def job3(worker, job):
30 |         return job.data['string3']
31 | 
32 |     # Main loop
33 |     if __name__ == '__main__':
34 |         # Need this to run in Windows
35 |         freeze_support()
36 |         # Import all of the jobs we handle
37 |         tasks = [
38 |             Task('job1', job1),
39 |             {'task': 'job2', 'callback': job2},
40 |             ['job3', job3],
41 |             ]
42 |         # Initialize the server
43 |         server = GearmanTaskServer(
44 |             host_list      = ['localhost:4730'],
45 |             tasks          = tasks,
46 |             max_workers    = None, # Defaults to multiprocessing.cpu_count()
47 |             id_prefix      = 'myworker.',
48 |             worker_class   = GearmanWorker,
49 |             use_sighandler = True, # SIGINT and SIGTERM send KeyboardInterrupt
50 |             verbose        = True, # log.info() and log.error() messages
51 |             )
52 |         # Run the loop
53 |         server.serve_forever()
54 | 
55 | Download
56 | ~~~~~~~~
57 | 
58 | * https://github.com/ex-nerd/gmtasks
59 | * http://pypi.python.org/pypi/gmtasks/
60 | 


--------------------------------------------------------------------------------
/doc/Makefile:
--------------------------------------------------------------------------------
  1 | # Makefile for Sphinx documentation
  2 | #
  3 | 
  4 | # You can set these variables from the command line.
  5 | SPHINXOPTS    =
  6 | SPHINXBUILD   = sphinx-build
  7 | PAPER         =
  8 | BUILDDIR      = build
  9 | 
 10 | # Internal variables.
 11 | PAPEROPT_a4     = -D latex_paper_size=a4
 12 | PAPEROPT_letter = -D latex_paper_size=letter
 13 | ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 14 | # the i18n builder cannot share the environment and doctrees with the others
 15 | I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 16 | 
 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
 18 | 
 19 | help:
 20 | 	@echo "Please use \`make <target>' where <target> is one of"
 21 | 	@echo "  html       to make standalone HTML files"
 22 | 	@echo "  dirhtml    to make HTML files named index.html in directories"
 23 | 	@echo "  singlehtml to make a single large HTML file"
 24 | 	@echo "  pickle     to make pickle files"
 25 | 	@echo "  json       to make JSON files"
 26 | 	@echo "  htmlhelp   to make HTML files and a HTML help project"
 27 | 	@echo "  qthelp     to make HTML files and a qthelp project"
 28 | 	@echo "  devhelp    to make HTML files and a Devhelp project"
 29 | 	@echo "  epub       to make an epub"
 30 | 	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 31 | 	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
 32 | 	@echo "  text       to make text files"
 33 | 	@echo "  man        to make manual pages"
 34 | 	@echo "  texinfo    to make Texinfo files"
 35 | 	@echo "  info       to make Texinfo files and run them through makeinfo"
 36 | 	@echo "  gettext    to make PO message catalogs"
 37 | 	@echo "  changes    to make an overview of all changed/added/deprecated items"
 38 | 	@echo "  linkcheck  to check all external links for integrity"
 39 | 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 40 | 
 41 | clean:
 42 | 	-rm -rf $(BUILDDIR)/*
 43 | 
 44 | html:
 45 | 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 46 | 	@echo
 47 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 48 | 
 49 | dirhtml:
 50 | 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 51 | 	@echo
 52 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 53 | 
 54 | singlehtml:
 55 | 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 56 | 	@echo
 57 | 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 58 | 
 59 | pickle:
 60 | 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 61 | 	@echo
 62 | 	@echo "Build finished; now you can process the pickle files."
 63 | 
 64 | json:
 65 | 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 66 | 	@echo
 67 | 	@echo "Build finished; now you can process the JSON files."
 68 | 
 69 | htmlhelp:
 70 | 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 71 | 	@echo
 72 | 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 73 | 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 74 | 
 75 | qthelp:
 76 | 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 77 | 	@echo
 78 | 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
 79 | 	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
 80 | 	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/gmtasks.qhcp"
 81 | 	@echo "To view the help file:"
 82 | 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/gmtasks.qhc"
 83 | 
 84 | devhelp:
 85 | 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 86 | 	@echo
 87 | 	@echo "Build finished."
 88 | 	@echo "To view the help file:"
 89 | 	@echo "# mkdir -p $$HOME/.local/share/devhelp/gmtasks"
 90 | 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/gmtasks"
 91 | 	@echo "# devhelp"
 92 | 
 93 | epub:
 94 | 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
 95 | 	@echo
 96 | 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
 97 | 
 98 | latex:
 99 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
100 | 	@echo
101 | 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
102 | 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
103 | 	      "(use \`make latexpdf' here to do that automatically)."
104 | 
105 | latexpdf:
106 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
107 | 	@echo "Running LaTeX files through pdflatex..."
108 | 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
109 | 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
110 | 
111 | text:
112 | 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
113 | 	@echo
114 | 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
115 | 
116 | man:
117 | 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
118 | 	@echo
119 | 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
120 | 
121 | texinfo:
122 | 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
123 | 	@echo
124 | 	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
125 | 	@echo "Run \`make' in that directory to run these through makeinfo" \
126 | 	      "(use \`make info' here to do that automatically)."
127 | 
128 | info:
129 | 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
130 | 	@echo "Running Texinfo files through makeinfo..."
131 | 	make -C $(BUILDDIR)/texinfo info
132 | 	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
133 | 
134 | gettext:
135 | 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
136 | 	@echo
137 | 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
138 | 
139 | changes:
140 | 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
141 | 	@echo
142 | 	@echo "The overview file is in $(BUILDDIR)/changes."
143 | 
144 | linkcheck:
145 | 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
146 | 	@echo
147 | 	@echo "Link check complete; look for any errors in the above output " \
148 | 	      "or in $(BUILDDIR)/linkcheck/output.txt."
149 | 
150 | doctest:
151 | 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
152 | 	@echo "Testing of doctests in the sources finished, look at the " \
153 | 	      "results in $(BUILDDIR)/doctest/output.txt."
154 | 


--------------------------------------------------------------------------------
/doc/source/conf.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #
  3 | # gmtasks documentation build configuration file, created by
  4 | # sphinx-quickstart on Thu Jan 19 17:32:25 2012.
  5 | #
  6 | # This file is execfile()d with the current directory set to its containing dir.
  7 | #
  8 | # Note that not all possible configuration values are present in this
  9 | # autogenerated file.
 10 | #
 11 | # All configuration values have a default; values that are commented out
 12 | # serve to show the default.
 13 | 
 14 | import sys, os
 15 | 
 16 | # If extensions (or modules to document with autodoc) are in another directory,
 17 | # add these directories to sys.path here. If the directory is relative to the
 18 | # documentation root, use os.path.abspath to make it absolute, like shown here.
 19 | #sys.path.insert(0, os.path.abspath('.'))
 20 | 
 21 | # -- General configuration -----------------------------------------------------
 22 | 
 23 | # If your documentation needs a minimal Sphinx version, state it here.
 24 | #needs_sphinx = '1.0'
 25 | 
 26 | # Add any Sphinx extension module names here, as strings. They can be extensions
 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 28 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.viewcode']
 29 | 
 30 | # Add any paths that contain templates here, relative to this directory.
 31 | templates_path = ['_templates']
 32 | 
 33 | # The suffix of source filenames.
 34 | source_suffix = '.rst'
 35 | 
 36 | # The encoding of source files.
 37 | #source_encoding = 'utf-8-sig'
 38 | 
 39 | # The master toctree document.
 40 | master_doc = 'index'
 41 | 
 42 | # General information about the project.
 43 | project = u'gmtasks'
 44 | copyright = u'2012, Chris Petersen'
 45 | 
 46 | # The version info for the project you're documenting, acts as replacement for
 47 | # |version| and |release|, also used in various other places throughout the
 48 | # built documents.
 49 | #
 50 | # The short X.Y version.
 51 | version = '0.3'
 52 | # The full version, including alpha/beta/rc tags.
 53 | release = '0.3'
 54 | 
 55 | # The language for content autogenerated by Sphinx. Refer to documentation
 56 | # for a list of supported languages.
 57 | #language = None
 58 | 
 59 | # There are two options for replacing |today|: either, you set today to some
 60 | # non-false value, then it is used:
 61 | #today = ''
 62 | # Else, today_fmt is used as the format for a strftime call.
 63 | #today_fmt = '%B %d, %Y'
 64 | 
 65 | # List of patterns, relative to source directory, that match files and
 66 | # directories to ignore when looking for source files.
 67 | exclude_patterns = []
 68 | 
 69 | # The reST default role (used for this markup: `text`) to use for all documents.
 70 | #default_role = None
 71 | 
 72 | # If true, '()' will be appended to :func: etc. cross-reference text.
 73 | #add_function_parentheses = True
 74 | 
 75 | # If true, the current module name will be prepended to all description
 76 | # unit titles (such as .. function::).
 77 | #add_module_names = True
 78 | 
 79 | # If true, sectionauthor and moduleauthor directives will be shown in the
 80 | # output. They are ignored by default.
 81 | #show_authors = False
 82 | 
 83 | # The name of the Pygments (syntax highlighting) style to use.
 84 | pygments_style = 'sphinx'
 85 | 
 86 | # A list of ignored prefixes for module index sorting.
 87 | #modindex_common_prefix = []
 88 | 
 89 | 
 90 | # -- Options for HTML output ---------------------------------------------------
 91 | 
 92 | # The theme to use for HTML and HTML Help pages.  See the documentation for
 93 | # a list of builtin themes.
 94 | html_theme = 'default'
 95 | 
 96 | # Theme options are theme-specific and customize the look and feel of a theme
 97 | # further.  For a list of options available for each theme, see the
 98 | # documentation.
 99 | #html_theme_options = {}
100 | 
101 | # Add any paths that contain custom themes here, relative to this directory.
102 | #html_theme_path = []
103 | 
104 | # The name for this set of Sphinx documents.  If None, it defaults to
105 | # "<project> v<release> documentation".
106 | #html_title = None
107 | 
108 | # A shorter title for the navigation bar.  Default is the same as html_title.
109 | #html_short_title = None
110 | 
111 | # The name of an image file (relative to this directory) to place at the top
112 | # of the sidebar.
113 | #html_logo = None
114 | 
115 | # The name of an image file (within the static path) to use as favicon of the
116 | # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
117 | # pixels large.
118 | #html_favicon = None
119 | 
120 | # Add any paths that contain custom static files (such as style sheets) here,
121 | # relative to this directory. They are copied after the builtin static files,
122 | # so a file named "default.css" will overwrite the builtin "default.css".
123 | html_static_path = ['_static']
124 | 
125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
126 | # using the given strftime format.
127 | #html_last_updated_fmt = '%b %d, %Y'
128 | 
129 | # If true, SmartyPants will be used to convert quotes and dashes to
130 | # typographically correct entities.
131 | #html_use_smartypants = True
132 | 
133 | # Custom sidebar templates, maps document names to template names.
134 | #html_sidebars = {}
135 | 
136 | # Additional templates that should be rendered to pages, maps page names to
137 | # template names.
138 | #html_additional_pages = {}
139 | 
140 | # If false, no module index is generated.
141 | #html_domain_indices = True
142 | 
143 | # If false, no index is generated.
144 | #html_use_index = True
145 | 
146 | # If true, the index is split into individual pages for each letter.
147 | #html_split_index = False
148 | 
149 | # If true, links to the reST sources are added to the pages.
150 | #html_show_sourcelink = True
151 | 
152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
153 | #html_show_sphinx = True
154 | 
155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
156 | #html_show_copyright = True
157 | 
158 | # If true, an OpenSearch description file will be output, and all pages will
159 | # contain a <link> tag referring to it.  The value of this option must be the
160 | # base URL from which the finished HTML is served.
161 | #html_use_opensearch = ''
162 | 
163 | # This is the file name suffix for HTML files (e.g. ".xhtml").
164 | #html_file_suffix = None
165 | 
166 | # Output file base name for HTML help builder.
167 | htmlhelp_basename = 'gmtasksdoc'
168 | 
169 | 
170 | # -- Options for LaTeX output --------------------------------------------------
171 | 
172 | latex_elements = {
173 | # The paper size ('letterpaper' or 'a4paper').
174 | #'papersize': 'letterpaper',
175 | 
176 | # The font size ('10pt', '11pt' or '12pt').
177 | #'pointsize': '10pt',
178 | 
179 | # Additional stuff for the LaTeX preamble.
180 | #'preamble': '',
181 | }
182 | 
183 | # Grouping the document tree into LaTeX files. List of tuples
184 | # (source start file, target name, title, author, documentclass [howto/manual]).
185 | latex_documents = [
186 |   ('index', 'gmtasks.tex', u'gmtasks Documentation',
187 |    u'Chris Petersen', 'manual'),
188 | ]
189 | 
190 | # The name of an image file (relative to this directory) to place at the top of
191 | # the title page.
192 | #latex_logo = None
193 | 
194 | # For "manual" documents, if this is true, then toplevel headings are parts,
195 | # not chapters.
196 | #latex_use_parts = False
197 | 
198 | # If true, show page references after internal links.
199 | #latex_show_pagerefs = False
200 | 
201 | # If true, show URL addresses after external links.
202 | #latex_show_urls = False
203 | 
204 | # Documents to append as an appendix to all manuals.
205 | #latex_appendices = []
206 | 
207 | # If false, no module index is generated.
208 | #latex_domain_indices = True
209 | 
210 | 
211 | # -- Options for manual page output --------------------------------------------
212 | 
213 | # One entry per manual page. List of tuples
214 | # (source start file, name, description, authors, manual section).
215 | man_pages = [
216 |     ('index', 'gmtasks', u'gmtasks Documentation',
217 |      [u'Chris Petersen'], 1)
218 | ]
219 | 
220 | # If true, show URL addresses after external links.
221 | #man_show_urls = False
222 | 
223 | 
224 | # -- Options for Texinfo output ------------------------------------------------
225 | 
226 | # Grouping the document tree into Texinfo files. List of tuples
227 | # (source start file, target name, title, author,
228 | #  dir menu entry, description, category)
229 | texinfo_documents = [
230 |   ('index', 'gmtasks', u'gmtasks Documentation',
231 |    u'Chris Petersen', 'gmtasks', 'One line description of project.',
232 |    'Miscellaneous'),
233 | ]
234 | 
235 | # Documents to append as an appendix to all manuals.
236 | #texinfo_appendices = []
237 | 
238 | # If false, no module index is generated.
239 | #texinfo_domain_indices = True
240 | 
241 | # How to display URL addresses: 'footnote', 'no', or 'inline'.
242 | #texinfo_show_urls = 'footnote'
243 | 
244 | 
245 | # Example configuration for intersphinx: refer to the Python standard library.
246 | intersphinx_mapping = {
247 |     'python': ('http://docs.python.org/', None),
248 |     'gearman': ('http://packages.python.org/gearman', None),
249 | }
250 | 


--------------------------------------------------------------------------------
/doc/source/example.rst:
--------------------------------------------------------------------------------
 1 | ==============
 2 | Example server
 3 | ==============
 4 | 
 5 | Sometimes it's easier to just learn by looking at some sample code, so here
 6 | is a simple server that demonstrates most of how to use gmtasks.
 7 | 
 8 | ::
 9 | 
10 |     from multiprocessing   import freeze_support
11 |     from gmtasks.jsonclass import GearmanWorker
12 |     from gmtasks           import GearmanTaskServer, Task
13 | 
14 |     # Jobs
15 |     def job1(worker, job):
16 |         return job.data['string1']
17 |     def job2(worker, job):
18 |         return job.data['string2']
19 |     def job3(worker, job):
20 |         return job.data['string3']
21 | 
22 |     # Main loop
23 |     if __name__ == '__main__':
24 |         # Need this to run in Windows
25 |         freeze_support()
26 |         # Import all of the jobs we handle
27 |         tasks = [
28 |             # You can use a Task()
29 |             Task('job1', job1),
30 |             # Or a dict
31 |             {'task': 'job2', 'callback': job2},
32 |             # Or a list/tuple
33 |             ['job3', job3],
34 |             ]
35 |         # Initialize the server
36 |         server = GearmanTaskServer(
37 |             host_list      = ['localhost:4730'],
38 |             tasks          = tasks,
39 |             max_workers    = None, # Defaults to multiprocessing.cpu_count()
40 |             id_prefix      = 'myworker.',
41 |             worker_class   = GearmanWorker,
42 |             use_sighandler = True, # SIGINT and SIGTERM send KeyboardInterrupt
43 |             verbose        = True, # log.info() and log.error() messages
44 |             )
45 |         # Run the loop
46 |         server.serve_forever()
47 | 


--------------------------------------------------------------------------------
/doc/source/gmtasks.rst:
--------------------------------------------------------------------------------
 1 | GearmanTaskServer
 2 | =================
 3 | 
 4 | .. automodule:: gmtasks
 5 | 
 6 | .. autoclass:: gmtasks.Task
 7 | 
 8 | .. autoclass:: gmtasks.GearmanTaskServer
 9 | 
10 | .. automethod:: gmtasks.GearmanTaskServer.serve_forever
11 | 


--------------------------------------------------------------------------------
/doc/source/index.rst:
--------------------------------------------------------------------------------
 1 | .. gmtasks documentation master file, created by
 2 |    sphinx-quickstart on Thu Jan 19 15:53:57 2012.
 3 | 
 4 | Welcome to gmtasks's documentation!
 5 | ===================================
 6 | 
 7 | ``gmtasks`` contains a simple multiprocessing server for Gearman workers,
 8 | designed for ease of configuration and maximum availability.  It includes a
 9 | task wrapper class that traps any interrupts or exceptions that might be thrown
10 | by your task methods, and attempts to make sure that the affected job is
11 | re-inserted into the queue (which is not the default behavior if the worker
12 | script exits abnormally)
13 | 
14 | Contents:
15 | 
16 | .. toctree::
17 |    :maxdepth: 2
18 | 
19 |    gmtasks
20 |    jsonclass
21 |    example
22 | 
23 | 
24 | Indices and tables
25 | ==================
26 | 
27 | * :ref:`genindex`
28 | * :ref:`modindex`
29 | * :ref:`search`
30 | 
31 | Download
32 | ========
33 | 
34 | * https://github.com/ex-nerd/gmtasks
35 | * http://pypi.python.org/pypi/gmtasks/
36 | 


--------------------------------------------------------------------------------
/doc/source/jsonclass.rst:
--------------------------------------------------------------------------------
1 | JSON Classes
2 | ============
3 | 
4 | .. automodule:: gmtasks.jsonclass
5 | 
6 | .. autoclass:: gmtasks.jsonclass.GearmanWorker
7 | 
8 | .. autoclass:: gmtasks.jsonclass.GearmanClient
9 | 


--------------------------------------------------------------------------------
/gmtasks/__init__.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Simple multiprocessing server for gearman.
  3 | """
  4 | 
  5 | __version__ = '0.3'
  6 | 
  7 | import time
  8 | import sys, traceback
  9 | import gearman
 10 | import signal
 11 | 
 12 | from multiprocessing import Process, Queue, cpu_count, active_children
 13 | from Queue import Empty
 14 | 
 15 | 
 16 | # Logging
 17 | import logging
 18 | log = logging.getLogger(__name__)
 19 | 
 20 | #
 21 | # A simple task function wrapper designed to exit gracefully and re-queue jobs
 22 | # when interrupted.
 23 | #
 24 | 
 25 | class Task(object):
 26 |     """
 27 |     This class is a simple wrapper around worker functions that does its best to
 28 |     return a job to the queue if the function receives a KeyboardInterrupt or
 29 |     raises an exception.
 30 | 
 31 |     **task**
 32 |         The gearman task name.
 33 |     **callback**
 34 |         Worker callback function.
 35 |     **verbose**
 36 |         If true, log.error() with exception details.
 37 |     """
 38 |     def __init__(self, task, callback, verbose=False):
 39 |         self.task     = task
 40 |         self.callback = callback
 41 |         self.verbose  = verbose
 42 |     def __call__(self, worker, job):
 43 |         try:
 44 |             return self.callback(worker, job)
 45 |         except Exception, e:
 46 |             if self.verbose:
 47 |                 log.error('WORKER FAILED:  {0}, {1}\n{2}'.format(
 48 |                     self.task,
 49 |                     str(e),
 50 |                     traceback.format_exc()
 51 |                     ))
 52 |             # Disconnect so this job goes back into the queue
 53 |             worker.shutdown()
 54 |             # Continue with the exception
 55 |             raise
 56 | 
 57 | #
 58 | # Main server class
 59 | #
 60 | 
 61 | class GearmanTaskServer(object):
 62 |     """
 63 |     The main task server class.
 64 | 
 65 |     **host_list**
 66 |         List of gearman hosts to connect to.  See :py:mod:`gearman.worker` for
 67 |         more documentation.
 68 |     **tasks**
 69 |         List of tasks.  Tasks may be Task() objects, dicts, lists or tuples.
 70 |     **max_workers**
 71 |         Number of worker processes to launch.  Defaults to
 72 |         :py:func:`~multiprocessing.cpu_count()`
 73 |     **id_prefix**
 74 |         If you want your workers to register a client_id with gearman, provide
 75 |         a prefix here.  GearmanTaskServer will append an incrementing number
 76 |         to the end of this, representing the total number of subprocesses
 77 |         started in this run.
 78 |     **worker_class**
 79 |         GearmanWorker class to use.  Defaults to :py:class:`gearman.worker.GearmanWorker`.
 80 |         You could also use :py:class:`jsonclass.GearmanWorker`.
 81 |     **use_sighandler**
 82 |         Set to False if you would prefer to use your own signal handlers
 83 |         instead of trapping SIGINT and SIGTERM as KeyboardInterrupt events.
 84 |     **verbose**
 85 |         Set to True to enable logger.
 86 |     """
 87 | 
 88 |     def __init__(self,
 89 |             host_list, tasks, max_workers=None,
 90 |             id_prefix = None, worker_class=None, use_sighandler=True, verbose=False
 91 |             ):
 92 |         self.host_list   = host_list
 93 |         self.tasks       = tasks
 94 | 
 95 |         if max_workers:
 96 |             self.max_workers = int(max_workers)
 97 |         else:
 98 |             try:
 99 |                 self.max_workers = int(cpu_count())
100 |             except:
101 |                 self.max_workers = 1
102 | 
103 |         self.worker      = worker_class
104 |         self.id_prefix   = id_prefix
105 |         self.verbose     = verbose
106 |         if not self.worker:
107 |             self.worker = gearman.GearmanWorker
108 |         # Signal Handler override?
109 |         if use_sighandler:
110 |             self._setup_sighandler()
111 | 
112 | 
113 |     def serve_forever(self):
114 |         """
115 |         Launch the multi-process server and process jobs until an interrupt
116 |         is received.
117 |         """
118 |         # Initialize a queue designed to track child processes that exit.
119 |         doneq = Queue()
120 |         # Keep track of how many clients we have created, so they can have unique IDs
121 |         process_counter = 0
122 |         # Loop
123 |         workers = []
124 |         try:
125 |             while True:
126 |                 while len(workers) < self.max_workers:
127 |                     process_counter += 1
128 |                     client_id = None
129 |                     if self.id_prefix:
130 |                         client_id = '{0}{1}'.format(self.id_prefix, process_counter)
131 |                     p = Process(target=_worker_process, args=(
132 |                             self.tasks,
133 |                             doneq,
134 |                             self.host_list,
135 |                             self.worker,
136 |                             client_id,
137 |                             self.verbose
138 |                             ))
139 |                     p.start()
140 |                     workers.append(p)
141 |                     if self.verbose:
142 |                         log.info("Num workers:  {0} of {1}".format(len(workers), self.max_workers))
143 |                 # Use the queue as a poor man's wait/select against the first
144 |                 # child process to finish.  Add a timeout so we can repopulate
145 |                 # processes that terminate abnormally.
146 |                 try:
147 |                     r = doneq.get(True, 5)
148 |                 except Empty:
149 |                     r = None
150 |                 if r is not None:
151 |                     if isinstance(r, gearman.errors.ServerUnavailable):
152 |                         #: @todo non-blocking doneq.get() to clear it out of
153 |                         #        similar errors? or maybe just reset doneq above
154 |                         #        when len(workers)==0
155 |                         if self.verbose:
156 |                             log.info("Reconnecting.")
157 |                         time.sleep(2)
158 |                     elif r is True:
159 |                         # Job exited normally.  Except this shouldn't actually happen.
160 |                         log.info('Normal process exit (May actually be a problem)')
161 |                 # Give things a fraction of a second to catch up, then filter out
162 |                 # the finished process(es)
163 |                 time.sleep(0.1)
164 |                 workers = filter(lambda w: w in workers, active_children())
165 |         except KeyboardInterrupt:
166 |             log.error('EXIT.  RECEIVED INTERRUPT')
167 | 
168 |     def _setup_sighandler(self):
169 |         """
170 |         Initialize our slight change to the signal handler setup
171 | 
172 |         Provided as a separate function so that users can opt in to this feature,
173 |         or write their own.
174 |         """
175 |         signal.signal(signal.SIGINT,  _interrupt_handler)
176 |         signal.signal(signal.SIGTERM, _interrupt_handler)
177 | 
178 | #
179 | # Our own interrupt handler
180 | #
181 | 
182 | 
183 | def _interrupt_handler(signum, frame):
184 |     """
185 |     Python maps SIGINT to KeyboardInterrupt by default, but we need to
186 |     catch SIGTERM as well, so we can give jobs as much of a chance as
187 |     possible to alert the gearman server to requeue the job.
188 |     """
189 |     raise KeyboardInterrupt()
190 | 
191 | #
192 | # Worker process/function
193 | #
194 | 
195 | def _worker_process(tasks, doneq, host_list, worker_class=None, client_id=None, verbose=False):
196 |     try:
197 |         if not worker_class:
198 |             worker_class = gearman.GearmanWorker
199 |         try:
200 |             # Connect
201 |             gm_worker = worker_class(host_list = host_list)
202 |             if client_id:
203 |                 gm_worker.set_client_id(client_id)
204 |             # Load the tasks
205 |             for task in tasks:
206 |                 taskname = callback = None
207 |                 if isinstance(task, dict):
208 |                     taskname = task['task']
209 |                     callback = task['callback']
210 |                 elif isinstance(task, list) or isinstance(task, tuple):
211 |                     taskname, callback = task
212 |                 else:
213 |                     taskname = task.task
214 |                     callback = task
215 |                 if verbose:
216 |                     log.info("Registering {0} task {1}".format(client_id, taskname))
217 |                 gm_worker.register_task(taskname, callback)
218 |             # Enter the work loop
219 |             gm_worker.work()
220 |         except gearman.errors.ServerUnavailable, e:
221 |             doneq.put(e)
222 |             return
223 |         # Really should never touch this code but it's here just in case.
224 |         doneq.put(True)
225 |     except KeyboardInterrupt:
226 |         # Prevent this child process from printing a traceback
227 |         pass
228 | 


--------------------------------------------------------------------------------
/gmtasks/jsonclass.py:
--------------------------------------------------------------------------------
 1 | """
 2 | GearmanWorker and GearmanClient classes that send/receive data in JSON format.
 3 | 
 4 | Please note that the default python json encoder has been extended to also
 5 | encode Decimal data.
 6 | """
 7 | 
 8 | import json
 9 | import decimal
10 | import gearman
11 | 
12 | class _JSONEncoder(json.JSONEncoder):
13 |     """
14 |     An override JSON encoder class that recognizes Decimal objects.
15 |     """
16 |     def default(self, obj):
17 |         if isinstance(obj, decimal.Decimal):
18 |             return str(obj)
19 |         return super(_JSONEncoder, self).default(obj)
20 | 
21 | class _JSONDataEncoder(gearman.DataEncoder):
22 |     """
23 |     An override Gearman DataEncoder class to send/receive JSON data.
24 |     """
25 |     @classmethod
26 |     def encode(cls, encodable_object):
27 |         return json.dumps(encodable_object, cls=_JSONEncoder)
28 |     @classmethod
29 |     def decode(cls, decodable_string):
30 |         return json.loads(decodable_string)
31 | 
32 | class GearmanWorker(gearman.GearmanWorker):
33 |     """
34 |     Extend gearman.GearmanWorker to receive job data in JSON format.
35 |     """
36 |     data_encoder = _JSONDataEncoder
37 |     def after_poll(self, any_activity):
38 |         return True
39 | 
40 | class GearmanClient(gearman.GearmanClient):
41 |     """
42 |     Extend gearman.GearmanClient to send job data in JSON format.
43 |     """
44 |     data_encoder = _JSONDataEncoder
45 | 


--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [build_sphinx]
2 | source-dir = doc/source
3 | build-dir  = doc/build
4 | all_files  = 1
5 | 
6 | [upload_sphinx]
7 | upload-dir = doc/build/html
8 | 
9 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | from setuptools import setup, find_packages
 2 | import os, sys, re
 3 | 
 4 | 
 5 | # Load the version by reading prep.py, so we don't run into
 6 | # dependency loops by importing it into setup.py
 7 | version = None
 8 | with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), "gmtasks", "__init__.py")) as file:
 9 |     for line in file:
10 |         m = re.search(r'__version__\s*=\s*(.+?\n)', line)
11 |         if m:
12 |             version = eval(m.group(1))
13 |             break
14 | 
15 | setup_args = dict(
16 |     name             = 'gmtasks',
17 |     version          = version,
18 |     author           = 'Chris Petersen',
19 |     author_email     = 'geek@ex-nerd.com',
20 |     url              = 'https://github.com/ex-nerd/gmtasks',
21 |     license          = 'Modified BSD',
22 |     description      = 'Gearman Task Server',
23 |     long_description = open('README.rst').read(),
24 |     install_requires = ['gearman'],
25 |     packages         = find_packages(),
26 |     classifiers=[
27 |         "Development Status :: 3 - Alpha",
28 |         "Environment :: Web Environment",
29 |         "Intended Audience :: Developers",
30 |         "License :: OSI Approved :: BSD License",
31 |         "Operating System :: OS Independent",
32 |         "Programming Language :: Python",
33 |         "Topic :: Utilities",
34 |     ],
35 | )
36 | 
37 | if __name__ == '__main__':
38 |     setup(**setup_args)
39 | 


--------------------------------------------------------------------------------