├── .gitignore ├── LICENSE ├── Makefile ├── README.rst ├── docs ├── Makefile ├── code.rst ├── conf.py ├── index.rst ├── make.bat └── quick-start.en.rst ├── requires.txt ├── setup.py ├── tarantool_queue ├── __init__.py ├── tarantool_queue.py └── tarantool_tqueue.py └── tests ├── __init__.py ├── tarantool.cfg └── test_queue.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | __pycache__ 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 bigbes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | html: 2 | python setup.py build_sphinx 3 | test: 4 | python setup.py test 5 | upload: 6 | python setup.py register sdist --format=gztar,zip upload 7 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | queue-python 3 | ============ 4 | 5 | Python Bindings for `Tarantool Queue `_. 6 | 7 | Library depends on: 8 | 9 | * msgpack-python 10 | * tarantool 11 | 12 | Basic usage can be found in tests. Description on every command is in source code. 13 | 14 | Big thanks to Dmitriy Shveenkov and `Alexandr (FZambia) Emelin `_. 15 | 16 | For install of latest "stable" version type: 17 | 18 | .. code-block:: bash 19 | 20 | # using pip 21 | $ sudo pip install tarantool-queue 22 | # or using easy_install 23 | $ sudo easy_install tarantool-queue 24 | # or using python 25 | $ wget http://bit.ly/tarantool_queue -O tarantool_queue.tar.gz 26 | $ tar xzf tarantool_queue.tar.gz 27 | $ cd tarantool-queue-{version} 28 | $ sudo python setup.py install 29 | 30 | For install bleeding edge type: 31 | 32 | .. code-block:: bash 33 | 34 | $ sudo pip install git+https://github.com/tarantool/queue-python.git 35 | 36 | For configuring Queue in `Tarantool `_ read manual `Here `_. 37 | 38 | Then just **import** it, create **Queue**, create **Tube**, **put** and **take** some elements: 39 | 40 | .. code-block:: python 41 | 42 | >>> from tarantool_queue import Queue 43 | >>> queue = Queue("localhost", 33013, 0) 44 | >>> tube = queue.tube("name_of_tube") 45 | >>> tube.put([1, 2, 3]) 46 | Not taken task instance 47 | >>> task = tube.take() 48 | >>> task.data # take task and read data from it 49 | [1, 2, 3] 50 | >>> task.ack() # move this task into state DONE 51 | True 52 | 53 | That's all, folks! 54 | 55 | See Also 56 | ======== 57 | * `Documentation `_ 58 | * `Quick Start `_ 59 | * `Queue API `_ 60 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/tarantool-queue.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/tarantool-queue.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/tarantool-queue" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/tarantool-queue" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/code.rst: -------------------------------------------------------------------------------- 1 | .. _codeapi: 2 | 3 | Queue API 4 | ********* 5 | 6 | Basic Definitions: 7 | 8 | * ttl - Time to Live of task. 9 | * ttr - Time to Release of task. 10 | * pri - Priority of task. 11 | * delay - Delay for task to be added to queue. 12 | 13 | .. automodule:: tarantool_queue 14 | 15 | .. warning:: 16 | 17 | Don't use constructor of Task and Tube. 18 | Task's are created by Tube and Queue methods. 19 | For creating Tube object use :meth:`Queue.tube(name) ` 20 | 21 | .. autoclass:: Queue 22 | :members: 23 | 24 | .. autoclass:: Tube 25 | :members: 26 | 27 | .. autoclass:: Task 28 | :members: 29 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # tarantool-queue documentation build configuration file, created by 4 | # sphinx-quickstart on Wed Oct 30 15:14:56 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.insert(0, os.path.abspath('.')) 20 | sys.path.insert(0, os.path.abspath('..')) 21 | 22 | from tarantool_queue import __version__ 23 | 24 | 25 | # -- General configuration ----------------------------------------------------- 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | #needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be extensions 31 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 32 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.viewcode'] 33 | 34 | # Add any paths that contain templates here, relative to this directory. 35 | templates_path = ['_templates'] 36 | 37 | # The suffix of source filenames. 38 | source_suffix = '.rst' 39 | 40 | # The encoding of source files. 41 | #source_encoding = 'utf-8-sig' 42 | 43 | # The master toctree document. 44 | master_doc = 'index' 45 | 46 | # General information about the project. 47 | project = u'tarantool-queue' 48 | copyright = u'2013, bigbes' 49 | 50 | # The version info for the project you're documenting, acts as replacement for 51 | # |version| and |release|, also used in various other places throughout the 52 | # built documents. 53 | # 54 | # The short X.Y version. 55 | version = __version__ 56 | # The full version, including alpha/beta/rc tags. 57 | release = __version__ 58 | 59 | # The language for content autogenerated by Sphinx. Refer to documentation 60 | # for a list of supported languages. 61 | #language = None 62 | 63 | # There are two options for replacing |today|: either, you set today to some 64 | # non-false value, then it is used: 65 | #today = '' 66 | # Else, today_fmt is used as the format for a strftime call. 67 | #today_fmt = '%B %d, %Y' 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | exclude_patterns = ['_build'] 72 | 73 | # The reST default role (used for this markup: `text`) to use for all documents. 74 | #default_role = None 75 | 76 | # If true, '()' will be appended to :func: etc. cross-reference text. 77 | #add_function_parentheses = True 78 | 79 | # If true, the current module name will be prepended to all description 80 | # unit titles (such as .. function::). 81 | #add_module_names = True 82 | 83 | # If true, sectionauthor and moduleauthor directives will be shown in the 84 | # output. They are ignored by default. 85 | #show_authors = False 86 | 87 | # The name of the Pygments (syntax highlighting) style to use. 88 | pygments_style = 'sphinx' 89 | 90 | # A list of ignored prefixes for module index sorting. 91 | #modindex_common_prefix = [] 92 | 93 | # If true, keep warnings as "system message" paragraphs in the built documents. 94 | #keep_warnings = False 95 | 96 | 97 | # -- Options for HTML output --------------------------------------------------- 98 | 99 | # The theme to use for HTML and HTML Help pages. See the documentation for 100 | # a list of builtin themes. 101 | html_theme = 'default' 102 | 103 | # Theme options are theme-specific and customize the look and feel of a theme 104 | # further. For a list of options available for each theme, see the 105 | # documentation. 106 | #html_theme_options = {} 107 | 108 | # Add any paths that contain custom themes here, relative to this directory. 109 | #html_theme_path = [] 110 | 111 | # The name for this set of Sphinx documents. If None, it defaults to 112 | # " v documentation". 113 | #html_title = None 114 | 115 | # A shorter title for the navigation bar. Default is the same as html_title. 116 | #html_short_title = None 117 | 118 | # The name of an image file (relative to this directory) to place at the top 119 | # of the sidebar. 120 | #html_logo = None 121 | 122 | # The name of an image file (within the static path) to use as favicon of the 123 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 124 | # pixels large. 125 | #html_favicon = None 126 | 127 | # Add any paths that contain custom static files (such as style sheets) here, 128 | # relative to this directory. They are copied after the builtin static files, 129 | # so a file named "default.css" will overwrite the builtin "default.css". 130 | html_static_path = ['_static'] 131 | 132 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 133 | # using the given strftime format. 134 | #html_last_updated_fmt = '%b %d, %Y' 135 | 136 | # If true, SmartyPants will be used to convert quotes and dashes to 137 | # typographically correct entities. 138 | #html_use_smartypants = True 139 | 140 | # Custom sidebar templates, maps document names to template names. 141 | #html_sidebars = {} 142 | 143 | # Additional templates that should be rendered to pages, maps page names to 144 | # template names. 145 | #html_additional_pages = {} 146 | 147 | # If false, no module index is generated. 148 | #html_domain_indices = True 149 | 150 | # If false, no index is generated. 151 | #html_use_index = True 152 | 153 | # If true, the index is split into individual pages for each letter. 154 | #html_split_index = False 155 | 156 | # If true, links to the reST sources are added to the pages. 157 | #html_show_sourcelink = True 158 | 159 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 160 | #html_show_sphinx = True 161 | 162 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 163 | #html_show_copyright = True 164 | 165 | # If true, an OpenSearch description file will be output, and all pages will 166 | # contain a tag referring to it. The value of this option must be the 167 | # base URL from which the finished HTML is served. 168 | #html_use_opensearch = '' 169 | 170 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 171 | #html_file_suffix = None 172 | 173 | # Output file base name for HTML help builder. 174 | htmlhelp_basename = 'tarantool-queuedoc' 175 | 176 | 177 | # -- Options for LaTeX output -------------------------------------------------- 178 | 179 | latex_elements = { 180 | # The paper size ('letterpaper' or 'a4paper'). 181 | #'papersize': 'letterpaper', 182 | 183 | # The font size ('10pt', '11pt' or '12pt'). 184 | #'pointsize': '10pt', 185 | 186 | # Additional stuff for the LaTeX preamble. 187 | #'preamble': '', 188 | } 189 | 190 | # Grouping the document tree into LaTeX files. List of tuples 191 | # (source start file, target name, title, author, documentclass [howto/manual]). 192 | latex_documents = [ 193 | ('index', 'tarantool-queue.tex', u'tarantool-queue Documentation', 194 | u'bigbes', 'manual'), 195 | ] 196 | 197 | # The name of an image file (relative to this directory) to place at the top of 198 | # the title page. 199 | #latex_logo = None 200 | 201 | # For "manual" documents, if this is true, then toplevel headings are parts, 202 | # not chapters. 203 | #latex_use_parts = False 204 | 205 | # If true, show page references after internal links. 206 | #latex_show_pagerefs = False 207 | 208 | # If true, show URL addresses after external links. 209 | #latex_show_urls = False 210 | 211 | # Documents to append as an appendix to all manuals. 212 | #latex_appendices = [] 213 | 214 | # If false, no module index is generated. 215 | #latex_domain_indices = True 216 | 217 | 218 | # -- Options for manual page output -------------------------------------------- 219 | 220 | # One entry per manual page. List of tuples 221 | # (source start file, name, description, authors, manual section). 222 | man_pages = [ 223 | ('index', 'tarantool-queue', u'tarantool-queue Documentation', 224 | [u'bigbes'], 1) 225 | ] 226 | 227 | # If true, show URL addresses after external links. 228 | #man_show_urls = False 229 | 230 | 231 | # -- Options for Texinfo output ------------------------------------------------ 232 | 233 | # Grouping the document tree into Texinfo files. List of tuples 234 | # (source start file, target name, title, author, 235 | # dir menu entry, description, category) 236 | texinfo_documents = [ 237 | ('index', 'tarantool-queue', u'tarantool-queue Documentation', 238 | u'bigbes', 'tarantool-queue', 'One line description of project.', 239 | 'Miscellaneous'), 240 | ] 241 | 242 | # Documents to append as an appendix to all manuals. 243 | #texinfo_appendices = [] 244 | 245 | # If false, no module index is generated. 246 | #texinfo_domain_indices = True 247 | 248 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 249 | #texinfo_show_urls = 'footnote' 250 | 251 | # If true, do not generate a @detailmenu in the "Top" node's menu. 252 | #texinfo_no_detailmenu = False 253 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | tarantool-queue-python 3 | ====================== 4 | 5 | Python Bindings for `Tarantool Queue `_. 6 | 7 | Library depends on: 8 | 9 | * msgpack-python 10 | * tarantool 11 | 12 | Basic usage can be found in tests. Description on every command is in source code. 13 | 14 | Big thanks to Dmitriy Shveenkov and `Alexandr (FZambia) Emelin `_. 15 | 16 | For install of latest "stable" version type: 17 | 18 | .. code-block:: bash 19 | 20 | # using pip 21 | $ sudo pip install tarantool-queue 22 | # or using easy_install 23 | $ sudo easy_install tarantool-queue 24 | # or using python 25 | $ wget http://bit.ly/tarantool_queue -O tarantool_queue.tar.gz 26 | $ tar xzf tarantool_queue.tar.gz 27 | $ cd tarantool-queue-{version} 28 | $ sudo python setup.py install 29 | 30 | For install bleeding edge type: 31 | 32 | .. code-block:: bash 33 | 34 | $ sudo pip install git+https://github.com/tarantool/tarantool-queue-python.git 35 | 36 | For configuring Queue in `Tarantool `_ read manual `Here `_ or read :ref:`prepare_server`. 37 | 38 | Then just **import** it, create **Queue**, create **Tube**, **put** and **take** some elements: 39 | 40 | .. code-block:: python 41 | 42 | >>> from tarantool_queue import Queue 43 | >>> queue = Queue("localhost", 33013, 0) 44 | >>> tube = queue.tube("name_of_tube") 45 | >>> tube.put([1, 2, 3]) 46 | Not taken task instance 47 | >>> task = tube.take() 48 | >>> task.data # take task and read data from it 49 | [1, 2, 3] 50 | >>> task.ack() # move this task into state DONE 51 | True 52 | 53 | That's all, folks! 54 | 55 | See Also 56 | ======== 57 | * `Documentation `_ 58 | * `Repository `_ 59 | 60 | .. toctree:: 61 | :maxdepth: 1 62 | 63 | code 64 | quick-start.en 65 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\tarantool-queue.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\tarantool-queue.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/quick-start.en.rst: -------------------------------------------------------------------------------- 1 | .. _quick_start_en: 2 | 3 | =========== 4 | Quick Start 5 | =========== 6 | 7 | .. _prepare_server: 8 | 9 | -------------- 10 | Prepare server 11 | -------------- 12 | 13 | 1. Install tarantool on your favourite OS. For more information, please refer to `User Guide `_ (Section: Downloading and installing a binary package). 14 | 2. Download tarantool.cfg and init.lua from `Queue `_ repo 15 | 16 | .. code-block:: bash 17 | 18 | $ wget https://raw.github.com/tarantool/queue/master/tarantool.cfg # Download configuration file for tarantool 19 | $ wget https://github.com/tarantool/queue/blob/master/init.lua # Download queue script 20 | $ tarantool_box --init-storage # Init tarantool storage files 21 | $ tarantool_box # Start tarantool 22 | 23 | Done! 24 | 25 | ------------------------------ 26 | Install tarantool-queue-python 27 | ------------------------------ 28 | 29 | .. code-block:: bash 30 | 31 | # using pip 32 | $ sudo pip install tarantool-queue 33 | # or using easy_install 34 | $ sudo easy_install tarantool-queue 35 | # or using python 36 | $ wget http://bit.ly/tarantool_queue -O tarantool_queue.tar.gz 37 | $ tar xzf tarantool_queue.tar.gz 38 | $ cd tarantool-queue-{version} 39 | $ sudo python setup.py install 40 | 41 | ----------------------------------------- 42 | Connecting to server and basic operations 43 | ----------------------------------------- 44 | 45 | In the beggining you must **import tarantool-queue** and create **Queue** object: 46 | 47 | .. code-block:: python 48 | 49 | from tarantool_queue import Queue 50 | queue = Queue('localhost', 33020, 0) 51 | 52 | Queue object is an aggregator for Tubes: Tube is the queue. When you put task into queue, you associate name of tube with it. When you take task, you take if from some tube. 53 | For the beggining you must create Tube object with **Queue.tube(name)** method. 54 | Basic operations for Tubes are: **Tube.put(task)** and **Tube.get()** 55 | When you have done all you want with this task you must make **Task.ack()** it or **Task.release()** it back to the queue. 56 | 57 | .. code-block:: python 58 | 59 | # On producer: 60 | appetizers = queue.tube('appt-s') 61 | appetizers.put('Egg-Bacon') # put meal 62 | appetizers.put('Egg-Sausage-Bacon') 63 | appetizers.put('Egg and Spam') 64 | appetizers.put('Egg-Bacon and Spam') 65 | appetizers.put('Egg-Bacon-Sausage') 66 | appetizers.put('Spam-Bacon-Sausage-Spam') 67 | appetizers.put('Spam-Egg-Spam-Spam-Bacon-Spam') 68 | appetizers.put('Spam-Spam-Spam-Egg-Spam') 69 | # Spam, Spam, Spam, Spam… Lovely Spam! Wonderful Spam! 70 | ... 71 | 72 | .. code-block:: python 73 | 74 | # On consumer number 1 (Viking): 75 | appetizers = queue.tube('appt-s') 76 | meal = appetizers.take(30) # wait for 'meal' in blocking mode for 30 seconds 77 | while meal is not None: 78 | if meal.data.find('Spam') == -1: # we don't want meal without spam 79 | meal.release(delay=1) 80 | else: 81 | eat(meal.data) # do something 82 | meal.ack() # acknowlege, that you did all you want with this 'meal' 83 | meal = appetizers.take(30) # take next meal 84 | exit(1) # no meal for 30 seconds, go out from here 85 | 86 | .. code-block:: python 87 | 88 | # On consumer number 2 (Missus): 89 | appetizers = queue.tube('appt-s') 90 | meal = appetizers.take(30) # wait for 'meal' in blocking mode for 30 seconds 91 | while meal is not None: 92 | if meal.data.find('Spam') != -1: # she is tired from spam 93 | meal.release(delay=1) 94 | else: 95 | eat(meal.data) # do something 96 | meal.ack() # acknowlege, that you did all you want with this 'meal' 97 | meal = appetizers.take(30) # take next meal 98 | exit(1) # no meal for 30 seconds, go out from here 99 | 100 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 101 | What if we forget to ack or release the task? 102 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 103 | 104 | Task class has destructor, that automaticly releases the task, if you have done nothing with it. e.g: 105 | 106 | .. code-block:: python 107 | 108 | # You're consumer of some great spam: 109 | def eat_spam(tube): 110 | meal = tube.take() 111 | if (meal.data.find('Spam') != -1) 112 | meal.ack() 113 | consume(meal) # do_something 114 | return # oops! we forget to release task if it has not spam in it! 115 | # but that's ok, GC will do it when his time will come. 116 | 117 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 118 | What data we can push into tubes? 119 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 120 | 121 | Queue uses `msgpack `_ (It's like JSON. but fast and small) for default `serializing` of data, so by default you may `serialize` **dicts**, **tuples**/**lists**, **strings**, **numbers** and **others basic types**. 122 | 123 | If you want to push another objects to Tubes you may define another `serializers`. By default `serializers` of Tubes are None and it uses Queue `serializer`. If you set Tube `serializer` to callable object it will use it, instead of Queue `serializer`. e.g.: 124 | 125 | .. code-block:: python 126 | 127 | import bz2 128 | import json 129 | import pickle 130 | 131 | from tarantool_queue import Queue 132 | 133 | queue = Queue('localhost', 33020, 0) 134 | 135 | jsons = queue.tube('json') 136 | jsons.serialize = (lambda x: json.dumps(x)) # it's not necessary to use lambda in your projects 137 | jsons.deserialize = (lambda x: json.loads(x)) # but object, that'll serialize and deserialize must be callable or None 138 | 139 | pickls = queue.tube('pickle') 140 | pickls.serialize = (lambda x: pickle.dump(x)) 141 | pickls.deserialize = (lambda x: pickle.load(x)) 142 | 143 | bz2s = queue.tube('bz2') 144 | bz2s.serialize = (lambda x: bz2.compress(json.dumps(x))) 145 | bz2s.deserialize = (lambda x: json.loads(bz2.decompress(x))) 146 | 147 | default = queue.tube('default') 148 | 149 | jsons.put([1, 2, 3]) # it will put [1, 2, 3] in json into queue. 150 | pickls.put([2, 3, 4]) # it will pickle [2, 3, 4] and put it into queue. 151 | bz2.put([3, 4, 5]) # it will bzip' [3, 4, 5] in json and put it into queue. 152 | 153 | default.put([4, 5, 6]) # msgpack will pack it and put into queue. 154 | queue.serialize = (lambda x: pickle.dump(x)) 155 | queue.deserialize = (lambda x: pickle.load(x)) 156 | default.put([4, 5, 6]) # but now it'll be pickled. 157 | 158 | # to reset serializers you must simply assign None to serializer: 159 | queue.serialize = None # it will restore msgpack as serializer 160 | queue.deserialize = None # it will restore msgpack as deserializer 161 | bz2s.serialize = None # it will tell python to use Queue serializer(msgpack) instead of bz2 162 | bz2s.deserialize = None # it will tell python to use Queue deserializer(msgpack) instead of bz2 163 | default.put([4, 5, 6]) # msgpack will pack it again. 164 | 165 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 166 | But, i have very important task that needs to be done! 167 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 168 | 169 | It's OK! You must use **Tube.urgent(data)**! 170 | 171 | .. code-block:: python 172 | 173 | appetizers = queue.tube('appt-s') 174 | appetizers.put('Egg-Bacon') # put meal 175 | appetizers.put('Egg-Sausage-Bacon') # another boring meal 176 | appetizers.urgent('Spam-Egg-Spam-Spam-Bacon-Spam') # very very tasty meal with a lot of SPAM 177 | 178 | meal1 = appetizers.take() ; print meal1.data # Spam-Egg-Spam-Spam-Bacon-Spam 179 | meal2 = appetizers.take() ; print meal2.data # Egg-Bacon 180 | meal3 = appetizers.take() ; print meal3.data # Egg-Sausage-Bacon 181 | 182 | meal1.ack() ; meal2.ack() ; meal3.ack() 183 | 184 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 185 | Ok! But i've some spam today! I want to know how much. 186 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 187 | 188 | .. code-block:: python 189 | 190 | appetizers = queue.tube('appt-s') 191 | appetizers.statistics() # will show you how many spam you've 'baked' and 'sold' 192 | queue.statistics() # will show you overall stats of your cafe 193 | 194 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 195 | I have some spam, that is so awfully bad, that i want to bury deep inside. 196 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 197 | 198 | .. code-block:: python 199 | 200 | appetizers = queue.tube('appt-s') 201 | task = appetizers.get() 202 | task.bury() # it will bury meal deep inside 203 | task.dig() # it will 'unbury' meal, if you'll need it in future. 204 | task.delete() # it will destroy your 'meal' once and for all. 205 | appetizers.kick(number) # it will 'unbury' a number of tasks in this Tube. 206 | task.done('New great SPAM with SPAM and HAM') # or you may replace this 'meal' with another. 207 | 208 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 209 | But *Task.release()* returns task into the beggining! I want it to be in the end! 210 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 211 | 212 | Simply use **Task.requeue()** instead of **Task.release()**! 213 | 214 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 215 | SUDDENLY I have UUID of my 'meal', and i REALLY REALLY want this meal. What should i do? 216 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 217 | 218 | You must use **Queue.peek(uuid)** method! 219 | 220 | .. code-block:: python 221 | 222 | appetizers = queue.tube('appt-s') 223 | meal_uuid = '550e8400-e29b-41d4-a716-446655440000' 224 | task = queue.peek(meal_uuid) 225 | print task.data # Spam-Egg-Spam-Spam-Bacon-Spam 226 | 227 | ^^^^^^^^^^^^^^^ 228 | Question-Answer 229 | ^^^^^^^^^^^^^^^ 230 | *Q*. What should i do, to use my own great tarantool connector in this Queue? How may i 231 | reset it into defaults? 232 | 233 | *A*. You must simply use **Queue.tarantool_connector** field for setting it. Just ensure 234 | that your connector has **constructor** and **call** fields. 235 | 236 | For reseting it simply do: 237 | 238 | .. code-block:: python 239 | 240 | del(queue.tarantool_connector) 241 | # OR 242 | queue.tarantool_connector = None 243 | 244 | *Q*. I'm using another great coroutines library! I really need another locking mechanism, 245 | instead of your threading.Lock. 246 | 247 | *A*. It's ok! You may simply set **Queue.tarantool_lock** field with your lock. Just 248 | assure that your locking mechanism has **__enter__** and **__exit__** methods 249 | (your lock will be used in the "with LOCK:..." construction) 250 | 251 | For reseting it simply do: 252 | 253 | .. code-block:: python 254 | 255 | del(queue.tarantool_lock) 256 | # OR 257 | queue.tarantool_lock = None 258 | 259 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 260 | And Now for Something Completely Different.. 261 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 262 | 263 | .. image:: http://www.madmumblings.com/gallery/albums/userpics/10026/spam%20spam%20lovely%20spam.jpg 264 | -------------------------------------------------------------------------------- /requires.txt: -------------------------------------------------------------------------------- 1 | msgpack-python 2 | tarantool<0.4 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import codecs 4 | import os 5 | import re 6 | 7 | try: 8 | from setuptools import setup 9 | except ImportError: 10 | from distutils.core import setup 11 | 12 | cmdclass = {} 13 | 14 | try: 15 | from sphinx.setup_command import BuildDoc 16 | cmdclass["build_sphinx"] = BuildDoc 17 | except ImportError: 18 | pass 19 | 20 | 21 | def read(*parts): 22 | filename = os.path.join(os.path.dirname(__file__), *parts) 23 | with codecs.open(filename, encoding='utf-8') as fp: 24 | return fp.read() 25 | 26 | 27 | def find_version(*file_paths): 28 | version_file = read(*file_paths) 29 | version_match = re.search(r"""^__version__\s*=\s*(['"])(.+)\1""", 30 | version_file, re.M) 31 | if version_match: 32 | return version_match.group(2) 33 | raise RuntimeError("Unable to find version string.") 34 | 35 | 36 | setup( 37 | name='tarantool-queue', 38 | version=find_version('tarantool_queue', '__init__.py'), 39 | description=( 40 | 'Python bindings for Tarantool queue script' 41 | ' (http://github.com/tarantool/queue)' 42 | ), 43 | long_description=read('README.rst'), 44 | author='Eugine Blikh', 45 | author_email='bigbes@gmail.com', 46 | maintainer='Eugine Blikh', 47 | maintainer_email='bigbes@gmail.com', 48 | license='MIT', 49 | packages=['tarantool_queue'], 50 | platforms=["all"], 51 | install_requires=[ 52 | 'msgpack-python', 53 | 'tarantool<0.4' 54 | ], 55 | url='http://github.com/tarantool/tarantool-queue-python', 56 | test_suite='tests.test_queue', 57 | tests_require=[ 58 | 'msgpack-python', 59 | 'tarantool' 60 | ], 61 | classifiers=[ 62 | 'Development Status :: 4 - Beta', 63 | 'Operating System :: OS Independent', 64 | 'Intended Audience :: Developers', 65 | 'License :: OSI Approved :: MIT License', 66 | 'Programming Language :: Python :: 2.6', 67 | 'Programming Language :: Python :: 2.7', 68 | 'Topic :: Database :: Front-Ends', 69 | 'Environment :: Console' 70 | ], 71 | cmdclass=cmdclass 72 | ) 73 | -------------------------------------------------------------------------------- /tarantool_queue/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.4" 2 | 3 | from .tarantool_queue import Queue 4 | from .tarantool_tqueue import TQueue 5 | 6 | __all__ = [Queue, TQueue, __version__] 7 | -------------------------------------------------------------------------------- /tarantool_queue/tarantool_queue.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import re 3 | import struct 4 | import msgpack 5 | import threading 6 | 7 | import tarantool 8 | 9 | 10 | def unpack_long_long(value): 11 | return struct.unpack("` put, 287 | but it returns None if task exists 288 | """ 289 | return self._produce("queue.put_unique", data, **kwargs) 290 | 291 | def urgent(self, data=None, **kwargs): 292 | """ 293 | Same as :meth:`Tube.put() ` put, 294 | but set highest priority for this task. 295 | """ 296 | kwargs['delay'] = 0 297 | return self._produce("queue.urgent", data, **kwargs) 298 | 299 | def take(self, timeout=0): 300 | """ 301 | If there are tasks in the queue ready for execution, 302 | take the highest-priority task. Otherwise, wait for a 303 | ready task to appear in the queue, and, as soon as it appears, 304 | mark it as taken and return to the consumer. If there is a 305 | timeout, and the task doesn't appear until the timeout expires, 306 | return 'None'. If timeout is None, wait indefinitely until 307 | a task appears. 308 | 309 | :param timeout: timeout to wait. 310 | :type timeout: int or None 311 | :rtype: `Task` instance or None 312 | """ 313 | return self.queue._take(self.opt['tube'], timeout) 314 | 315 | def kick(self, count=None): 316 | """ 317 | 'Dig up' count tasks in a queue. If count is not given, digs up 318 | just one buried task. 319 | 320 | :rtype boolean 321 | """ 322 | return self.queue._kick(self.opt['tube'], count) 323 | 324 | def statistics(self): 325 | """ 326 | See :meth:`Queue.statistics() ` 327 | for more information. 328 | """ 329 | return self.queue.statistics(tube=self.opt['tube']) 330 | 331 | def truncate(self): 332 | """ 333 | Truncate tube 334 | """ 335 | return self.queue.truncate(tube=self.opt['tube']) 336 | 337 | 338 | class Queue(object): 339 | """ 340 | Tarantool queue wrapper. Surely pinned to space. May create tubes. 341 | By default it uses msgpack for serialization, but you may redefine 342 | serialize and deserialize methods. 343 | You must use Queue only for creating Tubes. 344 | For more usage, please, look into tests. 345 | Usage: 346 | 347 | >>> from tarantool_queue import Queue 348 | >>> queue = Queue() 349 | >>> tube1 = queue.create_tube('holy_grail', ttl=100, delay=5) 350 | # Put task into the queue 351 | >>> tube1.put([1, 2, 3]) 352 | # Put task into the beggining of queue (Highest priority) 353 | >>> tube1.urgent([2, 3, 4]) 354 | >>> tube1.get() # We get task and automaticaly release it 355 | >>> task1 = tube1.take() 356 | >>> task2 = tube1.take() 357 | >>> print(task1.data) 358 | [2, 3, 4] 359 | >>> print(task2.data) 360 | [1, 2, 3] 361 | >>> del task2 362 | >>> del task1 363 | >>> print(tube1.take().data) 364 | [1, 2, 3] 365 | # Take task and Ack it 366 | >>> tube1.take().ack() 367 | True 368 | """ 369 | 370 | DataBaseError = tarantool.DatabaseError 371 | NetworkError = tarantool.NetworkError 372 | 373 | class BadConfigException(Exception): 374 | pass 375 | 376 | class ZeroTupleException(Exception): 377 | pass 378 | 379 | @staticmethod 380 | def basic_serialize(data): 381 | return msgpack.packb(data) 382 | 383 | @staticmethod 384 | def basic_deserialize(data): 385 | return msgpack.unpackb(data) 386 | 387 | def __init__(self, host="localhost", port=33013, space=0, schema=None): 388 | if not(host and port): 389 | raise Queue.BadConfigException("host and port params " 390 | "must be not empty") 391 | 392 | if not isinstance(port, int): 393 | raise Queue.BadConfigException("port must be int") 394 | 395 | if not isinstance(space, int): 396 | raise Queue.BadConfigException("space must be int") 397 | 398 | self.host = host 399 | self.port = port 400 | self.space = space 401 | self.schema = schema 402 | self.tubes = {} 403 | self._serialize = self.basic_serialize 404 | self._deserialize = self.basic_deserialize 405 | 406 | # ---------------- 407 | @property 408 | def serialize(self): 409 | """ 410 | Serialize function: must be Callable. If sets to None or deleted, then 411 | it will use msgpack for serializing. 412 | """ 413 | if not hasattr(self, '_serialize'): 414 | self._serialize = self.basic_serialize 415 | return self._serialize 416 | 417 | @serialize.setter 418 | def serialize(self, func): 419 | if not (hasattr(func, '__call__') or func is None): 420 | raise TypeError("func must be Callable " 421 | "or None, but not " + str(type(func))) 422 | self._serialize = func if func is not None else self.basic_serialize 423 | 424 | @serialize.deleter 425 | def serialize(self): 426 | self._serialize = self.basic_serialize 427 | 428 | # ---------------- 429 | @property 430 | def deserialize(self): 431 | """ 432 | Deserialize function: must be Callable. If sets to None or delete, 433 | then it will use msgpack for deserializing. 434 | """ 435 | if not hasattr(self, '_deserialize'): 436 | self._deserialize = self.basic_deserialize 437 | return self._deserialize 438 | 439 | @deserialize.setter 440 | def deserialize(self, func): 441 | if not (hasattr(func, '__call__') or func is None): 442 | raise TypeError("func must be Callable " 443 | "or None, but not " + str(type(func))) 444 | self._deserialize = (func 445 | if func is not None 446 | else self.basic_deserialize) 447 | 448 | @deserialize.deleter 449 | def deserialize(self): 450 | self._deserialize = self.basic_deserialize 451 | 452 | # ---------------- 453 | @property 454 | def tarantool_connection(self): 455 | """ 456 | Tarantool Connection class: must be class with methods call and 457 | __init__. If it sets to None or deleted - it will use the default 458 | tarantool.Connection class for connection. 459 | """ 460 | if not hasattr(self, '_conclass'): 461 | self._conclass = tarantool.Connection 462 | return self._conclass 463 | 464 | @tarantool_connection.setter 465 | def tarantool_connection(self, cls): 466 | if 'call' not in dir(cls) or '__init__' not in dir(cls): 467 | if cls is not None: 468 | raise TypeError("Connection class must have" 469 | " connect and call methods or be None") 470 | self._conclass = cls if cls is not None else tarantool.Connection 471 | if hasattr(self, '_tnt'): 472 | self.__dict__.pop('_tnt') 473 | 474 | @tarantool_connection.deleter 475 | def tarantool_connection(self): 476 | if hasattr(self, '_conclass'): 477 | self.__dict__.pop('_conclass') 478 | if hasattr(self, '_tnt'): 479 | self.__dict__.pop('_tnt') 480 | 481 | # ---------------- 482 | @property 483 | def tarantool_lock(self): 484 | """ 485 | Locking class: must be locking instance with methods __enter__ 486 | and __exit__. If it sets to None or delete - it will use default 487 | threading.Lock() instance for locking in the connecting. 488 | """ 489 | if not hasattr(self, '_lockinst'): 490 | self._lockinst = threading.Lock() 491 | return self._lockinst 492 | 493 | @tarantool_lock.setter 494 | def tarantool_lock(self, lock): 495 | if '__enter__' not in dir(lock) or '__exit__' not in dir(lock): 496 | if lock is not None: 497 | raise TypeError("Lock class must have `__enter__`" 498 | " and `__exit__` methods or be None") 499 | self._lockinst = lock if lock is not None else threading.Lock() 500 | 501 | @tarantool_lock.deleter 502 | def tarantool_lock(self): 503 | if hasattr(self, '_lockinst'): 504 | self.__dict__.pop('_lockinst') 505 | 506 | # ---------------- 507 | @property 508 | def tnt(self): 509 | if not hasattr(self, '_tnt'): 510 | with self.tarantool_lock: 511 | if not hasattr(self, '_tnt'): 512 | self._tnt = self.tarantool_connection(self.host, self.port, 513 | schema=self.schema) 514 | return self._tnt 515 | 516 | def _take(self, tube, timeout=0): 517 | args = [str(self.space), str(tube)] 518 | if timeout is not None: 519 | args.append(str(timeout)) 520 | the_tuple = self.tnt.call("queue.take", tuple(args)) 521 | if the_tuple.rowcount == 0: 522 | return None 523 | return Task.from_tuple(self, the_tuple) 524 | 525 | def _ack(self, task_id): 526 | args = (str(self.space), task_id) 527 | the_tuple = self.tnt.call("queue.ack", args) 528 | return the_tuple.return_code == 0 529 | 530 | def _release(self, task_id, delay=0, ttl=0): 531 | the_tuple = self.tnt.call("queue.release", ( 532 | str(self.space), 533 | str(task_id), 534 | str(delay), 535 | str(ttl) 536 | )) 537 | return Task.from_tuple(self, the_tuple) 538 | 539 | def _requeue(self, task_id): 540 | args = (str(self.space), task_id) 541 | the_tuple = self.tnt.call("queue.requeue", args) 542 | return the_tuple.return_code == 0 543 | 544 | def _bury(self, task_id): 545 | args = (str(self.space), task_id) 546 | the_tuple = self.tnt.call("queue.bury", args) 547 | return the_tuple.return_code == 0 548 | 549 | def _delete(self, task_id): 550 | args = (str(self.space), task_id) 551 | the_tuple = self.tnt.call("queue.delete", args) 552 | return the_tuple.return_code == 0 553 | 554 | def _meta(self, task_id): 555 | args = (str(self.space), task_id) 556 | the_tuple = self.tnt.call("queue.meta", args) 557 | if the_tuple.rowcount: 558 | row = list(the_tuple[0]) 559 | for index in [3, 7, 8, 9, 10, 11, 12]: 560 | row[index] = unpack_long_long(row[index]) 561 | for index in [6]: 562 | row[index] = unpack_long(row[index]) 563 | keys = [ 564 | 'task_id', 'tube', 'status', 'event', 'ipri', 565 | 'pri', 'cid', 'created', 'ttl', 'ttr', 'cbury', 566 | 'ctaken', 'now' 567 | ] 568 | return dict(zip(keys, row)) 569 | return None 570 | 571 | def peek(self, task_id): 572 | """ 573 | Return a task by task id. 574 | 575 | :param task_id: UUID of task in HEX 576 | :type task_id: string 577 | :rtype: `Task` instance 578 | """ 579 | args = (str(self.space), task_id) 580 | the_tuple = self.tnt.call("queue.peek", args) 581 | return Task.from_tuple(self, the_tuple) 582 | 583 | def _dig(self, task_id): 584 | args = (str(self.space), task_id) 585 | the_tuple = self.tnt.call("queue.dig", args) 586 | return the_tuple.return_code == 0 587 | 588 | def _kick(self, tube, count=None): 589 | args = [str(self.space), str(tube)] 590 | if count: 591 | args.append(str(count)) 592 | the_tuple = self.tnt.call("queue.kick", tuple(args)) 593 | return the_tuple.return_code == 0 594 | 595 | def truncate(self, tube): 596 | """ 597 | Truncate queue tube, return quantity of deleted tasks 598 | 599 | :param tube: Name of tube 600 | :type tube: string 601 | :rtype: int 602 | """ 603 | args = (str(self.space), tube) 604 | deleted = self.tnt.call("queue.truncate", args) 605 | return unpack_long(deleted[0][0]) 606 | 607 | def statistics(self, tube=None): 608 | """ 609 | Return queue module statistics accumulated since server start. 610 | Output format: if tube != None, then output is dictionary with 611 | stats of current tube. If tube is None, then output is dict of 612 | t stats, ...} 613 | e.g.: 614 | 615 | >>> tube.statistics() 616 | # or queue.statistics('tube0') 617 | # or queue.statistics(tube.opt['tube']) 618 | {'ack': '233', 619 | 'meta': '35', 620 | 'put': '153', 621 | 'release': '198', 622 | 'take': '431', 623 | 'take_timeout': '320', 624 | 'tasks': {'buried': '0', 625 | 'delayed': '0', 626 | 'done': '0', 627 | 'ready': '0', 628 | 'taken': '0', 629 | 'total': '0'}, 630 | 'urgent': '80'} 631 | or 632 | >>> queue.statistics() 633 | {'tube0': {'ack': '233', 634 | 'meta': '35', 635 | 'put': '153', 636 | 'release': '198', 637 | 'take': '431', 638 | 'take_timeout': '320', 639 | 'tasks': {'buried': '0', 640 | 'delayed': '0', 641 | 'done': '0', 642 | 'ready': '0', 643 | 'taken': '0', 644 | 'total': '0'}, 645 | 'urgent': '80'}} 646 | 647 | :param tube: Name of tube 648 | :type tube: string or None 649 | :rtype: dict with statistics 650 | """ 651 | args = (str(self.space),) 652 | args = args if tube is None else args + (tube,) 653 | stat = self.tnt.call("queue.statistics", args) 654 | ans = {} 655 | if stat.rowcount > 0: 656 | for k, v in zip(stat[0][0::2], stat[0][1::2]): 657 | k_t = list( 658 | re.match(r'space([^.]*)\.(.*)\.([^.]*)', k).groups() 659 | ) 660 | if int(k_t[0]) != self.space: 661 | continue 662 | if k_t[1].endswith('.tasks'): 663 | k_t = k_t[0:1] + k_t[1].rsplit('.', 1) + k_t[2:3] 664 | if k_t[1] not in ans: 665 | ans[k_t[1]] = {'tasks': {}} 666 | if len(k_t) == 4: 667 | ans[k_t[1]]['tasks'][k_t[-1]] = v 668 | elif len(k_t) == 3: 669 | ans[k_t[1]][k_t[-1]] = v 670 | else: 671 | raise Queue.ZeroTupleException('stats: \ 672 | error when parsing respons') 673 | return ans[tube] if tube else ans 674 | 675 | def _touch(self, task_id): 676 | args = (str(self.space), task_id) 677 | the_tuple = self.tnt.call("queue.touch", tuple(args)) 678 | return the_tuple.return_code == 0 679 | 680 | def tube(self, name, **kwargs): 681 | """ 682 | Create Tube object, if not created before, and set kwargs. 683 | If existed, return existed Tube. 684 | 685 | :param name: name of Tube 686 | :param delay: default delay for Tube tasks (Not necessary, will be 0) 687 | :param ttl: default TTL for Tube tasks (Not necessary, will be 0) 688 | :param ttr: default TTR for Tube tasks (Not necessary, will be 0) 689 | :param pri: default priority for Tube tasks (Not necessary) 690 | :type name: string 691 | :type ttl: int 692 | :type delay: int 693 | :type ttr: int 694 | :type pri: int 695 | :rtype: `Tube` instance 696 | """ 697 | if name in self.tubes: 698 | tube = self.tubes[name] 699 | tube.update_options(**kwargs) 700 | else: 701 | tube = Tube(self, name, **kwargs) 702 | self.tubes[name] = tube 703 | return tube 704 | -------------------------------------------------------------------------------- /tarantool_queue/tarantool_tqueue.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import struct 3 | import msgpack 4 | import threading 5 | 6 | import tarantool 7 | 8 | 9 | def unpack_long_long(value): 10 | return struct.unpack(">> from tarantool_queue import TQueue 233 | >>> queue = TQueue() 234 | >>> tube1 = queue.create_tube('holy_grail', ttl=100, delay=5) 235 | # Put task into the queue 236 | >>> tube1.put([1, 2, 3]) 237 | # Put task into the beggining of queue (Highest priority) 238 | >>> tube1.urgent([2, 3, 4]) 239 | >>> tube1.get() # We get task and automaticaly release it 240 | >>> task1 = tube1.take() 241 | >>> task2 = tube1.take() 242 | >>> print(task1.data) 243 | [2, 3, 4] 244 | >>> print(task2.data) 245 | [1, 2, 3] 246 | >>> del task2 247 | >>> del task1 248 | >>> print(tube1.take().data) 249 | [1, 2, 3] 250 | # Take task and Ack it 251 | >>> tube1.take().ack() 252 | True 253 | """ 254 | 255 | DataBaseError = tarantool.DatabaseError 256 | NetworkError = tarantool.NetworkError 257 | 258 | class BadConfigException(Exception): 259 | pass 260 | 261 | class ZeroTupleException(Exception): 262 | pass 263 | 264 | class NoDataException(Exception): 265 | pass 266 | 267 | @staticmethod 268 | def basic_serialize(data): 269 | return msgpack.packb(data) 270 | 271 | @staticmethod 272 | def basic_deserialize(data): 273 | return msgpack.unpackb(data) 274 | 275 | def __init__(self, host="localhost", port=33013, space=0, schema=None): 276 | if not(host and port): 277 | raise TQueue.BadConfigException( 278 | "host and port params must be not empty") 279 | 280 | if not isinstance(port, int): 281 | raise TQueue.BadConfigException("port must be int") 282 | 283 | if not isinstance(space, int): 284 | raise TQueue.BadConfigException("space must be int") 285 | 286 | self.host = host 287 | self.port = port 288 | self.space = space 289 | self.schema = schema 290 | self.tubes = {} 291 | self._serialize = self.basic_serialize 292 | self._deserialize = self.basic_deserialize 293 | 294 | # ---------------- 295 | @property 296 | def serialize(self): 297 | """ 298 | Serialize function: must be Callable. If sets to None or deleted, then 299 | it will use msgpack for serializing. 300 | """ 301 | if not hasattr(self, '_serialize'): 302 | self.serialize = self.basic_serialize 303 | return self._serialize 304 | 305 | @serialize.setter 306 | def serialize(self, func): 307 | if not (hasattr(func, '__call__') or func is None): 308 | raise TypeError("func must be Callable " 309 | "or None, but not " + str(type(func))) 310 | self._serialize = func if func is not None else self.basic_serialize 311 | 312 | @serialize.deleter 313 | def serialize(self): 314 | self._serialize = self.basic_serialize 315 | 316 | # ---------------- 317 | @property 318 | def deserialize(self): 319 | """ 320 | Deserialize function: must be Callable. If sets to None or delete, 321 | then it will use msgpack for deserializing. 322 | """ 323 | if not hasattr(self, '_deserialize'): 324 | self._deserialize = self.basic_deserialize 325 | return self._deserialize 326 | 327 | @deserialize.setter 328 | def deserialize(self, func): 329 | if not (hasattr(func, '__call__') or func is None): 330 | raise TypeError("func must be Callable " 331 | "or None, but not " + str(type(func))) 332 | self._deserialize = (func 333 | if func is not None 334 | else self.basic_deserialize) 335 | 336 | @deserialize.deleter 337 | def deserialize(self): 338 | self._deserialize = self.basic_deserialize 339 | 340 | # ---------------- 341 | @property 342 | def tarantool_connection(self): 343 | """ 344 | Tarantool Connection class: must be class with methods call and 345 | __init__. If it sets to None or deleted - it will use the default 346 | tarantool.Connection class for connection. 347 | """ 348 | if not hasattr(self, '_conclass'): 349 | self._conclass = tarantool.Connection 350 | return self._conclass 351 | 352 | @tarantool_connection.setter 353 | def tarantool_connection(self, cls): 354 | if 'call' not in dir(cls) or '__init__' not in dir(cls): 355 | if cls is not None: 356 | raise TypeError("Connection class must have" 357 | " connect and call methods or be None") 358 | self._conclass = cls if cls is not None else tarantool.Connection 359 | if hasattr(self, '_tnt'): 360 | self.__dict__.pop('_tnt') 361 | 362 | @tarantool_connection.deleter 363 | def tarantool_connection(self): 364 | if hasattr(self, '_conclass'): 365 | self.__dict__.pop('_conclass') 366 | if hasattr(self, '_tnt'): 367 | self.__dict__.pop('_tnt') 368 | 369 | # ---------------- 370 | @property 371 | def tarantool_lock(self): 372 | """ 373 | Locking class: must be locking instance with methods __enter__ 374 | and __exit__. If it sets to None or delete - it will use default 375 | threading.Lock() instance for locking in the connecting. 376 | """ 377 | if not hasattr(self, '_lockinst'): 378 | self._lockinst = threading.Lock() 379 | return self._lockinst 380 | 381 | @tarantool_lock.setter 382 | def tarantool_lock(self, lock): 383 | if '__enter__' not in dir(lock) or '__exit__' not in dir(lock): 384 | if lock is not None: 385 | raise TypeError("Lock class must have `__enter__`" 386 | " and `__exit__` methods or be None") 387 | self._lockinst = lock if lock is not None else threading.Lock() 388 | 389 | @tarantool_lock.deleter 390 | def tarantool_lock(self): 391 | if hasattr(self, '_lockinst'): 392 | self.__dict__.pop('_lockinst') 393 | 394 | # ---------------- 395 | @property 396 | def tnt(self): 397 | if not hasattr(self, '_tnt'): 398 | with self.tarantool_lock: 399 | if not hasattr(self, '_tnt'): 400 | self._tnt = self.tarantool_connection(self.host, self.port, 401 | schema=self.schema) 402 | return self._tnt 403 | 404 | def _take(self, tube, timeout=0): 405 | args = [str(self.space), str(tube)] 406 | if timeout is not None: 407 | args.append(str(timeout)) 408 | the_tuple = self.tnt.call("box.queue.take", tuple(args)) 409 | if the_tuple.rowcount == 0: 410 | return None 411 | return TTask.from_tuple(self, the_tuple) 412 | 413 | def _ack(self, task_id): 414 | args = (str(self.space), str(task_id)) 415 | the_tuple = self.tnt.call("box.queue.ack", args) 416 | return the_tuple.return_code == 0 417 | 418 | def _release(self, task_id, prio=0x7fff, delay=0, ttr=300, ttl=0, retry=5): 419 | the_tuple = self.tnt.call("box.queue.release", ( 420 | str(self.space), 421 | str(task_id), 422 | )) 423 | return TTask.from_tuple(self, the_tuple) 424 | 425 | def _delete(self, task_id): 426 | args = (str(self.space), str(task_id)) 427 | the_tuple = self.tnt.call("box.queue.delete", args) 428 | return the_tuple.return_code == 0 429 | 430 | def tube(self, name, **kwargs): 431 | """ 432 | Create Tube object, if not created before, and set kwargs. 433 | If existed, return existed Tube. 434 | 435 | :param name: name of Tube 436 | :param delay: default delay for Tube tasks (Not necessary, will be 0) 437 | :param ttl: default TTL for Tube tasks (Not necessary, will be 0) 438 | :param ttr: default TTR for Tube tasks (Not necessary, will be 0) 439 | :param pri: default priority for Tube tasks (Not necessary) 440 | :type name: string 441 | :type ttl: int 442 | :type delay: int 443 | :type ttr: int 444 | :type pri: int 445 | :rtype: `Tube` instance 446 | """ 447 | if name in self.tubes: 448 | tube = self.tubes[name] 449 | tube.update_options(**kwargs) 450 | else: 451 | tube = TTube(self, name, **kwargs) 452 | self.tubes[name] = tube 453 | return tube 454 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tarantool/queue-python/87190c3618142a4f14b29f007047f45edc9b3622/tests/__init__.py -------------------------------------------------------------------------------- /tests/tarantool.cfg: -------------------------------------------------------------------------------- 1 | readahead = 16384 2 | 3 | primary_port = 33013 4 | secondary_port = 33014 5 | admin_port = 33015 6 | 7 | space = [ 8 | { 9 | enabled = 1, 10 | index = [ 11 | { 12 | type = "TREE", 13 | unique = 1, 14 | key_field = [ 15 | { 16 | fieldno = 0, 17 | type = "STR" 18 | } 19 | ] 20 | }, 21 | { 22 | type = "TREE", 23 | unique = 0, 24 | key_field = [ 25 | { 26 | fieldno = 1, # tube 27 | type = "STR" 28 | }, 29 | { 30 | fieldno = 2, # status 31 | type = "STR" 32 | }, 33 | { 34 | fieldno = 4, # ipri 35 | type = "STR" 36 | }, 37 | { 38 | fieldno = 5 # pri 39 | type = "STR" 40 | } 41 | ] 42 | }, 43 | { 44 | type = "TREE", 45 | unique = 0, 46 | key_field = [ 47 | { 48 | fieldno = 1, # tube 49 | type = "STR" 50 | }, 51 | { 52 | fieldno = 3, # next_event 53 | type = "NUM64" 54 | } 55 | ] 56 | }, 57 | # { 58 | # type = "TREE", 59 | # unique = 0, 60 | # key_field = [ 61 | # { 62 | # fieldno = 1, # tube 63 | # type = "STR" 64 | # }, 65 | # { 66 | # fieldno = 2, # status 67 | # type = "STR" 68 | # }, 69 | # { 70 | # fieldno = 12, # task data 71 | # type = "STR" 72 | # } 73 | # ] 74 | # } 75 | ] 76 | } 77 | ] 78 | -------------------------------------------------------------------------------- /tests/test_queue.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import msgpack 3 | import unittest 4 | import threading 5 | 6 | from tarantool_queue import Queue 7 | import tarantool 8 | 9 | 10 | class TestSuite_Basic(unittest.TestCase): 11 | @classmethod 12 | def setUpClass(cls): 13 | cls.queue = Queue("127.0.0.1", 33013, 0) 14 | cls.tube = cls.queue.tube("tube") 15 | 16 | @classmethod 17 | def tearDownClass(cls): 18 | sys.stdout.write("tearDown ...") 19 | for tube in cls.queue.tubes.values(): 20 | task = tube.take(1) 21 | while (task is not None): 22 | task.ack() 23 | task = tube.take(1) 24 | print(" ok") 25 | 26 | 27 | class TestSuite_00_ConnectionTest(TestSuite_Basic): 28 | def test_00_ConProp(self): 29 | conn = self.queue.tnt 30 | self.assertTrue(isinstance(conn, tarantool.connection.Connection)) 31 | self.assertTrue(isinstance(self.queue.tnt, tarantool.connection.Connection)) 32 | self.assertEqual(self.queue.tnt, conn) 33 | 34 | def test_01_Tubes(self): 35 | tube1 = self.queue.tube("tube1") 36 | self.assertEqual(self.queue.tube("tube1"), tube1) 37 | tube1.put([1, 2, 3]) 38 | tube1.put([2, 3, 4]) 39 | self.queue.tube("tube1").take().ack() 40 | self.queue.tube("tube1").take().ack() 41 | 42 | def test_02_Expire(self): 43 | with self.assertRaises(AttributeError): 44 | self.tube.take(1).ack() 45 | 46 | def test_03_TaskMeta(self): 47 | task = self.tube.put([1, 2, 3, 4]) 48 | task_meta = task.meta() 49 | self.assertIsInstance(task_meta, dict) 50 | self.assertEqual(task_meta.keys(), ['status', 'task_id', 'cid', 'ttr', 'tube', 'created', 'pri', 'ctaken', 'ipri', 'cbury', 'ttl', 'now', 'event']) 51 | self.tube.take().ack() 52 | 53 | def test_04_Stats(self): 54 | # in our case info for tube less than info for space 55 | stat_space = self.queue.statistics() 56 | stat_tube = self.tube.statistics() 57 | self.assertNotEqual(stat_space, stat_tube) 58 | 59 | def test_05_Urgent(self): 60 | task1_p = self.tube.put("basic prio") 61 | task2_p = self.tube.urgent("URGENT TASK") 62 | task3_p = self.tube.urgent("VERY VERY URGENT TASK") 63 | task1 = self.tube.take() 64 | task2 = self.tube.take() 65 | task3 = self.tube.take() 66 | self.assertEqual(task1_p.data, task3.data) 67 | self.assertEqual(task2_p.data, task2.data) 68 | self.assertEqual(task3_p.data, task1.data) 69 | task3.release() 70 | task1.release() 71 | task2.release() 72 | task1 = self.tube.take() 73 | task2 = self.tube.take() 74 | task3 = self.tube.take() 75 | # They must be in the same order 76 | self.assertEqual(task1_p.data, task3.data) 77 | self.assertEqual(task2_p.data, task2.data) 78 | self.assertEqual(task3_p.data, task1.data) 79 | task1.ack() 80 | task2.ack() 81 | task3.ack() 82 | 83 | def test_06_Destructor(self): 84 | task = self.tube.put("stupid task") 85 | # analog for del task; gc.gc() 86 | # task is not taken - must not send exception 87 | task.__del__() 88 | task = self.tube.take() 89 | # task in taken - must not send exception 90 | task.__del__() 91 | # task is acked - must not send exception 92 | self.tube.take().ack() 93 | 94 | def test_07_Truncate(self): 95 | self.tube.put("task#1") 96 | self.tube.put("task#2") 97 | self.tube.put("task#3") 98 | before_stat = self.tube.statistics() 99 | result = self.tube.truncate() 100 | after_stat = self.tube.statistics() 101 | self.assertEqual(before_stat['tasks']['ready'], '3') 102 | self.assertEqual(result, 3) 103 | self.assertEqual(after_stat['tasks']['ready'], '0') 104 | self.tube.put("task#1") 105 | self.tube.put("task#2") 106 | self.tube.put("task#3") 107 | before_stat = self.tube.statistics() 108 | result = self.queue.truncate('tube') 109 | after_stat = self.tube.statistics() 110 | self.assertEqual(before_stat['tasks']['ready'], '3') 111 | self.assertEqual(result, 3) 112 | self.assertEqual(after_stat['tasks']['ready'], '0') 113 | result1 = self.tube.truncate() 114 | result2 = self.tube.truncate() 115 | self.assertEqual(result1, result2) 116 | 117 | 118 | class TestSuite_01_SerializerTest(TestSuite_Basic): 119 | def test_00_CustomQueueSerializer(self): 120 | class A: 121 | def __init__(self, a=3, b=4): 122 | self.a = a 123 | self.b = b 124 | 125 | def __eq__(self, other): 126 | return (isinstance(self, type(other)) and 127 | self.a == other.a and 128 | self.b == other.b) 129 | 130 | self.queue.serialize = (lambda x: msgpack.packb([x.a, x.b])) 131 | self.queue.deserialize = (lambda x: A(*msgpack.unpackb(x))) 132 | a = A() 133 | task1 = self.tube.put(a) 134 | task2 = self.tube.take() 135 | self.assertEqual(task1.data, task2.data) 136 | self.assertEqual(a, task2.data) 137 | task2.ack() 138 | self.queue.serialize = self.queue.basic_serialize 139 | self.queue.deserialize = self.queue.basic_deserialize 140 | task1 = self.tube.put([1, 2, 3, "hello"]) 141 | task2 = self.tube.take() 142 | self.assertEqual(task1.data, task2.data) 143 | task2.ack() 144 | 145 | def test_01_CustomTubeQueueSerializers(self): 146 | class A: 147 | def __init__(self, a=3, b=4): 148 | self.a = a 149 | self.b = b 150 | 151 | def __eq__(self, other): 152 | return (isinstance(self, type(other)) and 153 | self.a == other.a and 154 | self.b == other.b) 155 | 156 | self.tube.serialize = (lambda x: msgpack.packb([x.a, x.b])) 157 | self.tube.deserialize = (lambda x: A(*msgpack.unpackb(x))) 158 | a = A() 159 | task1 = self.tube.put(a) 160 | task2 = self.tube.take() 161 | self.assertEqual(task1.data, task2.data) 162 | task2.ack() 163 | self.tube.serialize = None 164 | self.tube.deserialize = None 165 | a = [1, 2, 3, "hello"] 166 | task1 = self.tube.put(a) 167 | task2 = self.tube.take() 168 | self.assertEqual(task1.data, task2.data) 169 | self.assertEqual(a, task2.data) 170 | task2.ack() 171 | 172 | def test_02_CustomMixedSerializers(self): 173 | class A: 174 | def __init__(self, a=3, b=4): 175 | self.a = a 176 | self.b = b 177 | 178 | def __eq__(self, other): 179 | return (isinstance(self, type(other)) and 180 | self.a == other.a and 181 | self.b == other.b) 182 | 183 | class B: 184 | def __init__(self, a=5, b=6, c=7): 185 | self.a = a 186 | self.b = b 187 | self.c = c 188 | 189 | def __eq__(self, other): 190 | return (isinstance(self, type(other)) and 191 | self.a == other.a and 192 | self.b == other.b and 193 | self.c == other.c) 194 | 195 | self.queue.serialize = (lambda x: msgpack.packb([x.a, x.b])) 196 | self.queue.deserialize = (lambda x: A(*msgpack.unpackb(x))) 197 | self.tube.serialize = (lambda x: msgpack.packb([x.a, x.b, x.c])) 198 | self.tube.deserialize = (lambda x: B(*msgpack.unpackb(x))) 199 | b = B() 200 | task1 = self.tube.put(b) 201 | task2 = self.tube.take() 202 | task2.ack() 203 | self.assertEqual(task1.data, task2.data) 204 | self.assertEqual(b, task2.data) 205 | self.tube.serialize = None 206 | self.tube.deserialize = None 207 | a = A() 208 | task1 = self.tube.put(a) 209 | task2 = self.tube.take() 210 | task2.ack() 211 | self.assertEqual(task1.data, task2.data) 212 | self.assertEqual(a, task2.data) 213 | self.queue.serialize = self.queue.basic_serialize 214 | self.queue.deserialize = self.queue.basic_deserialize 215 | a = [1, 2, 3, "hello"] 216 | task1 = self.tube.put(a) 217 | task2 = self.tube.take() 218 | self.assertEqual(task1.data, task2.data) 219 | self.assertEqual(a, task2.data) 220 | task2.ack() 221 | 222 | 223 | class TestSuite_03_CustomLockAndConnection(TestSuite_Basic): 224 | def test_00_GoodLock(self): 225 | class GoodFake(object): 226 | def __enter__(self): 227 | pass 228 | 229 | def __exit__(self): 230 | pass 231 | 232 | self.queue.tarantool_lock = threading.Lock() 233 | self.queue.tarantool_lock = None 234 | self.assertTrue(isinstance(self.queue.tarantool_lock, type(threading.Lock()))) 235 | self.queue.tarantool_lock = threading.RLock() 236 | self.queue.tarantool_lock = GoodFake 237 | del(self.queue.tarantool_lock) 238 | self.assertTrue(isinstance(self.queue.tarantool_lock, type(threading.Lock()))) 239 | 240 | def test_01_GoodConnection(self): 241 | class GoodFake(object): 242 | def __init__(self): 243 | pass 244 | 245 | def call(self): 246 | pass 247 | 248 | self.queue.tarantool_connection = tarantool.Connection 249 | self.queue.statistics() # creating basic _tnt 250 | self.assertTrue(hasattr(self.queue, '_tnt')) # check that it exists 251 | self.queue.tarantool_connection = None # delete _tnt 252 | self.assertFalse(hasattr(self.queue, '_tnt')) # check that it doesn't exists 253 | self.assertEqual(self.queue.tarantool_connection, tarantool.Connection) 254 | self.queue.tarantool_connection = GoodFake 255 | del(self.queue.tarantool_connection) 256 | self.assertEqual(self.queue.tarantool_connection, tarantool.Connection) 257 | 258 | def test_02_BadLock(self): 259 | class BadFake(object): 260 | pass 261 | with self.assertRaises(TypeError): 262 | self.queue.tarantool_lock = BadFake 263 | with self.assertRaises(TypeError): 264 | self.queue.tarantool_lock = (lambda x: x) 265 | 266 | def test_03_BadConnection(self): 267 | class BadFake(object): 268 | pass 269 | with self.assertRaises(TypeError): 270 | self.queue.tarantool_lock = BadFake 271 | with self.assertRaises(TypeError): 272 | self.queue.tarantool_lock = (lambda x: x) 273 | 274 | def test_04_OverallTest(self): 275 | self.queue.tarantool_lock = threading.Lock() 276 | self.queue.tarantool_connection = tarantool.Connection 277 | self.assertIsNotNone(self.queue.statistics()) 278 | 279 | 280 | class TestSuite_04_StatisticsWithDotInTubeName(TestSuite_Basic): 281 | @classmethod 282 | def setUpClass(cls): 283 | cls.queue = Queue("127.0.0.1", 33013, 0) 284 | cls.tube = cls.queue.tube("tube.with.dot") 285 | cls.tube.put("task#1") 286 | cls.tube.put("task#2") 287 | cls.tube.put("task#3") 288 | 289 | def test_01_StatisticsOnTubeNotFail(self): 290 | stat = self.tube.statistics() 291 | self.assertTrue(stat['put']) 292 | self.assertTrue(stat['tasks']) 293 | self.assertTrue(stat['tasks']['total']) 294 | 295 | def test_02_StatisticsOnQueueNotFail(self): 296 | stat = self.queue.statistics() 297 | self.assertTrue("tube.with.dot" in stat) 298 | self.assertTrue(stat['tube.with.dot']['put']) 299 | self.assertTrue(stat['tube.with.dot']['tasks']) 300 | self.assertTrue(stat['tube.with.dot']['tasks']['total']) 301 | --------------------------------------------------------------------------------