├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.hacking.rst ├── README.rst ├── docs ├── Makefile ├── _static │ └── .empty ├── conf.py └── index.rst ├── park.py ├── run-tests ├── setup.py ├── test-requirements.txt ├── test_park.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg 3 | *.egg-info 4 | .coverage 5 | .tox 6 | dist/ 7 | docs/_build 8 | tests.* 9 | virtualenv/ 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.4" 4 | - "2.7" 5 | 6 | install: pip install -r test-requirements.txt --use-mirrors 7 | script: sh run-tests 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 litl, LLC. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include run-tests 4 | include test_park.py 5 | include test-requirements.txt 6 | -------------------------------------------------------------------------------- /README.hacking.rst: -------------------------------------------------------------------------------- 1 | Welcome! 2 | 3 | There are three ways the Park unit test suite is run. 4 | 5 | As a *developer*, use ``python setup.py test``. This runs the suite in 6 | the current development directory and does not require a virtualenv to 7 | be set up. 8 | 9 | As a *contributor*, your Github pull requests will be tested on Travis 10 | CI. The configuration for that is in .travis.yml. This runs the test 11 | suite on all supported platforms, which are enumerated in that file. 12 | 13 | As a *release maintainer*, install ``tox`` (in a virtualenv is fine) 14 | and run it in the top level Park source directory. This will build a 15 | source distribution, install that into a virtualenv for each supported 16 | platform, and run the test suite in each. 17 | 18 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://travis-ci.org/litl/park.svg?branch=master :alt: Build Status 2 | .. image:: https://coveralls.io/repos/litl/park/badge.svg?branch=master :alt: Coverage Status 3 | 4 | Park is a persistent key-value API for Python with ordered traversal 5 | of keys. Both keys and values are binary safe. It's similar in use to 6 | LevelDB, but has no dependencies outside the Python standard library. 7 | 8 | It is meant to be extremely easy to use and can scale to a few 9 | gigabytes of data. It allows you to be lazy until it doesn't meet your 10 | needs. Use it until then. 11 | 12 | It supports simple getting and setting of byte data: 13 | 14 | :: 15 | 16 | >>> kv = park.SQLiteStore("numbers.park") 17 | >>> kv.put("1", "one") 18 | >>> kv.put("2", "two") 19 | >>> kv.put("3", "three") 20 | >>> kv.put("4", "four") 21 | 22 | >>> kv.get("2") 23 | 'two' 24 | 25 | Batched setting of data from an iterable: 26 | 27 | :: 28 | 29 | >>> kv.put_many([("1", "one"), ("2", "two"), ("3", "three")]) 30 | 31 | >>> kv.get("3") 32 | 'three' 33 | 34 | Lexically ordered traversal of keys and items, with start and end 35 | sentinels (inclusive): 36 | 37 | :: 38 | 39 | >>> kv.put("1", "one") 40 | >>> kv.put("2", "two") 41 | >>> kv.put("3", "three") 42 | >>> kv.put("11", "eleven") 43 | >>> kv.put("12", "twelve") 44 | 45 | >>> list(kv.keys()) 46 | ['1', '11', '12', '2', '3'] 47 | 48 | >>> list(kv.keys(key_from="12")) 49 | ['12', '2', '3'] 50 | 51 | >>> list(kv.keys(key_from="12", key_to="2")) 52 | ['12', '2'] 53 | 54 | >>> list(kv.items(key_from="12")) 55 | [('12', 'twelve'), ('2', 'two'), ('3', 'three')] 56 | 57 | Iteration over all keys or items with a given prefix: 58 | 59 | :: 60 | 61 | >>> kv.put("pet/dog", "Canis lupus familiaris") 62 | >>> kv.put("pet/cat", "Felis catus") 63 | >>> kv.put("pet/wolf", "Canis lupus") 64 | 65 | >>> list(kv.prefix_keys("pet/")) 66 | ['pet/cat', 'pet/dog', 'pet/wolf'] 67 | 68 | >>> list(kv.prefix_keys("pet/", strip_prefix=True)) 69 | ['cat', 'dog', 'wolf'] 70 | 71 | >>> list(kv.prefix_items("pet/", strip_prefix=True)) 72 | [('cat', 'Felis catus'), 73 | ('dog', 'Canis lupus familiaris'), 74 | ('wolf', 'Canis lupus')] 75 | 76 | It plays well with generators, so you can e.g. park all the counting 77 | numbers (this will take a while): 78 | 79 | :: 80 | 81 | def numbers(): 82 | for num in itertools.count(1): 83 | key = value = str(num) 84 | yield key, value 85 | 86 | kv.put_many(numbers()) 87 | 88 | Or recursively park a directory's contents (keyed by relative paths) 89 | from the local filesystem: 90 | 91 | :: 92 | 93 | def file_item(filename): 94 | with open(filename, "r") as fd: 95 | return filename, fd.read() 96 | 97 | kv.put_many(file_item(os.path.join(root, name)) 98 | for root, dirs, files in os.walk(directory) 99 | for name in files) 100 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Park.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Park.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Park" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Park" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /docs/_static/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/litl/park/85738418b3c1db57046a5b2f217ee3f5d55851df/docs/_static/.empty -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Park documentation build configuration file, created by 4 | # sphinx-quickstart on Sat Sep 1 15:18:11 2012. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.insert(0, os.path.abspath('..')) 20 | 21 | from park import __version__ 22 | 23 | # -- General configuration ----------------------------------------------------- 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be extensions 29 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 30 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] 31 | 32 | # Add any paths that contain templates here, relative to this directory. 33 | templates_path = ['_templates'] 34 | 35 | # The suffix of source filenames. 36 | source_suffix = '.rst' 37 | 38 | # The encoding of source files. 39 | #source_encoding = 'utf-8-sig' 40 | 41 | # The master toctree document. 42 | master_doc = 'index' 43 | 44 | # General information about the project. 45 | project = u'Park' 46 | copyright = u'2012, Peter Teichman' 47 | 48 | # The version info for the project you're documenting, acts as replacement for 49 | # |version| and |release|, also used in various other places throughout the 50 | # built documents. 51 | # 52 | # The short X.Y version. 53 | version = ".".join(__version__.split(".")[:-1]) 54 | # The full version, including alpha/beta/rc tags. 55 | release = __version__ 56 | 57 | # The language for content autogenerated by Sphinx. Refer to documentation 58 | # for a list of supported languages. 59 | #language = None 60 | 61 | # There are two options for replacing |today|: either, you set today to some 62 | # non-false value, then it is used: 63 | #today = '' 64 | # Else, today_fmt is used as the format for a strftime call. 65 | #today_fmt = '%B %d, %Y' 66 | 67 | # List of patterns, relative to source directory, that match files and 68 | # directories to ignore when looking for source files. 69 | exclude_patterns = ['_build'] 70 | 71 | # The reST default role (used for this markup: `text`) to use for all documents. 72 | #default_role = None 73 | 74 | # If true, '()' will be appended to :func: etc. cross-reference text. 75 | #add_function_parentheses = True 76 | 77 | # If true, the current module name will be prepended to all description 78 | # unit titles (such as .. function::). 79 | #add_module_names = True 80 | 81 | # If true, sectionauthor and moduleauthor directives will be shown in the 82 | # output. They are ignored by default. 83 | #show_authors = False 84 | 85 | # The name of the Pygments (syntax highlighting) style to use. 86 | pygments_style = 'sphinx' 87 | 88 | # A list of ignored prefixes for module index sorting. 89 | #modindex_common_prefix = [] 90 | 91 | 92 | # -- Options for HTML output --------------------------------------------------- 93 | 94 | # The theme to use for HTML and HTML Help pages. See the documentation for 95 | # a list of builtin themes. 96 | html_theme = 'default' 97 | 98 | # Theme options are theme-specific and customize the look and feel of a theme 99 | # further. For a list of options available for each theme, see the 100 | # documentation. 101 | #html_theme_options = {} 102 | 103 | # Add any paths that contain custom themes here, relative to this directory. 104 | #html_theme_path = [] 105 | 106 | # The name for this set of Sphinx documents. If None, it defaults to 107 | # " v documentation". 108 | #html_title = None 109 | 110 | # A shorter title for the navigation bar. Default is the same as html_title. 111 | #html_short_title = None 112 | 113 | # The name of an image file (relative to this directory) to place at the top 114 | # of the sidebar. 115 | #html_logo = None 116 | 117 | # The name of an image file (within the static path) to use as favicon of the 118 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 119 | # pixels large. 120 | #html_favicon = None 121 | 122 | # Add any paths that contain custom static files (such as style sheets) here, 123 | # relative to this directory. They are copied after the builtin static files, 124 | # so a file named "default.css" will overwrite the builtin "default.css". 125 | html_static_path = ['_static'] 126 | 127 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 128 | # using the given strftime format. 129 | #html_last_updated_fmt = '%b %d, %Y' 130 | 131 | # If true, SmartyPants will be used to convert quotes and dashes to 132 | # typographically correct entities. 133 | #html_use_smartypants = True 134 | 135 | # Custom sidebar templates, maps document names to template names. 136 | #html_sidebars = {} 137 | 138 | # Additional templates that should be rendered to pages, maps page names to 139 | # template names. 140 | #html_additional_pages = {} 141 | 142 | # If false, no module index is generated. 143 | #html_domain_indices = True 144 | 145 | # If false, no index is generated. 146 | #html_use_index = True 147 | 148 | # If true, the index is split into individual pages for each letter. 149 | #html_split_index = False 150 | 151 | # If true, links to the reST sources are added to the pages. 152 | #html_show_sourcelink = True 153 | 154 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 155 | #html_show_sphinx = True 156 | 157 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 158 | #html_show_copyright = True 159 | 160 | # If true, an OpenSearch description file will be output, and all pages will 161 | # contain a tag referring to it. The value of this option must be the 162 | # base URL from which the finished HTML is served. 163 | #html_use_opensearch = '' 164 | 165 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 166 | #html_file_suffix = None 167 | 168 | # Output file base name for HTML help builder. 169 | htmlhelp_basename = 'Parkdoc' 170 | 171 | 172 | # -- Options for LaTeX output -------------------------------------------------- 173 | 174 | latex_elements = { 175 | # The paper size ('letterpaper' or 'a4paper'). 176 | #'papersize': 'letterpaper', 177 | 178 | # The font size ('10pt', '11pt' or '12pt'). 179 | #'pointsize': '10pt', 180 | 181 | # Additional stuff for the LaTeX preamble. 182 | #'preamble': '', 183 | } 184 | 185 | # Grouping the document tree into LaTeX files. List of tuples 186 | # (source start file, target name, title, author, documentclass [howto/manual]). 187 | latex_documents = [ 188 | ('index', 'Park.tex', u'Park Documentation', 189 | u'Peter Teichman', 'manual'), 190 | ] 191 | 192 | # The name of an image file (relative to this directory) to place at the top of 193 | # the title page. 194 | #latex_logo = None 195 | 196 | # For "manual" documents, if this is true, then toplevel headings are parts, 197 | # not chapters. 198 | #latex_use_parts = False 199 | 200 | # If true, show page references after internal links. 201 | #latex_show_pagerefs = False 202 | 203 | # If true, show URL addresses after external links. 204 | #latex_show_urls = False 205 | 206 | # Documents to append as an appendix to all manuals. 207 | #latex_appendices = [] 208 | 209 | # If false, no module index is generated. 210 | #latex_domain_indices = True 211 | 212 | 213 | # -- Options for manual page output -------------------------------------------- 214 | 215 | # One entry per manual page. List of tuples 216 | # (source start file, name, description, authors, manual section). 217 | man_pages = [ 218 | ('index', 'park', u'Park Documentation', 219 | [u'Peter Teichman'], 1) 220 | ] 221 | 222 | # If true, show URL addresses after external links. 223 | #man_show_urls = False 224 | 225 | 226 | # -- Options for Texinfo output ------------------------------------------------ 227 | 228 | # Grouping the document tree into Texinfo files. List of tuples 229 | # (source start file, target name, title, author, 230 | # dir menu entry, description, category) 231 | texinfo_documents = [ 232 | ('index', 'Park', u'Park Documentation', 233 | u'Peter Teichman', 'Park', 'One line description of project.', 234 | 'Miscellaneous'), 235 | ] 236 | 237 | # Documents to append as an appendix to all manuals. 238 | #texinfo_appendices = [] 239 | 240 | # If false, no module index is generated. 241 | #texinfo_domain_indices = True 242 | 243 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 244 | #texinfo_show_urls = 'footnote' 245 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Park: The Laziest Data Store 2 | ============================ 3 | 4 | .. include:: ../README.rst 5 | 6 | Details, details. 7 | ----------------- 8 | 9 | .. autoclass:: park.SQLiteStore 10 | :members: 11 | 12 | .. autoclass:: park.KVStore 13 | :members: 14 | -------------------------------------------------------------------------------- /park.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright 2012-2015 litl, LLC. All Rights Reserved. 3 | 4 | import abc 5 | import itertools 6 | import logging 7 | import os 8 | import sqlite3 9 | 10 | __version__ = "1.1.0" 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | __all__ = ["SQLiteStore", "KVStore"] 15 | 16 | import sys 17 | if sys.version_info < (3,): 18 | def b(x): 19 | return x 20 | 21 | def un_b(x): 22 | return x 23 | else: 24 | import codecs 25 | 26 | def b(x): 27 | return codecs.latin_1_encode(x)[0] 28 | 29 | def un_b(x): 30 | return codecs.latin_1_decode(x)[0] 31 | 32 | 33 | class KVStore(object): 34 | """An abstract key-value interface with support for range iteration.""" 35 | __metaclass__ = abc.ABCMeta 36 | 37 | # Implement the Python context manager protocol. 38 | def __enter__(self): 39 | return self 40 | 41 | def __exit__(self, *exc_info): 42 | self.close() 43 | 44 | def close(self): # pragma: no cover 45 | """Release any resources associated with a KVStore. 46 | 47 | This is used to support the Python context manager protocol 48 | with semantics similar to ``contextlib.closing()``. That means 49 | you can use any concrete implementation of KVStore like: 50 | 51 | :: 52 | 53 | with park.SQLiteStore("/path/to/db") as kv: 54 | kv.put("my_key", "my_value") 55 | 56 | """ 57 | # Typically overridden by subclasses, this default 58 | # implementation does nothing. 59 | pass 60 | 61 | def contains(self, key): 62 | """True if the store contains key.""" 63 | return self.get(key, default=None) is not None 64 | 65 | @abc.abstractmethod 66 | def get(self, key, default=None): # pragma: no cover 67 | """Get the value associated with a key. 68 | 69 | :param key: The key to retrieve. 70 | :type key: bytes 71 | 72 | :param default: A default value to return if the key is not 73 | present in the store. 74 | 75 | :returns: The value associated with ``key``. 76 | 77 | """ 78 | pass 79 | 80 | @abc.abstractmethod 81 | def put(self, key, value): # pragma: no cover 82 | """Put a key-value pair into the store. 83 | 84 | If the key is already present, this replaces its value. Both 85 | the key and value are binary safe. 86 | 87 | :param key: The key to set. 88 | :type key: bytes 89 | 90 | :param value: The value to set the key to. 91 | :type value: bytes 92 | 93 | """ 94 | pass 95 | 96 | @abc.abstractmethod 97 | def put_many(self, items): # pragma: no cover 98 | """Put many key-value pairs. 99 | 100 | This method may take advantage of performance or atomicity 101 | features of the underlying store. It does not guarantee that 102 | all items will be set in the same transaction, only that 103 | transactions may be used for performance. 104 | 105 | :param items: An iterable producing (key, value) tuples. 106 | 107 | """ 108 | for key, value in items: 109 | self.put(key, value) 110 | 111 | @abc.abstractmethod 112 | def delete(self, key): # pragma: no cover 113 | """Remove a key from the store. 114 | 115 | :param key: The key to remove. 116 | :type key: bytes 117 | 118 | """ 119 | pass 120 | 121 | @abc.abstractmethod 122 | def delete_many(self, keys): # pragma: no cover 123 | """Remove many keys from the store. 124 | 125 | :param keys: An iterable producing keys to remove. 126 | 127 | """ 128 | for key in keys: 129 | self.delete(key) 130 | 131 | @abc.abstractmethod 132 | def keys(self, key_from=None, key_to=None): # pragma: no cover 133 | """Get a lexically sorted range of keys. 134 | 135 | :param key_from: Lower bound (inclusive), or None for unbounded. 136 | :type key_from: bytes 137 | 138 | :param key_to: Upper bound (inclusive), or None for unbounded. 139 | :type key_to: bytes 140 | 141 | :yields: All keys from the store where ``key_from <= key <= key_to``. 142 | 143 | """ 144 | pass 145 | 146 | @abc.abstractmethod 147 | def items(self, key_from=None, key_to=None): # pragma: no cover 148 | """Get a lexically sorted range of (key, value) tuples. 149 | 150 | :param key_from: Lower bound (inclusive), or None for unbounded. 151 | :type key_from: bytes 152 | 153 | :param key_to: Upper bound (inclusive), or None for unbounded. 154 | :type key_to: bytes 155 | 156 | :yields: All (key, value) pairs from the store where 157 | ``key_from <= key <= key_to``. 158 | 159 | """ 160 | pass 161 | 162 | def prefix_items(self, prefix, strip_prefix=False): 163 | """Get all (key, value) pairs with keys that begin with ``prefix``. 164 | 165 | :param prefix: Lexical prefix for keys to search. 166 | :type prefix: bytes 167 | 168 | :param strip_prefix: True to strip the prefix from yielded items. 169 | :type strip_prefix: bool 170 | 171 | :yields: All (key, value) pairs in the store where the keys 172 | begin with the ``prefix``. 173 | 174 | """ 175 | items = self.items(key_from=prefix) 176 | 177 | start = 0 178 | if strip_prefix: 179 | start = len(prefix) 180 | 181 | for key, value in items: 182 | if not key.startswith(prefix): 183 | break 184 | yield key[start:], value 185 | 186 | def prefix_keys(self, prefix, strip_prefix=False): 187 | """Get all keys that begin with ``prefix``. 188 | 189 | :param prefix: Lexical prefix for keys to search. 190 | :type prefix: bytes 191 | 192 | :param strip_prefix: True to strip the prefix from yielded items. 193 | :type strip_prefix: bool 194 | 195 | :yields: All keys in the store that begin with ``prefix``. 196 | 197 | """ 198 | keys = self.keys(key_from=prefix) 199 | 200 | start = 0 201 | if strip_prefix: 202 | start = len(prefix) 203 | 204 | for key in keys: 205 | if not key.startswith(prefix): 206 | break 207 | yield key[start:] 208 | 209 | 210 | def ibatch(iterable, size): 211 | """Yield a series of batches from iterable, each size elements long.""" 212 | source = iter(iterable) 213 | while True: 214 | batch = itertools.islice(source, size) 215 | yield itertools.chain([next(batch)], batch) 216 | 217 | 218 | class SQLiteStore(KVStore): 219 | """A KVStore in an SQLite database. This is what you want to use. 220 | 221 | :param path: The filesystem path for the database, which will be 222 | created if it doesn't exist. 223 | :type path: str 224 | 225 | See :py:class:`park.KVStore` for what you can do with it. 226 | 227 | SQLiteStore uses an embarrassingly simple SQL schema: 228 | 229 | .. code-block:: sql 230 | 231 | CREATE TABLE kv ( 232 | key BLOB NOT NULL PRIMARY KEY, 233 | value BLOB NOT NULL) 234 | 235 | There are a few implications of this schema you might need to be 236 | aware of. 237 | 238 | 1. Declaring ``key`` as PRIMARY KEY automatically indexes it, 239 | which gives constant time ordered traversal of keys and O(log 240 | n) lookup. However, SQLite 3 indexes the keys separately from 241 | the table data, which means your keys are effectively stored 242 | twice in the database. A primary key also means the index can't 243 | be dropped during bulk inserts. 244 | 245 | 2. Using BLOBs for both columns keeps them binary safe, but it 246 | means everything going in must be type ``bytes``. Python 247 | ``str`` strings are converted automatically, but if you're 248 | dealing with Unicode data you'll need to encode it to bytes 249 | first. UTF-8 is a fine option: 250 | 251 | :: 252 | 253 | >>> kv.put("key", value.encode("utf-8")) 254 | >>> kv.get("key").decode("utf-8") 255 | 256 | """ 257 | def __init__(self, path): 258 | need_schema = not os.path.exists(path) 259 | 260 | self.conn = sqlite3.connect(path) 261 | 262 | # Don't create unicode objects for retrieved values 263 | self.conn.text_factory = memoryview 264 | 265 | # Disable the SQLite cache. Its pages tend to get swapped 266 | # out, even if the database file is in buffer cache. 267 | c = self.conn.cursor() 268 | c.execute("PRAGMA cache_size=0") 269 | c.execute("PRAGMA page_size=4096") 270 | 271 | # Use write-ahead logging if it's available, otherwise truncate 272 | journal_mode, = c.execute("PRAGMA journal_mode=WAL").fetchone() 273 | if journal_mode != "wal": 274 | c.execute("PRAGMA journal_mode=truncate") 275 | 276 | # Speed-for-reliability tradeoffs 277 | c.execute("PRAGMA temp_store=memory") 278 | c.execute("PRAGMA synchronous=OFF") 279 | 280 | if need_schema: 281 | self._create_db(self.conn) 282 | 283 | def close(self): 284 | self.conn.commit() 285 | self.conn.close() 286 | del self.conn 287 | 288 | def _create_db(self, conn): 289 | logger.debug("Creating SQLiteStore schema") 290 | c = conn.cursor() 291 | 292 | c.execute(""" 293 | CREATE TABLE kv ( 294 | key BLOB NOT NULL PRIMARY KEY, 295 | value BLOB NOT NULL)""") 296 | 297 | conn.commit() 298 | 299 | def get(self, key, default=None): 300 | q = "SELECT value FROM kv WHERE key = ?" 301 | c = self.conn.cursor() 302 | 303 | row = c.execute(q, (sqlite3.Binary(b(key)),)).fetchone() 304 | if not row: 305 | return default 306 | 307 | return un_b(bytes(row[0])) 308 | 309 | def put(self, key, value): 310 | q = "INSERT OR REPLACE INTO kv (key, value) VALUES (?, ?)" 311 | self.conn.execute(q, 312 | (sqlite3.Binary(b(key)), sqlite3.Binary(b(value)))) 313 | self.conn.commit() 314 | 315 | def put_many(self, items): 316 | q = "INSERT OR REPLACE INTO kv (key, value) VALUES (?, ?)" 317 | c = self.conn.cursor() 318 | 319 | blob = sqlite3.Binary 320 | for batch in ibatch(items, 30000): 321 | items = ((blob(b(key)), blob(b(value))) 322 | for key, value in batch) 323 | 324 | c.executemany(q, items) 325 | self.conn.commit() 326 | 327 | def delete(self, key): 328 | q = "DELETE FROM kv WHERE key = ?" 329 | self.conn.execute(q, (sqlite3.Binary(b(key)),)) 330 | self.conn.commit() 331 | 332 | def delete_many(self, keys): 333 | q = "DELETE FROM kv WHERE key = ?" 334 | c = self.conn.cursor() 335 | 336 | blob = sqlite3.Binary 337 | for batch in ibatch(keys, 30000): 338 | items = ((blob(b(key)),) for key in batch) 339 | 340 | c.executemany(q, items) 341 | self.conn.commit() 342 | 343 | def _range_where(self, key_from=None, key_to=None): 344 | if key_from is not None and key_to is None: 345 | key_from = b(key_from) 346 | return "WHERE key >= :key_from" 347 | 348 | if key_from is None and key_to is not None: 349 | key_to = b(key_to) 350 | return "WHERE key <= :key_to" 351 | 352 | if key_from is not None and key_to is not None: 353 | key_from = b(key_from) 354 | key_to = b(key_to) 355 | return "WHERE key BETWEEN :key_from AND :key_to" 356 | 357 | return "" 358 | 359 | def items(self, key_from=None, key_to=None): 360 | q = "SELECT key, value FROM kv %s ORDER BY key " \ 361 | % self._range_where(key_from, key_to) 362 | 363 | if key_from is not None: 364 | key_from = sqlite3.Binary(b(key_from)) 365 | 366 | if key_to is not None: 367 | key_to = sqlite3.Binary(b(key_to)) 368 | 369 | c = self.conn.cursor() 370 | for key, value in c.execute(q, dict(key_from=key_from, key_to=key_to)): 371 | yield un_b(bytes(key)), un_b(bytes(value)) 372 | 373 | def keys(self, key_from=None, key_to=None): 374 | q = "SELECT key FROM kv %s ORDER BY key " \ 375 | % self._range_where(key_from, key_to) 376 | 377 | if key_from is not None: 378 | key_from = sqlite3.Binary(b(key_from)) 379 | 380 | if key_to is not None: 381 | key_to = sqlite3.Binary(b(key_to)) 382 | 383 | c = self.conn.cursor() 384 | for key, in c.execute(q, dict(key_from=key_from, key_to=key_to)): 385 | yield un_b(bytes(key)) 386 | -------------------------------------------------------------------------------- /run-tests: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Install the dependencies in test-requirements.txt before running 4 | # this script. 5 | 6 | pep8 park.py test_park.py 7 | pyflakes park.py test_park.py 8 | python setup.py test 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import park 5 | 6 | # Require setuptools. See http://pypi.python.org/pypi/setuptools for 7 | # installation instructions, or run the ez_setup script found at 8 | # http://peak.telecommunity.com/dist/ez_setup.py 9 | from setuptools import setup 10 | 11 | # Load the test requirements. These are in a separate file so they can 12 | # be accessed from Travis CI and tox. 13 | with open("test-requirements.txt") as fd: 14 | tests_require = list(fd) 15 | 16 | 17 | setup( 18 | name="park", 19 | version=park.__version__, 20 | author="Peter Teichman", 21 | author_email="pteichman@litl.com", 22 | license="MIT", 23 | url="https://github.com/litl/park", 24 | description="A key-value store with ordered traversal of keys", 25 | py_modules=["park"], 26 | 27 | setup_requires=[ 28 | "unittest2==1.1.0" 29 | ], 30 | 31 | test_suite="unittest2.collector", 32 | tests_require=tests_require, 33 | 34 | classifiers=[ 35 | "Development Status :: 5 - Production/Stable", 36 | "Intended Audience :: Developers", 37 | "License :: OSI Approved :: MIT License", 38 | "Operating System :: OS Independent", 39 | "Programming Language :: Python" 40 | ] 41 | ) 42 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | coverage==4.0 2 | pep8==1.6.2 3 | pyflakes==1.0.0 4 | -------------------------------------------------------------------------------- /test_park.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright 2012-2015 litl, LLC. All Rights Reserved. 3 | 4 | import operator 5 | import os 6 | import unittest2 as unittest 7 | 8 | import park 9 | 10 | 11 | class KVStoreBase(object): 12 | """Base tests for KV Stores""" 13 | 14 | def test_get_default(self): 15 | self.assertIsNone(self.store.get("missing")) 16 | 17 | def test_put_get(self): 18 | key, value = "test_key", "test_value" 19 | 20 | self.assertIsNone(self.store.get(key, None)) 21 | self.store.put(key, value) 22 | 23 | self.assertTrue(self.store.contains(key)) 24 | self.assertEqual(value, self.store.get(key)) 25 | 26 | def test_delete(self): 27 | self.store.put("test_key1", "test_value1") 28 | self.store.put("test_key2", "test_value2") 29 | 30 | self.store.delete("test_key1") 31 | 32 | self.assertEqual(["test_key2"], list(self.store.keys())) 33 | 34 | def test_delete_many(self): 35 | self.store.put("test_key1", "test_value1") 36 | self.store.put("test_key2", "test_value2") 37 | 38 | self.store.delete_many(["test_key1", "test_key2"]) 39 | 40 | self.assertEqual([], list(self.store.keys())) 41 | 42 | def test_null_key(self): 43 | key, value = "\x00", "test_value" 44 | 45 | self.assertIsNone(self.store.get(key, None)) 46 | self.store.put(key, value) 47 | 48 | self.assertEqual(value, self.store.get(key)) 49 | 50 | self.assertEqual([key], list(self.store.keys())) 51 | self.assertEqual([(key, value)], list(self.store.items())) 52 | 53 | def test_null_value(self): 54 | key, value = "test_key", "\x00" 55 | 56 | self.assertIsNone(self.store.get(key, None)) 57 | self.store.put(key, value) 58 | 59 | self.assertEqual(value, self.store.get(key)) 60 | 61 | self.assertEqual([key], list(self.store.keys())) 62 | self.assertEqual([(key, value)], list(self.store.items())) 63 | 64 | def test_replace(self): 65 | key = "foo" 66 | self.assertIsNone(self.store.get(key, None)) 67 | 68 | self.store.put(key, "bar") 69 | self.assertEqual("bar", self.store.get(key)) 70 | 71 | self.store.put(key, "baz") 72 | self.assertEqual("baz", self.store.get(key)) 73 | 74 | def test_put_many(self): 75 | items = [ 76 | ("one", "value1"), 77 | ("two", "value2"), 78 | ("three", "value3"), 79 | ("four", "value4"), 80 | ("five", "value5"), 81 | ("six", "value6"), 82 | ("seven", "value7"), 83 | ("eight", "value8"), 84 | ("nine", "value9") 85 | ] 86 | 87 | self.store.put_many(items) 88 | 89 | for key, value in items: 90 | self.assertEqual(value, self.store.get(key)) 91 | 92 | def test_no_keys(self): 93 | self.assertEqual([], list(self.store.keys())) 94 | 95 | self.assertEqual([], list(self.store.keys(key_from="foo"))) 96 | self.assertEqual([], list(self.store.keys(key_to="bar"))) 97 | 98 | self.assertEqual([], list(self.store.keys(key_from="foo", 99 | key_to="bar"))) 100 | 101 | def test_no_items(self): 102 | self.assertEqual([], list(self.store.items())) 103 | 104 | self.assertEqual([], list(self.store.items(key_from="foo"))) 105 | self.assertEqual([], list(self.store.items(key_to="bar"))) 106 | 107 | self.assertEqual([], list(self.store.items(key_from="foo", 108 | key_to="bar"))) 109 | 110 | def test_keys(self): 111 | items = [ 112 | ("one", "value1"), 113 | ("two", "value2"), 114 | ("three", "value3"), 115 | ("four", "value4"), 116 | ("five", "value5"), 117 | ("six", "value6"), 118 | ("seven", "value7"), 119 | ("eight", "value8"), 120 | ("nine", "value9") 121 | ] 122 | 123 | for key, value in items: 124 | self.store.put(key, value) 125 | 126 | # Sorted order is: eight five four nine one seven six three two 127 | keys = list(self.store.keys()) 128 | expected = "eight five four nine one seven six three two".split() 129 | self.assertEqual(expected, keys) 130 | 131 | # Test key_from on keys that are present and missing in the db 132 | keys = list(self.store.keys(key_from="four")) 133 | expected = "four nine one seven six three two".split() 134 | self.assertEqual(expected, keys) 135 | 136 | keys = list(self.store.keys(key_from="fo")) 137 | expected = "four nine one seven six three two".split() 138 | self.assertEqual(expected, keys) 139 | 140 | # Test key_to 141 | keys = list(self.store.keys(key_to="six")) 142 | expected = "eight five four nine one seven six".split() 143 | self.assertEqual(expected, keys) 144 | 145 | keys = list(self.store.keys(key_to="si")) 146 | expected = "eight five four nine one seven".split() 147 | self.assertEqual(expected, keys) 148 | 149 | # And test them both together 150 | keys = list(self.store.keys(key_from="five", key_to="three")) 151 | expected = "five four nine one seven six three".split() 152 | self.assertEqual(expected, keys) 153 | 154 | def test_prefix_keys(self): 155 | # Fake some interesting keys and values to make sure the 156 | # prefix iterators are working 157 | store = self.store 158 | 159 | store.put("a/", "a") 160 | store.put("a/b", "b") 161 | store.put("a/c", "c") 162 | store.put("a/d", "d") 163 | store.put("a/e", "e") 164 | store.put("a/f", "f") 165 | store.put("b/", "b") 166 | store.put("c/", "c") 167 | store.put("d/", "d") 168 | 169 | a_list = list(store.prefix_keys("a/")) 170 | self.assertEqual("a/ a/b a/c a/d a/e a/f".split(), a_list) 171 | 172 | a_list = list(store.prefix_keys("a/", strip_prefix=True)) 173 | self.assertEqual(["", "b", "c", "d", "e", "f"], a_list) 174 | 175 | self.assertEqual(["b/"], list(store.prefix_keys("b/"))) 176 | self.assertEqual(["c/"], list(store.prefix_keys("c/"))) 177 | self.assertEqual(["d/"], list(store.prefix_keys("d/"))) 178 | 179 | def test_items(self): 180 | put_items = dict([ 181 | ("one", "value1"), 182 | ("two", "value2"), 183 | ("three", "value3"), 184 | ("four", "value4"), 185 | ("five", "value5"), 186 | ("six", "value6"), 187 | ("seven", "value7"), 188 | ("eight", "value8"), 189 | ("nine", "value9") 190 | ]) 191 | 192 | for key, value in list(put_items.items()): 193 | self.store.put(key, value) 194 | 195 | # Sorted order is: eight five four nine one seven six three two 196 | keys = list(self.store.items()) 197 | expected = sorted(list(put_items.items()), key=operator.itemgetter(0)) 198 | self.assertEqual(expected, keys) 199 | 200 | # Test key_from on keys that are present and missing in the db 201 | keys = list(self.store.items(key_from="four")) 202 | self.assertEqual(expected[2:], keys) 203 | 204 | keys = list(self.store.items(key_from="fo")) 205 | self.assertEqual(expected[2:], keys) 206 | 207 | # Test key_to 208 | keys = list(self.store.items(key_to="six")) 209 | self.assertEqual(expected[:7], keys) 210 | 211 | keys = list(self.store.items(key_to="si")) 212 | self.assertEqual(expected[:6], keys) 213 | 214 | # And test them both together 215 | keys = list(self.store.items(key_from="five", key_to="three")) 216 | self.assertEqual(expected[1:8], keys) 217 | 218 | def test_prefix_items(self): 219 | # Fake some interesting keys and values to make sure the 220 | # prefix iterators are working 221 | store = self.store 222 | 223 | store.put("a/", "a") 224 | store.put("a/b", "b") 225 | store.put("a/c", "c") 226 | store.put("a/d", "d") 227 | store.put("a/e", "e") 228 | store.put("a/f", "f") 229 | store.put("b/", "b") 230 | store.put("c/", "c") 231 | store.put("d/", "d") 232 | 233 | expected = [("a/", "a"), 234 | ("a/b", "b"), 235 | ("a/c", "c"), 236 | ("a/d", "d"), 237 | ("a/e", "e"), 238 | ("a/f", "f")] 239 | 240 | a_list = list(store.prefix_items("a/")) 241 | self.assertEqual(expected, a_list) 242 | 243 | expected = [("", "a"), 244 | ("b", "b"), 245 | ("c", "c"), 246 | ("d", "d"), 247 | ("e", "e"), 248 | ("f", "f")] 249 | 250 | a_list = list(store.prefix_items("a/", strip_prefix=True)) 251 | self.assertEqual(expected, a_list) 252 | 253 | def test_context_manager(self): 254 | with self.store as kv: 255 | kv.put("foo", "bar") 256 | kv.put("baz", "quux") 257 | 258 | self.assertEqual("bar", kv.get("foo")) 259 | 260 | 261 | class TestIbatch(unittest.TestCase): 262 | def test_ibatch(self): 263 | items = list(range(10)) 264 | 265 | batches = park.ibatch(items, 3) 266 | 267 | self.assertEqual([0, 1, 2], list(next(batches))) 268 | self.assertEqual([3, 4, 5], list(next(batches))) 269 | self.assertEqual([6, 7, 8], list(next(batches))) 270 | self.assertEqual([9], list(next(batches))) 271 | 272 | 273 | class TestSQLiteStore(unittest.TestCase, KVStoreBase): 274 | DB = "tests.test_sqlite_store" 275 | 276 | def setUp(self): 277 | self.store = park.SQLiteStore(self.DB) 278 | 279 | def cleanup(): 280 | if os.path.exists(self.DB): 281 | os.unlink(self.DB) 282 | 283 | self.addCleanup(cleanup) 284 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py34 3 | 4 | [testenv] 5 | deps= 6 | -r{toxinidir}/test-requirements.txt 7 | 8 | commands= 9 | {toxinidir}/run-tests 10 | --------------------------------------------------------------------------------