├── .gitignore ├── AUTHORS.txt ├── CHANGES.txt ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── docs ├── Makefile ├── conf.py ├── dbm.rst ├── index.rst ├── make.bat └── sshelve.rst ├── setup.cfg ├── setup.py ├── sqlite3dbm ├── __init__.py ├── dbm.py └── sshelve.py └── tests ├── __init__.py ├── dbm_test.py └── sshelve_test.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | 4 | build 5 | dist 6 | sqlite3dbm.egg-info 7 | -------------------------------------------------------------------------------- /AUTHORS.txt: -------------------------------------------------------------------------------- 1 | sqlite3dbm 0.x: 2 | =============== 3 | Primary authors: 4 | 5 | * Jason Fennell 6 | 7 | Contributors (in order of contribution, latest first): 8 | -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | ### v0.1.4-memory, 2011-09-16 -- Bugfix sqlite3 :memory: path 2 | * Allow for :memory: sqlite3 path for testing purposes 3 | 4 | ### v0.1.4, 2011-08-17 -- Bugfix for get_many unicode issue 5 | * get_many() and select() no longer miss unicode keys 6 | 7 | ### v0.1.3, 2011-02-07 -- Improve shelve update/clear performance 8 | * sshelve performance improvements 9 | * update() is faster. Should be preferred over many individual inserts 10 | * clear() is faster 11 | 12 | ### v0.1.2, 2011-01-28 -- Bumb version number 13 | * Do a full minor bump for pypi 14 | 15 | ### v0.1.1a, 2011-01-28 -- Fix MANIFEST 16 | * Fix the MANIFEST.in to include markdown files 17 | 18 | ### v0.1.1, 2011-01-27 -- Final API fixes, Beefing up docs 19 | * Only support bytestring k/v in sqlite3dbm for consistency with dbm 20 | * Make __init__.py better so 'import sqlite3dbm' in the only import needed 21 | * Improve docs: 22 | * Usage examples 23 | * Make README markdown 24 | * Better description on the root page 25 | * Version .gitignore 26 | 27 | ### v0.1.0, 2011-01-24 -- Initial release 28 | * sqlite3-backed dictionary conforming to dbm interface 29 | * shelve subclass to provide serialization for the sqlite3 dbm 30 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2011 Yelp 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | include *.md 3 | include docs/Makefile 4 | recursive-include docs *.rst 5 | prune docs/_build 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | sqlite3dbm 2 | ========== 3 | 4 | Description 5 | =========== 6 | This module provides a sqlite-backed dictionary conforming to the dbm 7 | interface, along with a shelve class that wraps the dict and provides 8 | serialization for it. 9 | 10 | Installation 11 | ============ 12 | * pip install sqlite3dbm 13 | * easy_install sqlite3dbm 14 | 15 | Testing 16 | ======= 17 | * `testify tests` from the root directory 18 | 19 | Links 20 | ===== 21 | * source 22 | * documentation 23 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/sqlite3dbm.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/sqlite3dbm.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/sqlite3dbm" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/sqlite3dbm" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # sqlite3dbm documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Jan 24 18:44:50 2011. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.insert(0, os.path.abspath('..')) 20 | import sqlite3dbm 21 | 22 | # -- General configuration ----------------------------------------------------- 23 | 24 | # If your documentation needs a minimal Sphinx version, state it here. 25 | #needs_sphinx = '1.0' 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be extensions 28 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 29 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] 30 | 31 | # Add any paths that contain templates here, relative to this directory. 32 | templates_path = ['_templates'] 33 | 34 | # The suffix of source filenames. 35 | source_suffix = '.rst' 36 | 37 | # The encoding of source files. 38 | #source_encoding = 'utf-8-sig' 39 | 40 | # The master toctree document. 41 | master_doc = 'index' 42 | 43 | # General information about the project. 44 | project = u'sqlite3dbm' 45 | copyright = u'2011, Yelp' 46 | 47 | # The version info for the project you're documenting, acts as replacement for 48 | # |version| and |release|, also used in various other places throughout the 49 | # built documents. 50 | # 51 | # The short X.Y version. 52 | version = sqlite3dbm.__version__.split('-')[0] 53 | # The full version, including alpha/beta/rc tags. 54 | version = sqlite3dbm.__version__ 55 | 56 | # The language for content autogenerated by Sphinx. Refer to documentation 57 | # for a list of supported languages. 58 | #language = None 59 | 60 | # There are two options for replacing |today|: either, you set today to some 61 | # non-false value, then it is used: 62 | #today = '' 63 | # Else, today_fmt is used as the format for a strftime call. 64 | #today_fmt = '%B %d, %Y' 65 | 66 | # List of patterns, relative to source directory, that match files and 67 | # directories to ignore when looking for source files. 68 | exclude_patterns = ['_build'] 69 | 70 | # The reST default role (used for this markup: `text`) to use for all documents. 71 | #default_role = None 72 | 73 | # If true, '()' will be appended to :func: etc. cross-reference text. 74 | #add_function_parentheses = True 75 | 76 | # If true, the current module name will be prepended to all description 77 | # unit titles (such as .. function::). 78 | #add_module_names = True 79 | 80 | # If true, sectionauthor and moduleauthor directives will be shown in the 81 | # output. They are ignored by default. 82 | #show_authors = False 83 | 84 | # The name of the Pygments (syntax highlighting) style to use. 85 | pygments_style = 'sphinx' 86 | 87 | # A list of ignored prefixes for module index sorting. 88 | #modindex_common_prefix = [] 89 | 90 | 91 | # -- Options for HTML output --------------------------------------------------- 92 | 93 | # The theme to use for HTML and HTML Help pages. See the documentation for 94 | # a list of builtin themes. 95 | html_theme = 'default' 96 | 97 | # Theme options are theme-specific and customize the look and feel of a theme 98 | # further. For a list of options available for each theme, see the 99 | # documentation. 100 | #html_theme_options = {} 101 | 102 | # Add any paths that contain custom themes here, relative to this directory. 103 | #html_theme_path = [] 104 | 105 | # The name for this set of Sphinx documents. If None, it defaults to 106 | # " v documentation". 107 | #html_title = None 108 | 109 | # A shorter title for the navigation bar. Default is the same as html_title. 110 | #html_short_title = None 111 | 112 | # The name of an image file (relative to this directory) to place at the top 113 | # of the sidebar. 114 | #html_logo = None 115 | 116 | # The name of an image file (within the static path) to use as favicon of the 117 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 118 | # pixels large. 119 | #html_favicon = None 120 | 121 | # Add any paths that contain custom static files (such as style sheets) here, 122 | # relative to this directory. They are copied after the builtin static files, 123 | # so a file named "default.css" will overwrite the builtin "default.css". 124 | html_static_path = ['_static'] 125 | 126 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 127 | # using the given strftime format. 128 | #html_last_updated_fmt = '%b %d, %Y' 129 | 130 | # If true, SmartyPants will be used to convert quotes and dashes to 131 | # typographically correct entities. 132 | #html_use_smartypants = True 133 | 134 | # Custom sidebar templates, maps document names to template names. 135 | #html_sidebars = {} 136 | 137 | # Additional templates that should be rendered to pages, maps page names to 138 | # template names. 139 | #html_additional_pages = {} 140 | 141 | # If false, no module index is generated. 142 | #html_domain_indices = True 143 | 144 | # If false, no index is generated. 145 | #html_use_index = True 146 | 147 | # If true, the index is split into individual pages for each letter. 148 | #html_split_index = False 149 | 150 | # If true, links to the reST sources are added to the pages. 151 | html_show_sourcelink = True 152 | 153 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 154 | #html_show_sphinx = True 155 | 156 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 157 | #html_show_copyright = True 158 | 159 | # If true, an OpenSearch description file will be output, and all pages will 160 | # contain a tag referring to it. The value of this option must be the 161 | # base URL from which the finished HTML is served. 162 | #html_use_opensearch = '' 163 | 164 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 165 | #html_file_suffix = None 166 | 167 | # Output file base name for HTML help builder. 168 | htmlhelp_basename = 'sqlite3dbmdoc' 169 | 170 | 171 | # -- Options for LaTeX output -------------------------------------------------- 172 | 173 | # The paper size ('letter' or 'a4'). 174 | #latex_paper_size = 'letter' 175 | 176 | # The font size ('10pt', '11pt' or '12pt'). 177 | #latex_font_size = '10pt' 178 | 179 | # Grouping the document tree into LaTeX files. List of tuples 180 | # (source start file, target name, title, author, documentclass [howto/manual]). 181 | latex_documents = [ 182 | ('index', 'sqlite3dbm.tex', u'sqlite3dbm Documentation', 183 | u'Jason Fennell', 'manual'), 184 | ] 185 | 186 | # The name of an image file (relative to this directory) to place at the top of 187 | # the title page. 188 | #latex_logo = None 189 | 190 | # For "manual" documents, if this is true, then toplevel headings are parts, 191 | # not chapters. 192 | #latex_use_parts = False 193 | 194 | # If true, show page references after internal links. 195 | #latex_show_pagerefs = False 196 | 197 | # If true, show URL addresses after external links. 198 | #latex_show_urls = False 199 | 200 | # Additional stuff for the LaTeX preamble. 201 | #latex_preamble = '' 202 | 203 | # Documents to append as an appendix to all manuals. 204 | #latex_appendices = [] 205 | 206 | # If false, no module index is generated. 207 | #latex_domain_indices = True 208 | 209 | 210 | # -- Options for manual page output -------------------------------------------- 211 | 212 | # One entry per manual page. List of tuples 213 | # (source start file, name, description, authors, manual section). 214 | man_pages = [ 215 | ('index', 'sqlite3dbm', u'sqlite3dbm Documentation', 216 | [u'Jason Fennell'], 1) 217 | ] 218 | 219 | 220 | # Example configuration for intersphinx: refer to the Python standard library. 221 | intersphinx_mapping = {'http://docs.python.org/': None} 222 | -------------------------------------------------------------------------------- /docs/dbm.rst: -------------------------------------------------------------------------------- 1 | :mod:`sqlite3dbm` --- Sqlite-backed dbm 2 | ============================================= 3 | 4 | .. module:: sqlite3dbm.dbm 5 | :synopsis: Sqlite-backed dbm 6 | 7 | .. note:: 8 | Hopefully :mod:`sqlite3dbm.dbm` will be included in Python 3.0, renamed to 9 | :mod:`dbm.sqlite`. 10 | 11 | .. index:: module: dbm 12 | 13 | This module is quite similar to the :mod:`dbm` module, but uses ``sqlite3`` 14 | instead for a backend store. It also provides a small extension to the 15 | traditional dictionary interface. 16 | 17 | Module Interface 18 | ---------------- 19 | The module defines the following constant and function: 20 | 21 | .. exception:: error 22 | 23 | Raised on ``sqlite3dbm``\ -specific errors, such as protection errors. 24 | :exc:`KeyError` is raised for general mapping errors like specifying an 25 | incorrect key. 26 | 27 | Accessible as ``sqlite3dbm.error``. 28 | 29 | 30 | .. function:: open(filename, [flag, [mode]]) 31 | 32 | Open a database and return a ``sqlite3dbm`` object. The 33 | *filename* argument is the path to the database file. 34 | 35 | The optional *flag* argument can be: 36 | 37 | +---------+-------------------------------------------+ 38 | | Value | Meaning | 39 | +=========+===========================================+ 40 | | ``'r'`` | Open existing database for reading only | 41 | | | (default) | 42 | +---------+-------------------------------------------+ 43 | | ``'w'`` | Open existing database for reading and | 44 | | | writing | 45 | +---------+-------------------------------------------+ 46 | | ``'c'`` | Open database for reading and writing, | 47 | | | creating it if it doesn't exist | 48 | +---------+-------------------------------------------+ 49 | | ``'n'`` | Always create a new, empty database, open | 50 | | | for reading and writing | 51 | +---------+-------------------------------------------+ 52 | 53 | The optional *mode* argument is the Unix mode of the file, used only when the 54 | database has to be created. It defaults to octal ``0666`` and respects the 55 | prevailing umask. 56 | 57 | Accessible as ``sqlite3dbm.open``. 58 | 59 | Extended Object Interface 60 | ------------------------- 61 | The underlying object is a ``SqliteMap``. In addition to the standard 62 | dictionary methods, such objects have the following methods: 63 | 64 | .. automethod:: sqlite3dbm.dbm.SqliteMap.__getitem__ 65 | .. automethod:: sqlite3dbm.dbm.SqliteMap.select 66 | .. automethod:: sqlite3dbm.dbm.SqliteMap.get_many 67 | 68 | Usage Example 69 | ------------- 70 | >>> import sqlite3dbm 71 | >>> db = sqlite3dbm.open('mydb.sqlite3', flag='c') 72 | >>> 73 | >>> # Print doesn't work, you need to do .items() 74 | >>> db 75 | 76 | >>> db.items() 77 | [] 78 | >>> 79 | >>> # Acts like a regular dict 80 | >>> db['foo'] = 'bar' 81 | >>> db['foo'] 82 | 'bar' 83 | >>> db.items() 84 | [('foo', 'bar')] 85 | >>> del db['foo'] 86 | >>> db.items() 87 | [] 88 | >>> 89 | >>> # Some extentions that allow for batch reads 90 | >>> db.update({'foo': 'one', 'bar': 'two', 'baz': 'three'}) 91 | >>> db['foo', 'bar'] 92 | ['one', 'two'] 93 | >>> db.select('foo', 'bar') 94 | ['one', 'two'] 95 | >>> db.select('foo', 'bar', 'qux') 96 | Traceback (most recent call last): 97 | File "", line 1, in 98 | File "./sqlite3dbm/dbm.py", line 343, in select 99 | raise KeyError('One of the requested keys is missing!') 100 | KeyError: 'One of the requested keys is missing!' 101 | >>> db.get_many('foo', 'bar', 'qux') 102 | ['one', 'two', None] 103 | >>> db.get_many('foo', 'bar', 'qux', default='') 104 | ['one', 'two', ''] 105 | >>> 106 | >>> # Persistent! 107 | >>> db.items() 108 | [('baz', 'three'), ('foo', 'one'), ('bar', 'two')] 109 | >>> del db 110 | >>> reopened_db = sqlite3dbm.open('mydb.sqlite3') 111 | >>> reopened_db.items() 112 | [('baz', 'three'), ('foo', 'one'), ('bar', 'two')] 113 | >>> 114 | >>> # Be aware that the default flag is 'r' 115 | >>> reopened_db['qux'] = 'four' 116 | Traceback (most recent call last): 117 | File "", line 1, in 118 | File "./sqlite3dbm/dbm.py", line 164, in __setitem__ 119 | raise error('DB is readonly') 120 | sqlite3dbm.dbm.SqliteMapException: DB is readonly 121 | >>> writeable_db = sqlite3dbm.open('mydb.sqlite3', flag='w') # 'c' would be fine too 122 | >>> writeable_db['qux'] = 'four' 123 | >>> reopened_db.items() 124 | [('baz', 'three'), ('foo', 'one'), ('bar', 'two'), ('qux', 'four')] 125 | >>> writeable_db.items() 126 | [('baz', 'three'), ('foo', 'one'), ('bar', 'two'), ('qux', 'four')] 127 | >>> 128 | >>> # Catching sqlite3dbm-specific errors 129 | >>> try: 130 | ... reopened_db['foo'] = 'blargh' 131 | ... except sqlite3dbm.error: 132 | ... print 'Caught a module-specific error' 133 | ... 134 | Caught a module-specific error 135 | 136 | 137 | .. seealso:: 138 | 139 | Module :mod:`dbm` 140 | Standard Unix database interface. 141 | 142 | Module :mod:`gdbm` 143 | Similar interface to the GUNU GDBM library. 144 | 145 | Module :mod:`sqlite3dbm.sshelve` 146 | Extension of :mod:`shelve` for a ``salite3dbm.dbm`` 147 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. sqlite3dbm documentation master file, created by 2 | sphinx-quickstart on Mon Jan 24 18:44:50 2011. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | sqlite3dbm 7 | ========== 8 | 9 | :mod:`sqlite3dbm` provides a sqlite-backed dictionary conforming to the dbm 10 | interface, along with a shelve class that wraps the dict and provides 11 | serialization for it. 12 | 13 | This module was born to provide random-access extra data for Hadoop jobs on 14 | Amazon's Elastic Map Reduce (EMR) cluster. We used to use :mod:`bsddb` for 15 | this because of its dead-simple dict interface. Unfortunately, :mod:`bsddb` is 16 | deprecated for removal from the standard library and also has inter-version 17 | compatability problems that make it not work on EMR. :mod:`sqlite3` is the 18 | obvious alternative for a persistent store, but its powerful SQL interface can 19 | be too complex when you just want a dict. Thus, :mod:`sqlite3dbm` was born to 20 | provide a simple dictionary API on top of the ubiquitous and easily available 21 | :mod:`sqlite3`. 22 | 23 | This module requres no setup or configuration once installed. Its goal is 24 | a stupid-simple solution whenever a persistent dictionary is desired. 25 | 26 | This module also provides a shelve class that allows the storage of arbitrary 27 | objects in the db (the dbm interface only handles raw strings). Using this 28 | interface is also easy: just open your database with 29 | :func:`sqlite3dbm.sshelve.open` instead of :func:`sqlite3dbm.open`. 30 | 31 | 32 | Standard Usage Example 33 | ---------------------- 34 | You have some inital job where you populate the db: 35 | >>> import sqlite3dbm 36 | >>> db = sqlite3dbm.sshelve.open('mydb.sqlite3') 37 | >>> db['foo'] = {'count': 100, 'ctr': .3} 38 | >>> db['bar'] = {'count': 314, 'ctr': .168} 39 | >>> db.items() 40 | [('foo', {'count': 100, 'ctr': 0.29999999999999999}), ('bar', {'count': 314, 'ctr': 0.16800000000000001})] 41 | 42 | Later, you have some other job that needs to use that data: 43 | >>> import sqlite3dbm 44 | >>> db = sqlite3dbm.sshelve.open('mydb.sqlite3') 45 | >>> db['foo']['count'] 46 | 100 47 | >>> db.items() 48 | [('foo', {'count': 100, 'ctr': 0.29999999999999999}), ('bar', {'count': 314, 'ctr': 0.16800000000000001})] 49 | 50 | 51 | Contents 52 | -------- 53 | 54 | .. toctree:: 55 | :maxdepth: 3 56 | :numbered: 57 | 58 | dbm.rst 59 | sshelve.rst 60 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | if NOT "%PAPER%" == "" ( 11 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 12 | ) 13 | 14 | if "%1" == "" goto help 15 | 16 | if "%1" == "help" ( 17 | :help 18 | echo.Please use `make ^` where ^ is one of 19 | echo. html to make standalone HTML files 20 | echo. dirhtml to make HTML files named index.html in directories 21 | echo. singlehtml to make a single large HTML file 22 | echo. pickle to make pickle files 23 | echo. json to make JSON files 24 | echo. htmlhelp to make HTML files and a HTML help project 25 | echo. qthelp to make HTML files and a qthelp project 26 | echo. devhelp to make HTML files and a Devhelp project 27 | echo. epub to make an epub 28 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 29 | echo. text to make text files 30 | echo. man to make manual pages 31 | echo. changes to make an overview over all changed/added/deprecated items 32 | echo. linkcheck to check all external links for integrity 33 | echo. doctest to run all doctests embedded in the documentation if enabled 34 | goto end 35 | ) 36 | 37 | if "%1" == "clean" ( 38 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 39 | del /q /s %BUILDDIR%\* 40 | goto end 41 | ) 42 | 43 | if "%1" == "html" ( 44 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 45 | if errorlevel 1 exit /b 1 46 | echo. 47 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 48 | goto end 49 | ) 50 | 51 | if "%1" == "dirhtml" ( 52 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 53 | if errorlevel 1 exit /b 1 54 | echo. 55 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 56 | goto end 57 | ) 58 | 59 | if "%1" == "singlehtml" ( 60 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 61 | if errorlevel 1 exit /b 1 62 | echo. 63 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 64 | goto end 65 | ) 66 | 67 | if "%1" == "pickle" ( 68 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 69 | if errorlevel 1 exit /b 1 70 | echo. 71 | echo.Build finished; now you can process the pickle files. 72 | goto end 73 | ) 74 | 75 | if "%1" == "json" ( 76 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished; now you can process the JSON files. 80 | goto end 81 | ) 82 | 83 | if "%1" == "htmlhelp" ( 84 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished; now you can run HTML Help Workshop with the ^ 88 | .hhp project file in %BUILDDIR%/htmlhelp. 89 | goto end 90 | ) 91 | 92 | if "%1" == "qthelp" ( 93 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 94 | if errorlevel 1 exit /b 1 95 | echo. 96 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 97 | .qhcp project file in %BUILDDIR%/qthelp, like this: 98 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\sqlite3dbm.qhcp 99 | echo.To view the help file: 100 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\sqlite3dbm.ghc 101 | goto end 102 | ) 103 | 104 | if "%1" == "devhelp" ( 105 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 106 | if errorlevel 1 exit /b 1 107 | echo. 108 | echo.Build finished. 109 | goto end 110 | ) 111 | 112 | if "%1" == "epub" ( 113 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 117 | goto end 118 | ) 119 | 120 | if "%1" == "latex" ( 121 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 122 | if errorlevel 1 exit /b 1 123 | echo. 124 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 125 | goto end 126 | ) 127 | 128 | if "%1" == "text" ( 129 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 130 | if errorlevel 1 exit /b 1 131 | echo. 132 | echo.Build finished. The text files are in %BUILDDIR%/text. 133 | goto end 134 | ) 135 | 136 | if "%1" == "man" ( 137 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 141 | goto end 142 | ) 143 | 144 | if "%1" == "changes" ( 145 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.The overview file is in %BUILDDIR%/changes. 149 | goto end 150 | ) 151 | 152 | if "%1" == "linkcheck" ( 153 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Link check complete; look for any errors in the above output ^ 157 | or in %BUILDDIR%/linkcheck/output.txt. 158 | goto end 159 | ) 160 | 161 | if "%1" == "doctest" ( 162 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 163 | if errorlevel 1 exit /b 1 164 | echo. 165 | echo.Testing of doctests in the sources finished, look at the ^ 166 | results in %BUILDDIR%/doctest/output.txt. 167 | goto end 168 | ) 169 | 170 | :end 171 | -------------------------------------------------------------------------------- /docs/sshelve.rst: -------------------------------------------------------------------------------- 1 | :mod:`sqlite3dbm.sshelve` --- Shelve extention for a ``sqlite3dbm.dbm`` object 2 | ============================================================================== 3 | 4 | .. module:: sqlite3dbm.sshelve 5 | :synopsis: Shelve extention for a ``sqlite3dbm.dbm`` object 6 | 7 | .. index:: module: shelve 8 | 9 | This module provides a subclass of :class:`shelve.Shelf` that works for a 10 | ``sqlite3dbm.dbm`` object. See the documentation for :class:`shelve.Shelf` to 11 | see the context this module fits into. 12 | 13 | Module Contents 14 | ---------------- 15 | 16 | .. function:: open(filename[, flag='c' [, mode=0666[, protocol=None[, writeback=False]]]]) 17 | 18 | Open a persistent :mod:`sqlite3`-backed dictionary. The *filename* specificed is the 19 | path to the underlying database. 20 | 21 | The *flag* and *mode* parameters have the same semantics as 22 | :func:`sqlite3dbm.dbm.open` (and, in fact, are directly passed through to 23 | this function). 24 | 25 | The *protocl* and *writeback* parameters behave as outlined in :func:`shelve.open`. 26 | 27 | .. class:: sqlite3dbm.sshelve.SqliteMapShelf 28 | 29 | A subclass of :class:`shelve.Shelf` supporting :mod:`sqlite3dbm.dbm`. 30 | 31 | Exposes :func:`~sqlite3dbm.dbm.SqliteMap.select` and 32 | :func:`~sqlite3dbm.dbm.SqliteMap.get_many` which are available in 33 | :mod:`sqlite3dbm.dbm` but none of the other database modules. The dict 34 | object passed to the constructor must support these methods, which is 35 | generally done by calling :func:`sqlite3dbm.dbm.open`. 36 | 37 | The optional `protocol` and `writeback` parameters behave the same as 38 | they do for :class:`shelve.Shelf`. 39 | 40 | Usage Example 41 | ------------- 42 | >>> import sqlite3dbm 43 | >>> db = sqlite3dbm.sshelve.open('mydb.sqlite3') 44 | >>> db['foo'] = 'bar' 45 | >>> db['baz'] = [1, 2, 3] 46 | >>> db['baz'] 47 | [1, 2, 3] 48 | >>> db.select('foo', 'baz') 49 | ['bar', [1, 2, 3]] 50 | >>> db.get_many('foo', 'qux', default='') 51 | ['bar', ''] 52 | 53 | .. seealso:: 54 | 55 | Module :mod:`shelve` 56 | General object persistence build on top of ``dbm`` interfaces. 57 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [build_sphinx] 2 | source-dir = docs/ 3 | build-dir = docs/_build 4 | all_files = 1 5 | 6 | [upload_sphinx] 7 | upload-dir = docs/_build/html 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from sqlite3dbm import __version__ 2 | 3 | import setuptools 4 | setuptools.setup( 5 | author='Jason Fennell', 6 | author_email='jfennell@yelp.com', 7 | classifiers=[ 8 | 'Development Status :: 4 - Beta', 9 | 'Intended Audience :: Developers', 10 | 'License :: OSI Approved :: Apache Software License', 11 | 'Natural Language :: English', 12 | 'Operating System :: OS Independent', 13 | 'Programming Language :: Python', 14 | 'Programming Language :: Python :: 2.5', 15 | 'Programming Language :: Python :: 2.6', 16 | 'Programming Language :: Python :: 2.7', 17 | 'Topic :: Database :: Database Engines/Servers', 18 | 'Topic :: Software Development :: Libraries :: Python Modules', 19 | ], 20 | description='sqlite-backed dictionary', 21 | license='Apache', 22 | long_description=open('README.md').read(), 23 | name='sqlite3dbm', 24 | packages=['sqlite3dbm'], 25 | provides=['sqlite3dbm'], 26 | url='http://github.com/Yelp/sqlite3dbm/', 27 | version=__version__, 28 | ) 29 | -------------------------------------------------------------------------------- /sqlite3dbm/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Yelp 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """This module provides a sqlite-backed dictionary conforming to the dbm 16 | interface, along with a shelve class that wraps the dict and provides 17 | serialization for it. 18 | """ 19 | 20 | __author__ = 'Jason Fennell ' 21 | __version__ = '0.1.4-memory' 22 | 23 | import sqlite3dbm.dbm as dbm 24 | import sqlite3dbm.sshelve as sshelve 25 | 26 | # Expose `open` and `error` here 27 | from sqlite3dbm.dbm import * 28 | -------------------------------------------------------------------------------- /sqlite3dbm/dbm.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Yelp 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """A SQLite-backed dictionary that respects the dbm interface. 16 | 17 | This module is quite similar to the `dbm` module, but uses `sqlite3` instead 18 | for a backend store. It also provides a small extension to the traditional 19 | dictionary interface. 20 | 21 | Extended Interface: 22 | * __getitem__: Like normal __getitem__ but also works on lists. 23 | * select: List-based version of __getitem__. Complement of update() 24 | * get_many: get() and select() combined. 25 | 26 | For additional docs, see the help()/pydoc for 27 | sqlite3dbm.dbm.SqliteMap., ie. sqlite3dbm.dbm.SqliteMap.select 28 | 29 | Usage Example: 30 | >>> import sqlite3dbm 31 | >>> db = sqlite3dbm.open('mydb.sqlite3', flag='c') 32 | >>> 33 | >>> # Print doesn't work, you need to do .items() 34 | >>> db 35 | 36 | >>> db.items() 37 | [] 38 | >>> 39 | >>> # Acts like a regular dict, but only for bytestrings 40 | >>> db['foo'] = 'bar' 41 | >>> db['foo'] 42 | 'bar' 43 | >>> db.items() 44 | [('foo', 'bar')] 45 | >>> del db['foo'] 46 | >>> db.items() 47 | [] 48 | >>> 49 | >>> # Some extentions that allow for batch reads 50 | >>> db.update({'foo': 'one', 'bar': 'two', 'baz': 'three'}) 51 | >>> db['foo', 'bar'] 52 | ['one', 'two'] 53 | >>> db.select('foo', 'bar') 54 | ['one', 'two'] 55 | >>> db.select('foo', 'bar', 'qux') 56 | Traceback (most recent call last): 57 | File "", line 1, in 58 | File "./sqlite3dbm/dbm.py", line 343, in select 59 | raise KeyError('One of the requested keys is missing!') 60 | KeyError: 'One of the requested keys is missing!' 61 | >>> db.get_many('foo', 'bar', 'qux') 62 | ['one', 'two', None] 63 | >>> db.get_many('foo', 'bar', 'qux', default='') 64 | ['one', 'two', ''] 65 | >>> 66 | >>> # Persistent! 67 | >>> db.items() 68 | [('baz', 'three'), ('foo', 'one'), ('bar', 'two')] 69 | >>> del db 70 | >>> reopened_db = sqlite3dbm.open('mydb.sqlite3') 71 | >>> reopened_db.items() 72 | [('baz', 'three'), ('foo', 'one'), ('bar', 'two')] 73 | >>> 74 | >>> # Be aware that the default flag is 'r' 75 | >>> reopened_db['qux'] = 'four' 76 | Traceback (most recent call last): 77 | File "", line 1, in 78 | File "./sqlite3dbm/dbm.py", line 164, in __setitem__ 79 | raise error('DB is readonly') 80 | sqlite3dbm.dbm.SqliteMapException: DB is readonly 81 | >>> writeable_db = sqlite3dbm.open('mydb.sqlite3', flag='w') # 'c' would be fine too 82 | >>> writeable_db['qux'] = 'four' 83 | >>> reopened_db.items() 84 | [('baz', 'three'), ('foo', 'one'), ('bar', 'two'), ('qux', 'four')] 85 | >>> writeable_db.items() 86 | [('baz', 'three'), ('foo', 'one'), ('bar', 'two'), ('qux', 'four')] 87 | >>> 88 | >>> # Catching sqlite3dbm-specific errors 89 | >>> try: 90 | ... reopened_db['foo'] = 'blargh' 91 | ... except sqlite3dbm.error: 92 | ... print 'Caught a module-specific error' 93 | ... 94 | Caught a module-specific error 95 | """ 96 | 97 | from __future__ import with_statement 98 | 99 | import os 100 | import sqlite3 101 | 102 | __all__ = [ 103 | 'open', 104 | 'error', 105 | ] 106 | 107 | # Maximum number of bindable parameters in a SQLite query 108 | SQLITE_MAX_QUERY_VARS = 999 109 | 110 | # Unique sentinel that we can do pointer comparisons against to check if an 111 | # optional kwarg has been supplied to pop. 112 | __POP_SENTINEL__ = ('__pop__',) 113 | 114 | # Unique sentinel that we can use to distinguish missing values from None 115 | # values in `select` 116 | __MISSING_SENTINEL__ = ('__missing__',) 117 | 118 | def _utf8(s): 119 | """Guarantee that the return value is a utf-8 encoded string.""" 120 | if isinstance(s, unicode): 121 | return s.encode('utf-8') 122 | assert isinstance(s, str) 123 | return s 124 | 125 | ## Pre-compile all queries as raw SQL for speed and 126 | ## to avoid outside dependencies 127 | 128 | _GET_QUERY = 'SELECT kv_table.val FROM kv_table WHERE kv_table.key = ?' 129 | _GET_ALL_QUERY = 'SELECT kv_table.key, kv_table.val FROM kv_table' 130 | _GET_ONE_QUERY = 'SELECT kv_table.key, kv_table.val FROM kv_table LIMIT 1 OFFSET 0' 131 | 132 | # The get-many query generation is slightly unfortunate in that sqlite does not 133 | # seem to have an interface for binding a list of values into a query. Thus, 134 | # we must generate a query string with the right number of missing parameter 135 | # values for the particular select we want to do 136 | _GET_MANY_QUERY_TEMPLATE = ( 137 | 'SELECT kv_table.key, kv_table.val FROM kv_table ' 138 | 'WHERE kv_table.key IN (%s)' 139 | ) 140 | def get_many_query(num_keys): 141 | # Cache the super-big query, as it may happen many times 142 | # through big select/get_many calls 143 | if (num_keys == SQLITE_MAX_QUERY_VARS and 144 | hasattr(get_many_query, '_big_query_cache')): 145 | return get_many_query._big_query_cache 146 | 147 | interpolation_params = ','.join('?' * num_keys) 148 | tmpl = _GET_MANY_QUERY_TEMPLATE % interpolation_params 149 | 150 | if num_keys == SQLITE_MAX_QUERY_VARS: 151 | get_many_query._big_query_cache = tmpl 152 | 153 | return tmpl 154 | 155 | # Do INSERT OR REPLACE instead of a vanilla INSERT 156 | # to mimic normal dict overwrite-on-insert behavior 157 | _SET_QUERY = 'INSERT OR REPLACE INTO kv_table (key, val) VALUES (?, ?)' 158 | 159 | _DEL_QUERY = 'DELETE FROM kv_table WHERE kv_table.key = ?' 160 | _CLEAR_QUERY = 'DELETE FROM kv_table; VACUUM;' 161 | 162 | _COUNT_QUERY = 'SELECT COUNT(*) FROM kv_table' 163 | 164 | # Table has a String key, which puts an upper bound on the 165 | # size of keys that can be inserted. The Text format of 166 | # the values should be fine in general, but could be optimized 167 | # if we were storing ints/floats. Also, should probably 168 | # be changed to Blob if we use a binary serialization format. 169 | # 170 | # TODO: Make this more configurable re the above comment 171 | # Use TEXT for the keys now... maybe want to limit it for better 172 | # indexing? Should do some performance profiling... 173 | _CREATE_TABLE = ( 174 | 'CREATE TABLE IF NOT EXISTS ' 175 | 'kv_table (key TEXT PRIMARY KEY, val TEXT)' 176 | ) 177 | 178 | class SqliteMapException(Exception): 179 | """Raised on module-specific errors, such as protection errors. 180 | 181 | KeyError is raised for general mapping errors like specifying an incorrect 182 | key. 183 | """ 184 | pass 185 | # DBM interface 186 | error = SqliteMapException 187 | 188 | 189 | class SqliteMap(object): 190 | """Dictionary interface backed by a SQLite DB. 191 | 192 | This dictionary only accepts string key/values. 193 | 194 | This is not remotely threadsafe. 195 | """ 196 | 197 | def __init__(self, path, flag='r', mode=0666): 198 | """Create an dict backed by a SQLite DB at `sqlite_db_path`. 199 | 200 | See `open` for explanation of the parameters. 201 | """ 202 | 203 | if flag not in ('c', 'n', 'w', 'r'): 204 | raise error('Invalid flag "%s"' % (flag,)) 205 | 206 | # Default behavior is to create if the file does not already exist. 207 | # We tweak from this default behavior to accommodate the other flag options 208 | 209 | self.readonly = flag == 'r' 210 | 211 | # Allow for :memory: sqlite3 path for testing purposes 212 | if path != ':memory:': 213 | # Need an absolute path to db on the filesystem 214 | path = os.path.abspath(path) 215 | 216 | # r and w require the db to exist ahead of time 217 | if not os.path.exists(path): 218 | if flag in ('r', 'w'): 219 | raise error('DB does not exist at %s' % (path,)) 220 | else: 221 | # Ghetto way of respecting mode, since unexposed by sqlite3.connect 222 | # Manually create the file before sqlite3 connects to it 223 | os.open(path, os.O_CREAT, mode) 224 | 225 | self.conn = sqlite3.connect(path) 226 | self.conn.text_factory = str 227 | self.conn.execute(_CREATE_TABLE) 228 | 229 | # n option requires us to clear out existing data 230 | if flag == 'n': 231 | self.clear() 232 | 233 | def __setitem__(self, k, v): 234 | """x.__setitem__(k, v) <==> x[k] = v""" 235 | if self.readonly: 236 | raise error('DB is readonly') 237 | 238 | self.conn.execute(_SET_QUERY, (k, v)) 239 | self.conn.commit() 240 | 241 | def __getitem__(self, k): 242 | """x.__getitem__(k) <==> x[k] 243 | 244 | This version of :meth:`__getitem__` also transparently works on lists: 245 | >>> smap.update({'1': 'a', '2': 'b', '3': 'c'}) 246 | >>> smap['1', '2', '3'] 247 | [u'a', u'b', u'c'] 248 | >>> smap[['1', '2', '3']] 249 | [u'a', u'b', u'c'] 250 | """ 251 | if hasattr(k, '__iter__'): 252 | return self.select(k) 253 | 254 | row = self.conn.execute(_GET_QUERY, (k,)).fetchone() 255 | if row is None: 256 | raise KeyError(k) 257 | return row[0] 258 | 259 | def __delitem__(self, k): 260 | """x.__delitem__(k) <==> del x[k]""" 261 | if self.readonly: 262 | raise error('DB is readonly') 263 | 264 | # So, the delete actually has no problem running when 265 | # the key k was not present in the map. Unfortunately, 266 | # this does not conform to the dict interface so we 267 | # do a __getitem__ here to make sure a KeyError gets 268 | # thrown when it should. I think this is dumb :-P 269 | self[k] 270 | 271 | self.conn.execute(_DEL_QUERY, (k,)) 272 | self.conn.commit() 273 | 274 | def __contains__(self, k): 275 | """D.__contains__(k) -> True if D has a key k, else False""" 276 | try: 277 | self[k] 278 | except KeyError: 279 | return False 280 | else: 281 | return True 282 | 283 | def clear(self): 284 | """D.clear() -> None. Remove all items from D.""" 285 | if self.readonly: 286 | raise error('DB is readonly') 287 | 288 | self.conn.executescript(_CLEAR_QUERY) 289 | self.conn.commit() 290 | 291 | def get(self, k, d=None): 292 | """D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.""" 293 | try: 294 | return self[k] 295 | except KeyError: 296 | return d 297 | 298 | def has_key(self, k): 299 | """D.has_key(k) -> True if D has a key k, else False.""" 300 | return k in self 301 | 302 | def pop(self, k, d=__POP_SENTINEL__): 303 | """D.pop(k[,d]) -> v, remove specified key and return the corresponding value. 304 | If key is not found, d is returned if given, otherwise KeyError is raised. 305 | """ 306 | if self.readonly: 307 | raise error('DB is readonly') 308 | 309 | try: 310 | val = self[k] 311 | del self[k] 312 | return val 313 | except KeyError: 314 | if d is __POP_SENTINEL__: 315 | raise KeyError(k) 316 | else: 317 | return d 318 | 319 | def popitem(self): 320 | """D.popitem() -> (k, v), remove and return some (key, value) pair as a 321 | 2-tuple; but raise KeyError if D is empty 322 | """ 323 | if self.readonly: 324 | raise error('DB is readonly') 325 | 326 | rows = [row for row in self.conn.execute(_GET_ONE_QUERY)] 327 | if len(rows) != 1: 328 | raise KeyError( 329 | 'Found %d rows when there should have been 1' % (len(rows),) 330 | ) 331 | 332 | key, val = rows[0] 333 | del self[key] 334 | return key, val 335 | 336 | def setdefault(self, k, d=None): 337 | """D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D""" 338 | if self.readonly: 339 | raise error('DB is readonly') 340 | 341 | if k in self: 342 | return self[k] 343 | else: 344 | self[k] = d 345 | return d 346 | 347 | def get_many(self, *args, **kwargs): 348 | """Basically :meth:`~sqlite3dbm.dbm.SqliteMap.get` 349 | and :meth:`~sqlite3dbm.dbm.SqliteMap.select` combined. 350 | 351 | The interface is the same as :meth:`~sqlite3dbm.dbm.SqliteMap.select` 352 | except for the additional option argument `default`. This argument 353 | specifies what value should be used for keys that are not present in 354 | the dict. 355 | """ 356 | default = kwargs.pop('default', None) 357 | if kwargs: 358 | raise TypeError( 359 | 'Got an unexpected keyword argument: %r' % (kwargs,) 360 | ) 361 | 362 | def k_gen(): 363 | """Generator to make iterating over args easy.""" 364 | for arg in args: 365 | if hasattr(arg, '__iter__'): 366 | for k in arg: 367 | yield k 368 | else: 369 | yield arg 370 | 371 | def lookup(keys): 372 | """Reuse the slightly weird logic to lookup values""" 373 | # Do all the selects in a single transaction 374 | key_to_val = dict(self.conn.execute(get_many_query(len(keys)), keys)) 375 | 376 | # Need to do this whole map lookup thing because the 377 | # select does not have a return order. 378 | # 379 | # We force the keys to be utf8 because that is what sqlite3 380 | # gives us back from the cursor. 381 | return (key_to_val.get(_utf8(key), default) for key in keys) 382 | 383 | keys = [] 384 | result = [] 385 | for k in k_gen(): 386 | if len(keys) < SQLITE_MAX_QUERY_VARS: 387 | keys.append(k) 388 | else: 389 | result.extend(lookup(keys)) 390 | keys = [k] 391 | if len(keys) > 0: 392 | result.extend(lookup(keys)) 393 | 394 | return result 395 | 396 | def select(self, *args): 397 | """List based version of :meth:`__getitem__`. Complement of :meth:`~sqlite3dbm.dbm.SqliteMap.update`. 398 | 399 | `args` are the keys to retrieve from the dict. All of the following work: 400 | >>> smap.update({'1': 'a', '2': 'b', '3': 'c'}) 401 | >>> smap.select('1', '2', '3') 402 | [u'a', u'b', u'c'] 403 | >>> smap.select(['1', '2', '3']) 404 | [u'a', u'b', u'c'] 405 | >>> smap.select(['1', '2'], '3') 406 | [u'a', u'b', u'c'] 407 | >>> smap.select(['1', '2'], ['3']) 408 | [u'a', u'b', u'c'] 409 | 410 | Returns: 411 | List of values corresponding to the requested keys in order 412 | 413 | Raises: 414 | KeyError if any of the keys are missing 415 | """ 416 | vals = self.get_many(default=__MISSING_SENTINEL__, *args) 417 | if __MISSING_SENTINEL__ in vals: 418 | raise KeyError('One of the requested keys is missing!') 419 | return vals 420 | 421 | def update(self, *args, **kwargs): 422 | """D.update(E, **F) -> None. Update D from E and F: for k in E: D[k] = E[k] 423 | (if E has keys else: for (k, v) in E: D[k] = v) then: for k in F: D[k] = F[k] 424 | """ 425 | if self.readonly: 426 | raise error('DB is readonly') 427 | 428 | def kv_gen(): 429 | """Generator that combines all the args for easy iteration.""" 430 | for arg in args: 431 | if isinstance(arg, dict): 432 | for k, v in arg.iteritems(): 433 | yield k, v 434 | else: 435 | for k, v in arg: 436 | yield k, v 437 | 438 | for k, v in kwargs.iteritems(): 439 | yield k, v 440 | rows = list(kv_gen()) 441 | 442 | # Do all the inserts in a single transaction for the sake of efficiency 443 | # TODO: Compare preformance of INSERT MANY to many INSERTS. Will 444 | # have to do it in blocks to not exceed query-size limits 445 | self.conn.executemany(_SET_QUERY, rows) 446 | self.conn.commit() 447 | 448 | def __len__(self): 449 | """x.__len__() <==> len(x)""" 450 | return self.conn.execute(_COUNT_QUERY).fetchone()[0] 451 | 452 | ## Iteration 453 | def iteritems(self): 454 | """D.iteritems() -> an iterator over the (key, value) items of D""" 455 | for key, val in self.conn.execute(_GET_ALL_QUERY): 456 | yield key, val 457 | 458 | def items(self): 459 | """D.items() -> list of D's (key, value) pairs, as 2-tuples""" 460 | return [(k, v) for k, v in self.iteritems()] 461 | def iterkeys(self): 462 | """D.iterkeys() -> an iterator over the keys of D""" 463 | return (k for k, _ in self.iteritems()) 464 | def keys(self): 465 | """D.iterkeys() -> an iterator over the keys of D""" 466 | return [k for k in self.iterkeys()] 467 | def itervalues(self): 468 | """D.itervalues() -> an iterator over the values of D""" 469 | return (v for _, v in self.iteritems()) 470 | def values(self): 471 | """D.values() -> list of D's values""" 472 | return [v for v in self.itervalues()] 473 | def __iter__(self): 474 | """Iterate over the keys of D. Consistent with dict.""" 475 | return self.iterkeys() 476 | 477 | def open(filename, flag='r', mode=0666): 478 | """Open a database and return a SqliteMap object. 479 | 480 | The `filename` argument is the path to the database file. 481 | 482 | The optional `flag` argument can be: 483 | r: Open existing db for reading only [default] 484 | w: Open existing db read/write 485 | c: Open db read/write, creating if it doesn't exist 486 | n: Open new and emtpy db read/write 487 | 488 | The optional `mode` argument is the Unix mode of the file, used only when 489 | the database has to be created. It defaults to octal 0666 and respects the 490 | prevailing umask. 491 | """ 492 | return SqliteMap(filename, flag=flag, mode=mode) 493 | -------------------------------------------------------------------------------- /sqlite3dbm/sshelve.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Yelp 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """shelve wrapper for a SqliteMap 16 | 17 | Usage Example: 18 | >>> import sqlite3dbm 19 | >>> db = sqlite3dbm.sshelve.open('mydb.sqlite3') 20 | >>> db['foo'] = 'bar' 21 | >>> db['baz'] = [1, 2, 3] 22 | >>> db['baz'] 23 | [1, 2, 3] 24 | >>> db.select('foo', 'baz') 25 | ['bar', [1, 2, 3]] 26 | >>> db.get_many('foo', 'qux', default='') 27 | ['bar', ''] 28 | """ 29 | 30 | import shelve 31 | # Try using cPickle and cStringIO if available. 32 | try: 33 | from cPickle import loads, dumps 34 | except ImportError: 35 | from pickle import loads, dumps 36 | 37 | import sqlite3dbm.dbm 38 | 39 | __all__ = [ 40 | 'SqliteMapShelf', 41 | 'open', 42 | ] 43 | 44 | class SqliteMapShelf(shelve.Shelf): 45 | """A subclass of shelve.Shelf supporting sqlite3dbm. 46 | 47 | Exposes select() and get_many() which are available in sqlite3dbm but none 48 | of the other database modules. The dict object passed to the constructor 49 | must support these methods, which is generally done by calling 50 | sqlite3dbm.open. 51 | 52 | The optional `protocol` and `writeback` parameters behave the same as 53 | they do for shelve.Shelf. 54 | """ 55 | 56 | def __init__(self, smap, protocol=None, writeback=False): 57 | # Force the Sqlite DB to return bytestrings. By default it returns 58 | # unicode by default, which causes Pickle to shit its pants. 59 | smap.conn.text_factory = str 60 | 61 | # SqliteMapShelf < Shelf < DictMixin which is an old style class :-P 62 | shelve.Shelf.__init__(self, smap, protocol, writeback) 63 | 64 | def get_many(self, *args, **kwargs): 65 | # Pickle 'default' for consistency when we de-pickle 66 | default = dumps(kwargs.get('default')) 67 | kwargs['default'] = default 68 | 69 | return [ 70 | loads(v) 71 | for v in self.dict.get_many(*args, **kwargs) 72 | ] 73 | 74 | def select(self, *args): 75 | return [ 76 | loads(v) 77 | for v in self.dict.select(*args) 78 | ] 79 | 80 | # Performance override: we want to batch writes into one transaction 81 | def update(self, *args, **kwargs): 82 | # Copied from sqlite3dbm.dbm 83 | def kv_gen(): 84 | """Generator that combines all the args for easy iteration.""" 85 | for arg in args: 86 | if isinstance(arg, dict): 87 | for k, v in arg.iteritems(): 88 | yield k, v 89 | else: 90 | for k, v in arg: 91 | yield k, v 92 | 93 | for k, v in kwargs.iteritems(): 94 | yield k, v 95 | inserts = list(kv_gen()) 96 | 97 | if self.writeback: 98 | self.cache.update(inserts) 99 | 100 | self.dict.update([ 101 | (k, dumps(v, protocol=self._protocol)) 102 | for k, v in inserts 103 | ]) 104 | 105 | # Performance override: clear in one sqlite command 106 | def clear(self): 107 | self.dict.clear() 108 | 109 | def open(filename, flag='c', mode=0666, protocol=None, writeback=False): 110 | """Open a persistent sqlite3-backed dictionary. The *filename* specificed 111 | is the path to the underlying database. 112 | 113 | The *flag* and *mode* parameters have the same semantics as sqlite3dbm.open 114 | (and, in fact, are directly passed through to this function). 115 | 116 | The *protocl* and *writeback* parameters behave as outlined in shelve.open. 117 | """ 118 | smap = sqlite3dbm.dbm.open(filename, flag=flag, mode=mode) 119 | return SqliteMapShelf(smap, protocol=protocol, writeback=writeback) 120 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Yelp 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/dbm_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2011 Yelp 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Test the sqlite3dbm module""" 17 | 18 | import os 19 | import shutil 20 | import stat 21 | import tempfile 22 | 23 | import testify 24 | 25 | import sqlite3dbm.dbm 26 | 27 | class SqliteMapTestCase(testify.TestCase): 28 | """Some common functionality for SqliteMap test cases""" 29 | 30 | @testify.setup 31 | def create_map(self): 32 | self.tmpdir = tempfile.mkdtemp() 33 | self.path = os.path.join(self.tmpdir, 'sqlite_map_test_db.sqlite') 34 | self.smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='c') 35 | 36 | @testify.teardown 37 | def teardown_map(self): 38 | shutil.rmtree(self.tmpdir) 39 | 40 | def prepopulate_map_test(self, d, smap): 41 | for k, v in d.iteritems(): 42 | smap[k] = v 43 | 44 | 45 | class TestSqliteMapInterface(SqliteMapTestCase): 46 | """Test the dictionary interface of the SqliteMap""" 47 | 48 | def test_setitem(self): 49 | self.smap['darwin'] = 'drools' 50 | 51 | cursor = self.smap.conn.cursor() 52 | cursor.execute('select * from kv_table') 53 | rows = list(cursor.fetchall()) 54 | 55 | testify.assert_equal(len(rows), 1) 56 | k, v = rows[0] 57 | testify.assert_equal(k, 'darwin') 58 | testify.assert_equal(v, 'drools') 59 | 60 | def test_getitem(self): 61 | self.smap['jugglers'] = 'awesomesauce' 62 | testify.assert_equal(self.smap['jugglers'], 'awesomesauce') 63 | testify.assert_raises( 64 | KeyError, 65 | lambda: self.smap['unicyclers'] 66 | ) 67 | 68 | def test_getitem_tuple(self): 69 | self.smap.update({ 70 | 'jason': 'fennell', 71 | 'dave': 'marin', 72 | }) 73 | 74 | testify.assert_equal(self.smap['jason','dave'], ['fennell', 'marin']) 75 | testify.assert_equal(self.smap['dave', 'jason'], ['marin', 'fennell']) 76 | testify.assert_equal(self.smap[('jason', 'dave')], ['fennell', 'marin']) 77 | gen = (x for x in ['jason', 'dave']) 78 | testify.assert_equal(self.smap[gen], ['fennell', 'marin']) 79 | testify.assert_raises( 80 | KeyError, 81 | lambda: self.smap['jason', 'brandon'] 82 | ) 83 | 84 | def test_overwrite(self): 85 | self.smap['yelp'] = 'yelp!' 86 | testify.assert_equal(self.smap['yelp'], 'yelp!') 87 | self.smap['yelp'] = 'yelp!1234' 88 | testify.assert_equal(self.smap['yelp'], 'yelp!1234') 89 | 90 | def test_delitem(self): 91 | self.smap['boo'] = 'ahhh!' 92 | testify.assert_equal(self.smap['boo'], 'ahhh!') 93 | del self.smap['boo'] 94 | testify.assert_not_in('boo', self.smap) 95 | 96 | testify.assert_raises(KeyError, lambda : self.smap['boo']) 97 | 98 | def try_delete(): 99 | del self.smap['boo'] 100 | testify.assert_raises(KeyError, try_delete) 101 | 102 | def test_contains(self): 103 | self.smap['containers'] = 'blah' 104 | testify.assert_in('containers', self.smap) 105 | 106 | def test_iteritems(self): 107 | expected_d = {'1': 'a', '2': 'b', '3': 'c'} 108 | self.prepopulate_map_test(expected_d, self.smap) 109 | real_d = dict(self.smap.iteritems()) 110 | testify.assert_equal(expected_d, real_d) 111 | 112 | def test_items(self): 113 | expected_d = {'1': 'a', '2': 'b', '3': 'c'} 114 | self.prepopulate_map_test(expected_d, self.smap) 115 | real_d = dict(self.smap.items()) 116 | testify.assert_equal(expected_d, real_d) 117 | 118 | def test_iterkeys(self): 119 | d = {'1': 'a', '2': 'b', '3': 'c'} 120 | self.prepopulate_map_test(d, self.smap) 121 | expected_keys = set(d.iterkeys()) 122 | real_keys = set(self.smap.iterkeys()) 123 | testify.assert_equal(expected_keys, real_keys) 124 | 125 | def test_keys(self): 126 | d = {'1': 'a', '2': 'b', '3': 'c'} 127 | self.prepopulate_map_test(d, self.smap) 128 | expected_keys = set(d.keys()) 129 | real_keys = set(self.smap.keys()) 130 | testify.assert_equal(expected_keys, real_keys) 131 | 132 | def test_itervalues(self): 133 | d = {'1': 'a', '2': 'b', '3': 'c'} 134 | self.prepopulate_map_test(d, self.smap) 135 | expected_values = set(d.itervalues()) 136 | real_values = set(self.smap.itervalues()) 137 | testify.assert_equal(expected_values, real_values) 138 | 139 | def test_values(self): 140 | d = {'1': 'a', '2': 'b', '3': 'c'} 141 | self.prepopulate_map_test(d, self.smap) 142 | expected_values = set(d.values()) 143 | real_values = set(self.smap.values()) 144 | testify.assert_equal(expected_values, real_values) 145 | 146 | def test_iter(self): 147 | # __iter__ should behave like iterkeys() 148 | d = {'1': 'a', '2': 'b', '3': 'c'} 149 | self.prepopulate_map_test(d, self.smap) 150 | expected_keys = set(d.iterkeys()) 151 | real_keys = set(self.smap) 152 | testify.assert_equal(expected_keys, real_keys) 153 | 154 | def test_len(self): 155 | self.smap['1'] = 'a' 156 | testify.assert_equal(len(self.smap), 1) 157 | self.smap['2'] = 'b' 158 | testify.assert_equal(len(self.smap), 2) 159 | self.smap['3'] = 'c' 160 | testify.assert_equal(len(self.smap), 3) 161 | del self.smap['2'] 162 | testify.assert_equal(len(self.smap), 2) 163 | self.smap['1'] = 'z' 164 | testify.assert_equal(len(self.smap), 2) 165 | 166 | def test_clear(self): 167 | d = {'1': 'a', '2': 'b', '3': 'c'} 168 | 169 | self.prepopulate_map_test(d, self.smap) 170 | for k in d: 171 | testify.assert_in(k, self.smap) 172 | 173 | self.smap.clear() 174 | for k in d: 175 | testify.assert_not_in(k, self.smap) 176 | 177 | def test_get(self): 178 | self.smap['jason'] = 'fennell' 179 | testify.assert_equal(self.smap.get('jason'), 'fennell') 180 | assert self.smap.get('brandon') is None 181 | testify.assert_equal(self.smap.get('brandon', 'dion'), 'dion') 182 | 183 | def test_has_key(self): 184 | self.smap['jason'] = 'fennell' 185 | assert self.smap.has_key('jason') 186 | assert not self.smap.has_key('brandon') 187 | 188 | def test_pop(self): 189 | self.smap['jason'] = 'fennell' 190 | testify.assert_equal(self.smap.pop('jason'), 'fennell') 191 | 192 | testify.assert_not_in('jason', self.smap) 193 | 194 | assert self.smap.pop('jason', None) is None 195 | testify.assert_raises( 196 | KeyError, 197 | lambda: self.smap.pop('jason') 198 | ) 199 | 200 | def test_popitem(self): 201 | d = {'1': 'a', '2': 'b'} 202 | self.prepopulate_map_test(d, self.smap) 203 | 204 | out_d = {} 205 | k, v = self.smap.popitem() 206 | out_d[k] = v 207 | k, v = self.smap.popitem() 208 | out_d[k] = v 209 | 210 | testify.assert_equal(out_d, d) 211 | 212 | testify.assert_raises( 213 | KeyError, 214 | lambda: self.smap.popitem() 215 | ) 216 | 217 | def test_setdefault(self): 218 | self.smap.setdefault('jason', 'fennell') 219 | testify.assert_equal(self.smap['jason'], 'fennell') 220 | 221 | self.smap.setdefault('jason', 'daniel') 222 | testify.assert_equal(self.smap['jason'], 'fennell') 223 | 224 | self.smap.setdefault('brandon') 225 | assert self.smap['brandon'] is None 226 | 227 | def test_update_dict(self): 228 | self.smap['foo'] = 'bar' 229 | 230 | names = { 231 | 'jason': 'fennell', 232 | 'brandon': 'fennell', 233 | } 234 | self.smap.update(names) 235 | names['foo'] = 'bar' 236 | testify.assert_equal(names, dict(self.smap.items())) 237 | 238 | middle_names = { 239 | 'jason': 'daniel', 240 | 'brandon': 'dion', 241 | } 242 | self.smap.update(middle_names) 243 | middle_names['foo'] = 'bar' 244 | testify.assert_equal(middle_names, dict(self.smap.items())) 245 | 246 | def test_update_list(self): 247 | self.smap['foo'] = 'bar' 248 | 249 | names = { 250 | 'jason': 'fennell', 251 | 'brandon': 'fennell', 252 | } 253 | self.smap.update(names.items()) 254 | names['foo'] = 'bar' 255 | testify.assert_equal(names, dict(self.smap.items())) 256 | 257 | def test_update_iter(self): 258 | self.smap['foo'] = 'bar' 259 | 260 | names = { 261 | 'jason': 'fennell', 262 | 'brandon': 'fennell', 263 | } 264 | self.smap.update(names.iteritems()) 265 | names['foo'] = 'bar' 266 | testify.assert_equal(names, dict(self.smap.items())) 267 | 268 | def test_update_kwargs(self): 269 | self.smap['foo'] = 'bar' 270 | 271 | names = { 272 | 'jason': 'fennell', 273 | 'brandon': 'fennell', 274 | } 275 | self.smap.update(**names) 276 | names['foo'] = 'bar' 277 | testify.assert_equal(names, dict(self.smap.items())) 278 | 279 | def test_select(self): 280 | self.smap.update({ 281 | 'jason': 'fennell', 282 | 'dave': 'marin', 283 | 'benjamin': 'goldenberg', 284 | }) 285 | 286 | testify.assert_equal( 287 | self.smap.select('jason', 'dave'), 288 | ['fennell', 'marin'] 289 | ) 290 | testify.assert_equal( 291 | self.smap.select(['dave', 'jason']), 292 | ['marin', 'fennell'] 293 | ) 294 | gen = (x for x in ['benjamin', 'dave', 'jason']) 295 | testify.assert_equal( 296 | self.smap.select(gen), 297 | ['goldenberg', 'marin', 'fennell'] 298 | ) 299 | testify.assert_raises( 300 | KeyError, 301 | lambda: self.smap.select('jason', 'brandon') 302 | ) 303 | 304 | def test_get_many(self): 305 | self.smap.update({ 306 | 'jason': 'fennell', 307 | 'dave': 'marin', 308 | 'benjamin': 'goldenberg', 309 | }) 310 | 311 | testify.assert_equal(self.smap.get_many([]), []) 312 | testify.assert_equal( 313 | self.smap.get_many('jason', 'dave'), 314 | ['fennell', 'marin'] 315 | ) 316 | testify.assert_equal( 317 | self.smap.get_many(['dave', 'jason']), 318 | ['marin', 'fennell'] 319 | ) 320 | gen = (x for x in ['benjamin', 'dave', 'jason']) 321 | testify.assert_equal( 322 | self.smap.get_many(gen), 323 | ['goldenberg', 'marin', 'fennell'] 324 | ) 325 | testify.assert_equal( 326 | self.smap.get_many('jason', 'brandon'), 327 | ['fennell', None] 328 | ) 329 | testify.assert_equal( 330 | self.smap.get_many('jason', 'brandon', default=''), 331 | ['fennell', ''] 332 | ) 333 | 334 | 335 | class TestSqliteRegressions(SqliteMapTestCase): 336 | """A place for regression tests""" 337 | 338 | def test_huge_selects(self): 339 | """There is a 1000-variable limit when binding variables in sqlite 340 | statements. Make sure we can do selects bigger than this 341 | transparently. 342 | """ 343 | 344 | select_sizes = [10, 37, 100, 849, 1000, 2348, 10000] 345 | for size in select_sizes: 346 | self.smap.clear() 347 | self.smap.update((str(x), str(x)) for x in xrange(size)) 348 | keys = [str(x) for x in xrange(size)] 349 | expected = [str(x) for x in xrange(size)] 350 | testify.assert_equal( 351 | self.smap.select(keys), 352 | expected, 353 | message='Select failed on %d elements' % size 354 | ) 355 | 356 | def test_get_many_unicode_keys(self): 357 | """Make sure get_many works correctly with unicode keys.""" 358 | k = u'\u6d77\u5bf6\u9ede\u5fc3\u7f8e\u98df\u574a' 359 | v = 'hello' 360 | self.smap[k] = v 361 | 362 | testify.assert_equal(self.smap[k], v) 363 | testify.assert_equal(self.smap.get(k), v) 364 | testify.assert_equal(self.smap.get_many([k]), [v]) 365 | 366 | 367 | class TestSqliteStorage(SqliteMapTestCase): 368 | """Tests things like key capacity and persistence to disk""" 369 | 370 | def test_multiple_open_maps_per_path(self): 371 | smap1 = self.smap 372 | smap2 = sqlite3dbm.dbm.SqliteMap(self.path, flag='w') 373 | 374 | # Write in the first map 375 | smap1['foo'] = 'a' 376 | testify.assert_equal(smap1['foo'], 'a') 377 | testify.assert_equal(smap2['foo'], 'a') 378 | 379 | # Write in the second map 380 | smap2['bar'] = 'b' 381 | testify.assert_equal(smap1['bar'], 'b') 382 | testify.assert_equal(smap2['bar'], 'b') 383 | 384 | # Overwrite 385 | smap1['foo'] = 'c' 386 | testify.assert_equal(smap1['foo'], 'c') 387 | testify.assert_equal(smap2['foo'], 'c') 388 | 389 | # Delete 390 | del smap1['foo'] 391 | testify.assert_not_in('foo', smap1) 392 | testify.assert_not_in('foo', smap2) 393 | 394 | def test_persistence_through_reopens(self): 395 | self.smap['foo'] = 'a' 396 | testify.assert_equal(self.smap['foo'], 'a') 397 | 398 | # Remove/close the map and open a new one 399 | del self.smap 400 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='w') 401 | testify.assert_equal(smap['foo'], 'a') 402 | 403 | 404 | class TestSqliteMemoryStorage(testify.TestCase): 405 | """Test that storage for in-memory databases works as expected.""" 406 | 407 | def test_multiple_in_memory_maps(self): 408 | # In-memory maps should not share state 409 | smap1 = sqlite3dbm.dbm.SqliteMap(':memory:', flag='w') 410 | smap2 = sqlite3dbm.dbm.SqliteMap(':memory:', flag='w') 411 | 412 | # Write to just the first map 413 | smap1['foo'] = 'a' 414 | testify.assert_equal(smap1['foo'], 'a') 415 | testify.assert_not_in('foo', smap2) 416 | 417 | # Write to just the second map 418 | smap2['bar'] = 'b' 419 | testify.assert_not_in('bar', smap1) 420 | testify.assert_equal(smap2['bar'], 'b') 421 | 422 | def test_not_persistent_through_reopen(self): 423 | smap = sqlite3dbm.dbm.SqliteMap(':memory:', flag='w') 424 | smap['foo'] = 'a' 425 | testify.assert_equal(smap['foo'], 'a') 426 | 427 | # We shuld have an empty map after closing & opening a new onw 428 | del smap 429 | smap = sqlite3dbm.dbm.SqliteMap(':memory:', flag='w') 430 | testify.assert_equal(smap.items(), []) 431 | 432 | 433 | class SqliteCreationTest(testify.TestCase): 434 | """Base class for tests checking creation of SqliteMap backend stores""" 435 | @testify.setup 436 | def create_tmp_working_area(self): 437 | self.tmpdir = tempfile.mkdtemp() 438 | self.path = os.path.join(self.tmpdir, 'sqlite_map_test_db.sqlite') 439 | # Do not yet create a db. Most of what we are testing 440 | # here involves the constructor 441 | 442 | @testify.teardown 443 | def teardown_tmp_working_area(self): 444 | shutil.rmtree(self.tmpdir) 445 | 446 | 447 | class TestFlags(SqliteCreationTest): 448 | def test_create(self): 449 | # Should be able to create a db when none was present 450 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='c') 451 | 452 | # Writeable 453 | smap['foo'] = 'bar' 454 | testify.assert_equal(smap['foo'], 'bar') 455 | 456 | # Persists across re-open 457 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='c') 458 | testify.assert_equal(smap['foo'], 'bar') 459 | 460 | def test_read_only(self): 461 | # Read mode exects db to already exist 462 | testify.assert_raises( 463 | sqlite3dbm.dbm.error, 464 | lambda: sqlite3dbm.dbm.SqliteMap(self.path, flag='r'), 465 | ) 466 | # Create the db then re-open read-only 467 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='c') 468 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='r') 469 | 470 | # Check that all mutators raise exceptions 471 | mutator_raises = lambda callable_method: testify.assert_raises( 472 | sqlite3dbm.dbm.error, 473 | callable_method 474 | ) 475 | def do_setitem(): 476 | smap['foo'] = 'bar' 477 | mutator_raises(do_setitem) 478 | def do_delitem(): 479 | del smap['foo'] 480 | mutator_raises(do_delitem) 481 | mutator_raises(lambda: smap.clear()) 482 | mutator_raises(lambda: smap.pop('foo')) 483 | mutator_raises(lambda: smap.popitem()) 484 | mutator_raises(lambda: smap.update({'baz': 'qux'})) 485 | 486 | def test_default_read_only(self): 487 | """Check that the default flag is read-only""" 488 | # Should be upset if db is not there already 489 | testify.assert_raises( 490 | sqlite3dbm.dbm.error, 491 | lambda: sqlite3dbm.dbm.SqliteMap(self.path) 492 | ) 493 | # Create and re-open 494 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='c') 495 | smap = sqlite3dbm.dbm.SqliteMap(self.path) 496 | 497 | # Setitem should cause an error 498 | def do_setitem(): 499 | smap['foo'] = 'bar' 500 | testify.assert_raises( 501 | sqlite3dbm.dbm.error, 502 | do_setitem 503 | ) 504 | 505 | def test_writeable(self): 506 | # Read/write mode requites db to already exist 507 | testify.assert_raises( 508 | sqlite3dbm.dbm.error, 509 | lambda: sqlite3dbm.dbm.SqliteMap(self.path, flag='w') 510 | ) 511 | # Create db and re-open for writing 512 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='c') 513 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='w') 514 | 515 | # Check writeable 516 | smap['foo'] = 'bar' 517 | testify.assert_equal(smap['foo'], 'bar') 518 | 519 | # Check persistent through re-open 520 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='w') 521 | testify.assert_equal(smap['foo'], 'bar') 522 | 523 | def test_new_db(self): 524 | # New, empty db should be fine with file not existing 525 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='n') 526 | 527 | # Writeable 528 | smap['foo'] = 'bar' 529 | testify.assert_equal(smap['foo'], 'bar') 530 | 531 | # Re-open should give an empty db 532 | smap = sqlite3dbm.dbm.SqliteMap(self.path, flag='n') 533 | testify.assert_not_in('foo', smap) 534 | testify.assert_equal(len(smap), 0) 535 | 536 | 537 | class TestModes(SqliteCreationTest): 538 | # Make sure that any changes to umask do not 539 | # corrupt the state of other tests 540 | @testify.setup 541 | def fix_umask_setup(self): 542 | # Have to set a umask in order to get the current one 543 | self.orig_umask = os.umask(0000) 544 | os.umask(self.orig_umask) 545 | 546 | @testify.teardown 547 | def fix_umask_teardown(self): 548 | # Reset to the original umask 549 | os.umask(self.orig_umask) 550 | 551 | def get_perm_mask(self, path): 552 | """Pull out the permission bits of a file""" 553 | return stat.S_IMODE(os.stat(path).st_mode) 554 | 555 | def test_default_mode(self): 556 | # Turn off umask for inital testing of modes 557 | os.umask(0000) 558 | 559 | # Create a db, check default mode is 0666 560 | sqlite3dbm.dbm.SqliteMap(self.path, flag='c') 561 | testify.assert_equal(self.get_perm_mask(self.path), 0666) 562 | 563 | def test_custom_mode(self): 564 | # Turn off umask 565 | os.umask(0000) 566 | 567 | # Create a db with a custom mode 568 | mode = 0600 569 | sqlite3dbm.dbm.SqliteMap(self.path, flag='c', mode=mode) 570 | testify.assert_equal(self.get_perm_mask(self.path), mode) 571 | 572 | def test_respects_umask(self): 573 | mode = 0777 574 | umask = 0002 575 | os.umask(umask) 576 | expected_mode = mode & ~umask 577 | 578 | sqlite3dbm.dbm.SqliteMap(self.path, flag='c', mode=mode) 579 | testify.assert_equal(self.get_perm_mask(self.path), expected_mode) 580 | 581 | 582 | class SanityCheckOpen(SqliteCreationTest): 583 | def test_open_creates(self): 584 | smap = sqlite3dbm.dbm.open(self.path, flag='c') 585 | smap['foo'] = 'bar' 586 | testify.assert_equal(smap['foo'], 'bar') 587 | 588 | 589 | if __name__ == '__main__': 590 | testify.run() 591 | -------------------------------------------------------------------------------- /tests/sshelve_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2011 Yelp 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Test the shelve wrapper around the SqliteMap""" 17 | 18 | import os 19 | import shutil 20 | import tempfile 21 | import time 22 | 23 | import testify 24 | 25 | import sqlite3dbm 26 | 27 | class TestSqliteShelf(testify.TestCase): 28 | @testify.setup 29 | def create_shelf(self): 30 | self.tmpdir = tempfile.mkdtemp() 31 | self.path = os.path.join(self.tmpdir, 'sqlite_map_test_db.sqlite') 32 | self.smap = sqlite3dbm.open(self.path, flag='c') 33 | self.smap_shelf = sqlite3dbm.sshelve.SqliteMapShelf(self.smap) 34 | 35 | @testify.teardown 36 | def teardown_shelf(self): 37 | shutil.rmtree(self.tmpdir) 38 | 39 | def test_basic_serialization(self): 40 | def check(k, v): 41 | self.smap_shelf[k] = v 42 | testify.assert_equal(self.smap_shelf[k], v) 43 | 44 | check('foo', 'bar') 45 | check('foo', True) 46 | check('foo', False) 47 | check('foo', [1, 2, 3]) 48 | check('foo', (1, 2, 3)) 49 | check('foo', {'a': 1, 'b': 2}) 50 | check('foo', [{'a': 1}, [2, 3], 4]) 51 | 52 | def test_select(self): 53 | droid = ['R2-D2', 'C-3P0'] 54 | self.smap_shelf.update({ 55 | 'jason': 'fennell', 56 | 'droid': droid, 57 | 'pi': 3.14 58 | }) 59 | 60 | testify.assert_equal( 61 | self.smap_shelf.select('jason', 'droid', 'pi'), 62 | ['fennell', droid, 3.14] 63 | ) 64 | testify.assert_raises( 65 | KeyError, 66 | lambda: self.smap_shelf.select('jason', 'droid', 'brandon'), 67 | ) 68 | 69 | def test_get_many(self): 70 | droid = ['R2-D2', 'C-3P0'] 71 | self.smap_shelf.update({ 72 | 'jason': 'fennell', 73 | 'droid': droid, 74 | 'pi': 3.14 75 | }) 76 | 77 | testify.assert_equal( 78 | self.smap_shelf.get_many('jason', 'droid', 'pi'), 79 | ['fennell', droid, 3.14] 80 | ) 81 | testify.assert_equal( 82 | self.smap_shelf.get_many('jason', 'droid', 'brandon'), 83 | ['fennell', droid, None] 84 | ) 85 | testify.assert_equal( 86 | self.smap_shelf.get_many('jason', 'droid', 'brandon', default=0), 87 | ['fennell', droid, 0] 88 | ) 89 | 90 | def test_update(self): 91 | droid = ['R2-D2', 'C-3P0'] 92 | self.smap_shelf.update({ 93 | 'jason': 'fennell', 94 | 'droid': droid, 95 | 'pi': 3.14 96 | }) 97 | 98 | testify.assert_equal(self.smap_shelf['jason'], 'fennell') 99 | testify.assert_equal(self.smap_shelf['droid'], droid) 100 | testify.assert_equal(self.smap_shelf['pi'], 3.14) 101 | 102 | def test_clear(self): 103 | droid = ['R2-D2', 'C-3P0'] 104 | self.smap_shelf.update({ 105 | 'jason': 'fennell', 106 | 'droid': droid, 107 | 'pi': 3.14 108 | }) 109 | 110 | testify.assert_equal(self.smap_shelf['jason'], 'fennell') 111 | testify.assert_equal(len(self.smap_shelf), 3) 112 | 113 | self.smap_shelf.clear() 114 | 115 | testify.assert_equal(len(self.smap_shelf), 0) 116 | testify.assert_not_in('jason', self.smap_shelf) 117 | 118 | def test_preserves_unicode(self): 119 | """Be paranoid about unicode.""" 120 | k = u'café'.encode('utf-8') 121 | v = u'bläserforum' 122 | self.smap_shelf[k] = v 123 | 124 | testify.assert_equal(self.smap_shelf[k], v) 125 | testify.assert_equal(self.smap_shelf.get_many([k]), [v]) 126 | 127 | 128 | class TestShelfOpen(testify.TestCase): 129 | @testify.setup 130 | def create_shelf(self): 131 | self.tmpdir = tempfile.mkdtemp() 132 | self.path = os.path.join(self.tmpdir, 'sqlite_map_test_db.sqlite') 133 | 134 | @testify.teardown 135 | def teardown_shelf(self): 136 | shutil.rmtree(self.tmpdir) 137 | 138 | def test_open(self): 139 | smap_shelf = sqlite3dbm.sshelve.open(self.path) 140 | smap_shelf['foo'] = ['bar', 'baz', 'qux'] 141 | testify.assert_equal(smap_shelf['foo'], ['bar', 'baz', 'qux']) 142 | 143 | 144 | class TestShelfPerf(testify.TestCase): 145 | @testify.setup 146 | def create_environ(self): 147 | self.tmpdir = tempfile.mkdtemp() 148 | 149 | @testify.teardown 150 | def teardown(self): 151 | shutil.rmtree(self.tmpdir) 152 | 153 | def test_update_perf(self): 154 | """update() should be faster than lots of individual inserts""" 155 | 156 | # Knobs that control how long this test takes vs. how accurate it is 157 | # This test *should not flake*, but if you run into problems then you 158 | # should increase `insert_per_iter` (the test will take longer though) 159 | num_iters = 5 160 | insert_per_iter = 300 161 | min_ratio = 10 162 | 163 | # Setup dbs 164 | def setup_dbs(name): 165 | name = name + '%d' 166 | db_paths = [ 167 | os.path.join(self.tmpdir, name % i) 168 | for i in xrange(num_iters) 169 | ] 170 | return [sqlite3dbm.sshelve.open(path) for path in db_paths] 171 | update_dbs = setup_dbs('update') 172 | insert_dbs = setup_dbs('insert') 173 | 174 | # Setup data 175 | insert_data = [ 176 | ('foo%d' % i, 'bar%d' % i) 177 | for i in xrange(insert_per_iter) 178 | ] 179 | 180 | # Time upates 181 | update_start = time.time() 182 | for update_db in update_dbs: 183 | update_db.update(insert_data) 184 | update_time = time.time() - update_start 185 | 186 | # Time inserts 187 | insert_start = time.time() 188 | for insert_db in insert_dbs: 189 | for k, v in insert_data: 190 | insert_db[k] = v 191 | insert_time = time.time() - insert_start 192 | 193 | # Inserts should take a subsantially greater amount of time 194 | testify.assert_gt(insert_time, min_ratio*update_time) 195 | 196 | 197 | if __name__ == '__main__': 198 | testify.run() 199 | --------------------------------------------------------------------------------