├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── README.rst ├── __init__.py ├── setup.cfg ├── setup.py ├── sqlite3worker.py └── sqlite3worker_test.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Python build files 2 | *.pyc 3 | *.clean 4 | *~ 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *,cover 52 | .hypothesis/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # dotenv 88 | .env 89 | 90 | # virtualenv 91 | .venv 92 | venv/ 93 | ENV/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Automated testing. 2 | language: python 3 | python: 4 | - "2.7" 5 | - "3.4" 6 | - "3.5" 7 | - "3.6" 8 | - "3.7" 9 | - "3.8" 10 | - "3.9" 11 | # command to run tests 12 | script: python setup.py test 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Palantir Technologies 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sqlite3Worker 2 | 3 | A threadsafe sqlite worker. 4 | 5 | This library implements a thread pool pattern with sqlite3 being the desired 6 | output. 7 | 8 | sqllite3 implementation lacks the ability to safely modify the sqlite3 database 9 | with multiple threads outside of the compile time options. This library was 10 | created to address this by bringing the responsibility of managing the threads 11 | to the python layer and is agnostic to the server setup of sqlite3. 12 | 13 | [![Build Status](https://travis-ci.org/dashawn888/sqlite3worker.svg?branch=master)](https://travis-ci.org/dashawn888/sqlite3worker) 14 | 15 | ## Install 16 | You can use pip: 17 | ```sh 18 | sudo pip install sqlite3worker 19 | ``` 20 | 21 | You can use setup.py: 22 | ```sh 23 | sudo python setup.py install 24 | ``` 25 | 26 | Alternatively one can use ``pip`` to install directly from the git repository 27 | without having to clone first: 28 | 29 | ```sh 30 | sudo pip install git+https://github.com/dashawn888/sqlite3worker#egg=sqlite3worker 31 | ``` 32 | 33 | One may also use ``pip`` to install on a per-user basis without requiring 34 | super-user permissions: 35 | 36 | ```sh 37 | pip install --user git+https://github.com/dashawn888/sqlite3worker#egg=sqlite3worker 38 | ``` 39 | 40 | ## Example 41 | ```python 42 | from sqlite3worker import Sqlite3Worker 43 | 44 | sql_worker = Sqlite3Worker("/tmp/test.sqlite") 45 | sql_worker.execute("CREATE TABLE tester (timestamp DATETIME, uuid TEXT)") 46 | sql_worker.execute("INSERT into tester values (?, ?)", ("2010-01-01 13:00:00", "bow")) 47 | sql_worker.execute("INSERT into tester values (?, ?)", ("2011-02-02 14:14:14", "dog")) 48 | 49 | results = sql_worker.execute("SELECT * from tester") 50 | for timestamp, uuid in results: 51 | print(timestamp, uuid) 52 | 53 | sql_worker.close() 54 | ``` 55 | 56 | ## When to use sqlite3worker 57 | If you have multiple threads all needing to write to a sqlite3 database this 58 | library will serialize the sqlite3 write requests. 59 | 60 | ## When NOT to use sqlite3worker 61 | If your code DOES NOT use multiple threads then you don't need to use a thread 62 | safe sqlite3 implementation. 63 | 64 | If you need multiple applications to write to a sqlite3 db then sqlite3worker 65 | will not protect you from corrupting the data. 66 | 67 | ## Internals 68 | The library creates a queue to manage multiple queries sent to the database. 69 | Instead of directly calling the sqlite3 interface, you will call the 70 | Sqlite3Worker which inserts your query into a Queue.Queue() object. The queries 71 | are processed in the order that they are inserted into the queue (first in, 72 | first out). In order to ensure that the multiple threads are managed in the 73 | same queue, you will need to pass the same Sqlite3Worker object to each thread. 74 | 75 | ## Python docs for sqlite3 76 | https://docs.python.org/2/library/sqlite3.html 77 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Sqlite3Worker 2 | ============= 3 | 4 | A threadsafe sqlite worker. 5 | 6 | This library implements a thread pool pattern with sqlite3 being the 7 | desired output. 8 | 9 | sqllite3 implementation lacks the ability to safely modify the sqlite3 10 | database with multiple threads outside of the compile time options. This 11 | library was created to address this by bringing the responsibility of 12 | managing the threads to the python layer and is agnostic to the server 13 | setup of sqlite3. 14 | 15 | |Build Status| 16 | 17 | Install 18 | ------- 19 | 20 | You can use pip: 21 | 22 | .. code:: sh 23 | 24 | sudo pip install sqlite3worker 25 | 26 | You can use setup.py: 27 | 28 | .. code:: sh 29 | 30 | sudo python setup.py install 31 | 32 | Alternatively one can use ``pip`` to install directly from the git 33 | repository without having to clone first: 34 | 35 | .. code:: sh 36 | 37 | sudo pip install git+https://github.com/dashawn888/sqlite3worker#egg=sqlite3worker 38 | 39 | One may also use ``pip`` to install on a per-user basis without 40 | requiring super-user permissions: 41 | 42 | .. code:: sh 43 | 44 | pip install --user git+https://github.com/dashawn888/sqlite3worker#egg=sqlite3worker 45 | 46 | Example 47 | ------- 48 | 49 | .. code:: python 50 | 51 | from sqlite3worker import Sqlite3Worker 52 | 53 | sql_worker = Sqlite3Worker("/tmp/test.sqlite") 54 | sql_worker.execute("CREATE TABLE tester (timestamp DATETIME, uuid TEXT)") 55 | sql_worker.execute("INSERT into tester values (?, ?)", ("2010-01-01 13:00:00", "bow")) 56 | sql_worker.execute("INSERT into tester values (?, ?)", ("2011-02-02 14:14:14", "dog")) 57 | 58 | results = sql_worker.execute("SELECT * from tester") 59 | for timestamp, uuid in results: 60 | print(timestamp, uuid) 61 | 62 | sql_worker.close() 63 | 64 | When to use sqlite3worker 65 | ------------------------- 66 | 67 | If you have multiple threads all needing to write to a sqlite3 database 68 | this library will serialize the sqlite3 write requests. 69 | 70 | When NOT to use sqlite3worker 71 | ----------------------------- 72 | 73 | If your code DOES NOT use multiple threads then you don't need to use a 74 | thread safe sqlite3 implementation. 75 | 76 | If you need multiple applications to write to a sqlite3 db then 77 | sqlite3worker will not protect you from corrupting the data. 78 | 79 | Internals 80 | --------- 81 | 82 | The library creates a queue to manage multiple queries sent to the 83 | database. Instead of directly calling the sqlite3 interface, you will 84 | call the Sqlite3Worker which inserts your query into a Queue.Queue() 85 | object. The queries are processed in the order that they are inserted 86 | into the queue (first in, first out). In order to ensure that the 87 | multiple threads are managed in the same queue, you will need to pass 88 | the same Sqlite3Worker object to each thread. 89 | 90 | Python docs for sqlite3 91 | ----------------------- 92 | 93 | https://docs.python.org/2/library/sqlite3.html 94 | 95 | .. |Build Status| image:: https://travis-ci.org/dashawn888/sqlite3worker.svg?branch=master 96 | :target: https://travis-ci.org/dashawn888/sqlite3worker 97 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2014 Palantir Technologies 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | """Init.""" 22 | 23 | __author__ = "Shawn Lee" 24 | __email__ = "dashawn@gmail.com" 25 | __license__ = "MIT" 26 | __version__ = "1.1.7" 27 | 28 | try: 29 | # Python 2 30 | from sqlite3worker import Sqlite3Worker 31 | except ImportError: 32 | # Python 3 33 | from sqlite3worker.sqlite3worker import Sqlite3Worker 34 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2014 Palantir Technologies 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | """Setup.""" 23 | 24 | __author__ = "Shawn Lee" 25 | __email__ = "dashawn@gmail.com" 26 | __license__ = "MIT" 27 | 28 | from setuptools import setup 29 | import os 30 | 31 | short_description = ("Thread safe sqlite3 interface",) 32 | long_description = short_description 33 | if os.path.exists("README.rst"): 34 | long_description = open("README.rst").read() 35 | 36 | setup( 37 | name="sqlite3worker", 38 | version="1.1.7", 39 | description=short_description, 40 | author="Shawn Lee", 41 | author_email="dashawn@gmail.com", 42 | url="https://github.com/dashawn888/sqlite3worker", 43 | packages=["sqlite3worker"], 44 | package_dir={"sqlite3worker": "."}, 45 | keywords=["sqlite", "sqlite3", "thread", "multithread", "multithreading"], 46 | classifiers=[ 47 | "Development Status :: 5 - Production/Stable", 48 | "Intended Audience :: Developers", 49 | "License :: OSI Approved :: MIT License", 50 | "Programming Language :: Python :: 2", 51 | "Programming Language :: Python :: 2.7", 52 | "Programming Language :: Python :: 3", 53 | "Programming Language :: Python :: 3.0", 54 | "Programming Language :: Python :: 3.1", 55 | "Programming Language :: Python :: 3.2", 56 | "Programming Language :: Python :: 3.3", 57 | "Programming Language :: Python :: 3.4", 58 | "Programming Language :: Python :: 3.5", 59 | "Programming Language :: Python :: 3.6", 60 | "Topic :: Database", 61 | ], 62 | test_suite="sqlite3worker_test", 63 | ) 64 | -------------------------------------------------------------------------------- /sqlite3worker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2014 Palantir Technologies 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | """Thread safe sqlite3 interface.""" 23 | 24 | __author__ = "Shawn Lee" 25 | __email__ = "dashawn@gmail.com" 26 | __license__ = "MIT" 27 | 28 | import logging 29 | import sqlite3 30 | import threading 31 | import uuid 32 | 33 | try: 34 | import queue as Queue # module re-named in Python 3 35 | except ImportError: 36 | import Queue 37 | 38 | LOGGER = logging.getLogger("sqlite3worker") 39 | 40 | 41 | class Sqlite3Worker(threading.Thread): 42 | """Sqlite thread safe object. 43 | 44 | Example: 45 | from sqlite3worker import Sqlite3Worker 46 | sql_worker = Sqlite3Worker("/tmp/test.sqlite") 47 | sql_worker.execute( 48 | "CREATE TABLE tester (timestamp DATETIME, uuid TEXT)") 49 | sql_worker.execute( 50 | "INSERT into tester values (?, ?)", ("2010-01-01 13:00:00", "bow")) 51 | sql_worker.execute( 52 | "INSERT into tester values (?, ?)", ("2011-02-02 14:14:14", "dog")) 53 | sql_worker.execute("SELECT * from tester") 54 | sql_worker.close() 55 | """ 56 | 57 | def __init__(self, file_name, max_queue_size=100, raise_on_error=True): 58 | """Automatically starts the thread. 59 | 60 | Args: 61 | file_name: The name of the file. 62 | max_queue_size: The max queries that will be queued. 63 | raise_on_error: raise the exception on commit error 64 | """ 65 | threading.Thread.__init__(self, name=__name__) 66 | self.daemon = True 67 | self._sqlite3_conn = sqlite3.connect( 68 | file_name, check_same_thread=False, detect_types=sqlite3.PARSE_DECLTYPES 69 | ) 70 | self._sqlite3_cursor = self._sqlite3_conn.cursor() 71 | self._sql_queue = Queue.Queue(maxsize=max_queue_size) 72 | self._results = {} 73 | self._max_queue_size = max_queue_size 74 | self._raise_on_error = raise_on_error 75 | # Event that is triggered once the run_query has been executed. 76 | self._select_events = {} 77 | # Event to start the close process. 78 | self._close_event = threading.Event() 79 | # Event that closes out the threads. 80 | self._close_lock = threading.Lock() 81 | self.start() 82 | 83 | def run(self): 84 | """Thread loop. 85 | 86 | This is an infinite loop. The iter method calls self._sql_queue.get() 87 | which blocks if there are not values in the queue. As soon as values 88 | are placed into the queue the process will continue. 89 | 90 | If many executes happen at once it will churn through them all before 91 | calling commit() to speed things up by reducing the number of times 92 | commit is called. 93 | """ 94 | LOGGER.debug("run: Thread started") 95 | execute_count = 0 96 | for token, query, values in iter(self._sql_queue.get, None): 97 | if query: 98 | LOGGER.debug("_sql_queue: %s", self._sql_queue.qsize()) 99 | LOGGER.debug("run: %s", query) 100 | self._run_query(token, query, values) 101 | execute_count += 1 102 | # Let the executes build up a little before committing to disk 103 | # to speed things up. 104 | if self._sql_queue.empty() or execute_count == self._max_queue_size: 105 | try: 106 | LOGGER.debug("run: commit") 107 | self._sqlite3_conn.commit() 108 | execute_count = 0 109 | except Exception as e: 110 | LOGGER.error(e, exc_info=True) 111 | if self._raise_on_error: 112 | raise e 113 | # Only close if the queue is empty. Otherwise keep getting 114 | # through the queue until it's empty. 115 | if self._close_event.is_set() and self._sql_queue.empty(): 116 | self._sqlite3_conn.commit() 117 | self._sqlite3_conn.close() 118 | return 119 | 120 | def _run_query(self, token, query, values): 121 | """Run a query. 122 | 123 | Args: 124 | token: A uuid object of the query you want returned. 125 | query: A sql query with ? placeholders for values. 126 | values: A tuple of values to replace "?" in query. 127 | """ 128 | if query.lower().strip().startswith("select"): 129 | try: 130 | self._sqlite3_cursor.execute(query, values) 131 | self._results[token] = self._sqlite3_cursor.fetchall() 132 | except sqlite3.Error as err: 133 | # Put the error into the output queue since a response 134 | # is required. 135 | self._results[token] = "Query returned error: %s: %s: %s" % ( 136 | query, 137 | values, 138 | err, 139 | ) 140 | LOGGER.error("Query returned error: %s: %s: %s", query, values, err) 141 | finally: 142 | # Wake up the thread waiting on the execution of the select 143 | # query. 144 | self._select_events.setdefault(token, threading.Event()) 145 | self._select_events[token].set() 146 | else: 147 | try: 148 | self._sqlite3_cursor.execute(query, values) 149 | except sqlite3.Error as err: 150 | LOGGER.error("Query returned error: %s: %s: %s", query, values, err) 151 | 152 | def close(self): 153 | """Close down the thread.""" 154 | with self._close_lock: 155 | if not self.is_alive(): 156 | LOGGER.debug("Already Closed") 157 | return "Already Closed" 158 | self._close_event.set() 159 | # Put a value in the queue to push through the block waiting for 160 | # items in the queue. 161 | self._sql_queue.put(("", "", ""), timeout=5) 162 | # Check that the thread is done before returning. 163 | self.join() 164 | 165 | @property 166 | def queue_size(self): 167 | """Return the queue size.""" 168 | return self._sql_queue.qsize() 169 | 170 | def _query_results(self, token): 171 | """Get the query results for a specific token. 172 | 173 | Args: 174 | token: A uuid object of the query you want returned. 175 | 176 | Returns: 177 | Return the results of the query when it's executed by the thread. 178 | """ 179 | try: 180 | # Wait until the select query has executed 181 | self._select_events.setdefault(token, threading.Event()) 182 | self._select_events[token].wait() 183 | return self._results[token] 184 | finally: 185 | self._select_events[token].clear() 186 | del self._results[token] 187 | del self._select_events[token] 188 | 189 | def execute(self, query, values=None): 190 | """Execute a query. 191 | 192 | Args: 193 | query: The sql string using ? for placeholders of dynamic values. 194 | values: A tuple of values to be replaced into the ? of the query. 195 | 196 | Returns: 197 | If it's a select query it will return the results of the query. 198 | """ 199 | if self._close_event.is_set(): 200 | LOGGER.debug("Close set, not running: %s", query) 201 | return "Close Called" 202 | LOGGER.debug("execute: %s", query) 203 | values = values or [] 204 | # A token to track this query with. 205 | token = str(uuid.uuid4()) 206 | self._sql_queue.put((token, query, values), timeout=5) 207 | # If it's a select we queue it up with a token to mark the results 208 | # into the output queue so we know what results are ours. 209 | if query.lower().strip().startswith("select"): 210 | return self._query_results(token) 211 | -------------------------------------------------------------------------------- /sqlite3worker_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright (c) 2014 Palantir Technologies 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | """sqlite3worker test routines.""" 24 | 25 | __author__ = "Shawn Lee" 26 | __email__ = "dashawn@gmail.com" 27 | __license__ = "MIT" 28 | 29 | import os 30 | import tempfile 31 | import threading 32 | import time 33 | import uuid 34 | 35 | import unittest 36 | 37 | import sqlite3worker 38 | 39 | 40 | class Sqlite3WorkerTests(unittest.TestCase): # pylint:disable=R0904 41 | """Test out the sqlite3worker library.""" 42 | 43 | def setUp(self): # pylint:disable=D0102 44 | self.tmp_file = tempfile.NamedTemporaryFile( 45 | suffix="pytest", prefix="sqlite" 46 | ).name 47 | self.sqlite3worker = sqlite3worker.Sqlite3Worker(self.tmp_file) 48 | # Create sql db. 49 | self.sqlite3worker.execute( 50 | "CREATE TABLE tester (timestamp DATETIME, uuid TEXT)" 51 | ) 52 | 53 | def tearDown(self): # pylint:disable=D0102 54 | self.sqlite3worker.close() 55 | os.unlink(self.tmp_file) 56 | 57 | def test_bad_select(self): 58 | """Test a bad select query.""" 59 | query = "select THIS IS BAD SQL" 60 | self.assertIn( 61 | self.sqlite3worker.execute(query), 62 | [ 63 | "Query returned error: select THIS IS BAD SQL: []: no such column: THIS", 64 | "Query returned error: select THIS IS BAD SQL: []: no such column: BAD", 65 | ], 66 | ) 67 | 68 | def test_bad_insert(self): 69 | """Test a bad insert query.""" 70 | query = "insert THIS IS BAD SQL" 71 | self.sqlite3worker.execute(query) 72 | # Give it one second to clear the queue. 73 | if self.sqlite3worker.queue_size != 0: 74 | time.sleep(1) 75 | self.assertEqual(self.sqlite3worker.queue_size, 0) 76 | self.assertEqual(self.sqlite3worker.execute("SELECT * from tester"), []) 77 | 78 | def test_valid_insert(self): 79 | """Test a valid insert and select statement.""" 80 | self.sqlite3worker.execute( 81 | "INSERT into tester values (?, ?)", ("2010-01-01 13:00:00", "bow") 82 | ) 83 | self.assertEqual( 84 | self.sqlite3worker.execute("SELECT * from tester"), 85 | [("2010-01-01 13:00:00", "bow")], 86 | ) 87 | self.sqlite3worker.execute( 88 | "INSERT into tester values (?, ?)", ("2011-02-02 14:14:14", "dog") 89 | ) 90 | # Give it one second to clear the queue. 91 | if self.sqlite3worker.queue_size != 0: 92 | time.sleep(1) 93 | self.assertEqual( 94 | self.sqlite3worker.execute("SELECT * from tester"), 95 | [("2010-01-01 13:00:00", "bow"), ("2011-02-02 14:14:14", "dog")], 96 | ) 97 | 98 | def test_run_after_close(self): 99 | """Test to make sure all events are cleared after object closed.""" 100 | self.sqlite3worker.close() 101 | self.sqlite3worker.execute( 102 | "INSERT into tester values (?, ?)", ("2010-01-01 13:00:00", "bow") 103 | ) 104 | self.assertEqual( 105 | self.sqlite3worker.execute("SELECT * from tester"), "Close Called" 106 | ) 107 | 108 | def test_double_close(self): 109 | """Make sure double closeing messages properly.""" 110 | self.sqlite3worker.close() 111 | self.assertEqual(self.sqlite3worker.close(), "Already Closed") 112 | 113 | def test_db_closed_propertly(self): 114 | """Make sure sqlite object is properly closed out.""" 115 | self.sqlite3worker.close() 116 | with self.assertRaises(self.sqlite3worker._sqlite3_conn.ProgrammingError): 117 | self.sqlite3worker._sqlite3_conn.total_changes 118 | 119 | def test_many_threads(self): 120 | """Make sure lots of threads work together.""" 121 | 122 | class threaded(threading.Thread): 123 | def __init__(self, sqlite_obj): 124 | threading.Thread.__init__(self, name=__name__) 125 | self.sqlite_obj = sqlite_obj 126 | self.daemon = True 127 | self.failed = False 128 | self.completed = False 129 | self.start() 130 | 131 | def run(self): 132 | for _ in range(5): 133 | token = str(uuid.uuid4()) 134 | self.sqlite_obj.execute( 135 | "INSERT into tester values (?, ?)", 136 | ("2010-01-01 13:00:00", token), 137 | ) 138 | resp = self.sqlite_obj.execute( 139 | "SELECT * from tester where uuid = ?", (token,) 140 | ) 141 | if resp != [("2010-01-01 13:00:00", token)]: 142 | self.failed = True 143 | break 144 | self.completed = True 145 | 146 | threads = [] 147 | for _ in range(5): 148 | threads.append(threaded(self.sqlite3worker)) 149 | 150 | for i in range(5): 151 | while not threads[i].completed: 152 | time.sleep(0.1) 153 | self.assertEqual(threads[i].failed, False) 154 | threads[i].join() 155 | 156 | 157 | if __name__ == "__main__": 158 | unittest.main() 159 | --------------------------------------------------------------------------------