├── .gitignore ├── CodernityDB3.egg-info ├── PKG-INFO ├── SOURCES.txt ├── dependency_links.txt └── top_level.txt ├── CodernityDB3 ├── __init__.py ├── database.py ├── database_gevent.py ├── database_safe_shared.py ├── database_super_thread_safe.py ├── database_thread_safe.py ├── debug_stuff.py ├── env.py ├── hash_index.py ├── index.py ├── indexcreator.py ├── lfu_cache.py ├── lfu_cache_with_lock.py ├── migrate.py ├── misc.py ├── patch.py ├── rr_cache.py ├── rr_cache_with_lock.py ├── sharded_hash.py ├── sharded_index.py ├── storage.py └── tree_index.py ├── MANIFEST.in ├── README.rst ├── pytest.ini ├── python3_codernity_duplicates.py ├── python3_codernity_get_query.py ├── python3_codernity_insert_save_store.py ├── python3_codernity_ordered.py ├── python3_codernity_update_delete.py ├── requirements.txt ├── setup.py ├── tests ├── __init__.py ├── conftest.py ├── hash_tests.py ├── index_files │ ├── 03md5_index.py │ ├── 04withA_index.py │ └── 05custom_hash_index.py ├── misc │ └── words.txt ├── patch.py ├── shard_tests.py ├── shared.py ├── test_db.py ├── test_db_super_thread_safe.py ├── test_db_thread_safe.py ├── test_indexcreator_db.py ├── test_indexcreator_exec.py └── tree_tests.py ├── tox_cov.ini └── tox_no_cov.ini /.gitignore: -------------------------------------------------------------------------------- 1 | # Python and IDE extensions 2 | *.py[cod] 3 | *.wpu 4 | *.wpr 5 | *.wpr2 6 | *.komodoproject 7 | /.komodotools/ 8 | .spyderproject 9 | .idea/ 10 | 11 | # Source control folders 12 | .svn 13 | .hg 14 | 15 | # Database file extensions 16 | *.db 17 | *.dbm 18 | *.sql 19 | *.sqlite 20 | *.sqlite3 21 | *-journal 22 | 23 | # Output file extensions 24 | *.bson 25 | #*.html 26 | #*.json 27 | *.ltsv 28 | *.odt 29 | *.rst 30 | *.tex 31 | #*.txt 32 | #*.xml 33 | #*.yml 34 | *.zip 35 | 36 | # Backup file extensions 37 | *.bak 38 | *.back 39 | *.backup 40 | 41 | # Mac files 42 | .DS_Store 43 | 44 | # Build directories 45 | */dist/ 46 | */build/ 47 | */*/build/ 48 | */*/_build/ 49 | 50 | # Log files 51 | *.log 52 | 53 | # Temp files 54 | _tmp.txt 55 | tests/_tmp.txt 56 | 57 | # C files generated by the Cython test. 58 | *.c 59 | 60 | # TLD names database. 61 | thirdparty_libs/tldextract/.tld_set 62 | 63 | # User configuration file. 64 | user.conf 65 | 66 | # Test data for the HTML report. 67 | plugins/report/html_report/js/database.js 68 | 69 | # Testing files 70 | .coverage 71 | htmlcov/ -------------------------------------------------------------------------------- /CodernityDB3.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 1.1 2 | Name: CodernityDB3 3 | Version: 0.4.2 4 | Summary: Python 3 port of pure python, fast, schema-less, NoSQL database 5 | Home-page: http://labs.codernity.com/codernitydb 6 | Author: cr0hn 7 | Author-email: cr0hn@cr0hn.com 8 | License: Apache 2.0 9 | Description: This repository 10 | =============== 11 | 12 | 13 | This is an intent **to port CodernityDB to Python 3**, from de [original source](http://labs.codernity.com/codernitydb), written for Python 2.x, 14 | 15 | CodernityDB is opensource, pure python (no 3rd party dependency), fast (really fast check Speed in documentation if you don't believe in words), multiplatform, schema-less, NoSQL_ database. It has optional support for HTTP server version (CodernityDB-HTTP), and also Python client library (CodernityDB-PyClient) that aims to be 100% compatible with embeded version. 16 | 17 | **Although this port is a beta** yet, it works in the basic usage cases. 18 | 19 | Calling for help 20 | ================ 21 | 22 | Any help to port CodernityDB to Python 3 is wellcome. It's a hard works. 23 | 24 | Feel free to clone the repo an send any patch. 25 | 26 | Status 27 | ====== 28 | 29 | Following the official examples, I was able to determinate: 30 | 31 | - Insert/Save: Works oks! 32 | - Get query: Works oks! 33 | - Duplicates: Works oks! 34 | - Update/delete: Doesn't works yet. 35 | - Ordered data: Doesn't works yet. 36 | 37 | 38 | Ported examples 39 | =============== 40 | 41 | There he ported examples: 42 | 43 | Insert/Save 44 | ----------- 45 | 46 | .. code-block:: python 47 | 48 | from CodernityDB3.database import Database 49 | 50 | def main(): 51 | db = Database('/tmp/tut1') 52 | db.create() 53 | for x in range(100): 54 | print(db.insert(dict(x=x))) 55 | for curr in db.all('id'): 56 | print(curr) 57 | 58 | main() 59 | 60 | 61 | Get query 62 | --------- 63 | 64 | .. code-block:: python 65 | 66 | from CodernityDB3.database import Database 67 | from CodernityDB3.hash_index import HashIndex 68 | 69 | 70 | class WithXIndex(HashIndex): 71 | 72 | def __init__(self, *args, **kwargs): 73 | kwargs['key_format'] = 'I' 74 | super(WithXIndex, self).__init__(*args, **kwargs) 75 | 76 | def make_key_value(self, data): 77 | a_val = data.get("x") 78 | if a_val is not None: 79 | return a_val, None 80 | return None 81 | 82 | def make_key(self, key): 83 | return key 84 | 85 | 86 | def main(): 87 | db = Database('/tmp/tut2') 88 | db.create() 89 | x_ind = WithXIndex(db.path, 'x') 90 | db.add_index(x_ind) 91 | 92 | for x in range(100): 93 | db.insert(dict(x=x)) 94 | 95 | for y in range(100): 96 | db.insert(dict(y=y)) 97 | 98 | print(db.get('x', 10, with_doc=True)) 99 | 100 | if __name__ == '__main__': 101 | main() 102 | 103 | 104 | Duplicates 105 | ---------- 106 | 107 | .. code-block:: python 108 | 109 | from CodernityDB3.database import Database 110 | from CodernityDB3.hash_index import HashIndex 111 | 112 | 113 | class WithXIndex(HashIndex): 114 | 115 | def __init__(self, *args, **kwargs): 116 | kwargs['key_format'] = 'I' 117 | super(WithXIndex, self).__init__(*args, **kwargs) 118 | 119 | def make_key_value(self, data): 120 | a_val = data.get("x") 121 | if a_val is not None: 122 | return a_val, None 123 | return None 124 | 125 | def make_key(self, key): 126 | return key 127 | 128 | 129 | def main(): 130 | db = Database('/tmp/tut3') 131 | db.create() 132 | x_ind = WithXIndex(db.path, 'x') 133 | db.add_index(x_ind) 134 | 135 | for x in range(100): 136 | db.insert(dict(x=x)) 137 | 138 | for x in range(100): 139 | db.insert(dict(x=x)) 140 | 141 | for y in range(100): 142 | db.insert(dict(y=y)) 143 | 144 | print(db.get('x', 10, with_doc=True)) 145 | for curr in db.get_many('x', 10, limit=-1, with_doc=True): 146 | print(curr) 147 | 148 | if __name__ == '__main__': 149 | main() 150 | 151 | 152 | 153 | Update/delete 154 | ------------- 155 | 156 | .. code-block:: python 157 | 158 | from CodernityDB3.database import Database 159 | from CodernityDB3.tree_index import TreeBasedIndex 160 | 161 | 162 | class WithXIndex(TreeBasedIndex): 163 | 164 | def __init__(self, *args, **kwargs): 165 | kwargs['node_capacity'] = 10 166 | kwargs['key_format'] = 'I' 167 | super(WithXIndex, self).__init__(*args, **kwargs) 168 | 169 | def make_key_value(self, data): 170 | t_val = data.get('x') 171 | if t_val is not None: 172 | return t_val, None 173 | return None 174 | 175 | def make_key(self, key): 176 | return key 177 | 178 | 179 | def main(): 180 | db = Database('/tmp/tut_update') 181 | db.create() 182 | x_ind = WithXIndex(db.path, 'x') 183 | db.add_index(x_ind) 184 | 185 | # full examples so we had to add first the data 186 | # the same code as in previous step 187 | 188 | for x in range(100): 189 | db.insert(dict(x=x)) 190 | 191 | for y in range(100): 192 | db.insert(dict(y=y)) 193 | 194 | # end of insert part 195 | 196 | print(db.count(db.all, 'x')) 197 | 198 | for curr in db.all('x', with_doc=True): 199 | doc = curr['doc'] 200 | if curr['key'] % 7 == 0: 201 | db.delete(doc) 202 | elif curr['key'] % 5 == 0: 203 | doc['updated'] = True 204 | db.update(doc) 205 | 206 | print(db.count(db.all, 'x')) 207 | 208 | for curr in db.all('x', with_doc=True): 209 | print(curr) 210 | 211 | if __name__ == '__main__': 212 | main() 213 | 214 | 215 | Ordered 216 | ------- 217 | 218 | .. code-block:: python 219 | 220 | from CodernityDB3.database import Database 221 | from CodernityDB3.tree_index import TreeBasedIndex 222 | 223 | 224 | class WithXIndex(TreeBasedIndex): 225 | 226 | def __init__(self, *args, **kwargs): 227 | kwargs['node_capacity'] = 10 228 | kwargs['key_format'] = 'I' 229 | super(WithXXIndex, self).__init__(*args, **kwargs) 230 | 231 | def make_key_value(self, data): 232 | t_val = data.get('x') 233 | if t_val is not None: 234 | return t_val, data 235 | return None 236 | 237 | def make_key(self, key): 238 | return key 239 | 240 | 241 | def main(): 242 | db = Database('/tmp/tut4') 243 | db.create() 244 | x_ind = WithXIndex(db.path, 'x') 245 | db.add_index(x_ind) 246 | 247 | for x in range(11): 248 | db.insert(dict(x=x)) 249 | 250 | for y in range(11): 251 | db.insert(dict(y=y)) 252 | 253 | print(db.get('x', 10, with_doc=True)) 254 | 255 | for curr in db.get_many('x', start=15, end=25, limit=-1, with_doc=True): 256 | print(curr) 257 | 258 | 259 | if __name__ == '__main__': 260 | main() 261 | 262 | 263 | Keywords: database python nosql key-value key/value db 264 | Platform: any 265 | Classifier: License :: OSI Approved :: Apache Software License 266 | Classifier: Programming Language :: Python 267 | Classifier: Programming Language :: Python :: 3.2 268 | Classifier: Programming Language :: Python :: 3.4 269 | Classifier: Operating System :: OS Independent 270 | Classifier: Topic :: Internet 271 | Classifier: Topic :: Database 272 | Classifier: Topic :: Software Development 273 | Classifier: Intended Audience :: Developers 274 | Classifier: Development Status :: 4 - Beta 275 | -------------------------------------------------------------------------------- /CodernityDB3.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | MANIFEST.in 2 | README.rst 3 | setup.py 4 | CodernityDB3/__init__.py 5 | CodernityDB3/database.py 6 | CodernityDB3/database_gevent.py 7 | CodernityDB3/database_safe_shared.py 8 | CodernityDB3/database_super_thread_safe.py 9 | CodernityDB3/database_thread_safe.py 10 | CodernityDB3/debug_stuff.py 11 | CodernityDB3/env.py 12 | CodernityDB3/hash_index.py 13 | CodernityDB3/index.py 14 | CodernityDB3/indexcreator.py 15 | CodernityDB3/lfu_cache.py 16 | CodernityDB3/lfu_cache_with_lock.py 17 | CodernityDB3/migrate.py 18 | CodernityDB3/misc.py 19 | CodernityDB3/patch.py 20 | CodernityDB3/rr_cache.py 21 | CodernityDB3/rr_cache_with_lock.py 22 | CodernityDB3/sharded_hash.py 23 | CodernityDB3/sharded_index.py 24 | CodernityDB3/storage.py 25 | CodernityDB3/tree_index.py 26 | CodernityDB3.egg-info/PKG-INFO 27 | CodernityDB3.egg-info/SOURCES.txt 28 | CodernityDB3.egg-info/dependency_links.txt 29 | CodernityDB3.egg-info/top_level.txt -------------------------------------------------------------------------------- /CodernityDB3.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /CodernityDB3.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | CodernityDB3 2 | -------------------------------------------------------------------------------- /CodernityDB3/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | __version__ = '0.4.2' 20 | __license__ = "Apache 2.0" 21 | -------------------------------------------------------------------------------- /CodernityDB3/database_gevent.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import absolute_import 19 | from gevent.lock import RLock 20 | 21 | from CodernityDB3.env import cdb_environment 22 | 23 | cdb_environment['mode'] = "gevent" 24 | cdb_environment['rlock_obj'] = RLock 25 | 26 | 27 | # from CodernityDB3.database import Database 28 | from CodernityDB3.database_safe_shared import SafeDatabase 29 | 30 | 31 | class GeventDatabase(SafeDatabase): 32 | pass 33 | -------------------------------------------------------------------------------- /CodernityDB3/database_safe_shared.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import absolute_import 19 | import six 20 | 21 | from CodernityDB3.env import cdb_environment 22 | from CodernityDB3.database import PreconditionsException, RevConflict, Database 23 | # from database import Database 24 | 25 | from collections import defaultdict 26 | from functools import wraps 27 | from types import MethodType 28 | 29 | 30 | class th_safe_gen: 31 | 32 | def __init__(self, name, gen, l=None): 33 | self.lock = l 34 | self.__gen = gen 35 | self.name = name 36 | 37 | def __iter__(self): 38 | return self 39 | 40 | def __next__(self): 41 | with self.lock: 42 | return next(self.__gen) 43 | 44 | @staticmethod 45 | def wrapper(method, index_name, meth_name, l=None): 46 | @wraps(method) 47 | def _inner(*args, **kwargs): 48 | res = method(*args, **kwargs) 49 | return th_safe_gen(index_name + "_" + meth_name, res, l) 50 | return _inner 51 | 52 | 53 | def safe_wrapper(method, lock): 54 | @wraps(method) 55 | def _inner(*args, **kwargs): 56 | with lock: 57 | return method(*args, **kwargs) 58 | return _inner 59 | 60 | 61 | class SafeDatabase(Database): 62 | 63 | def __init__(self, path, *args, **kwargs): 64 | super(SafeDatabase, self).__init__(path, *args, **kwargs) 65 | self.indexes_locks = defaultdict( 66 | lambda: cdb_environment['rlock_obj']()) 67 | self.close_open_lock = cdb_environment['rlock_obj']() 68 | self.main_lock = cdb_environment['rlock_obj']() 69 | self.id_revs = {} 70 | 71 | def __patch_index_gens(self, name): 72 | ind = self.indexes_names[name] 73 | for c in ('all', 'get_many'): 74 | m = getattr(ind, c) 75 | if getattr(ind, c + "_orig", None): 76 | return 77 | m_fixed = th_safe_gen.wrapper(m, name, c, self.indexes_locks[name]) 78 | setattr(ind, c, m_fixed) 79 | setattr(ind, c + '_orig', m) 80 | 81 | def __patch_index_methods(self, name): 82 | ind = self.indexes_names[name] 83 | lock = self.indexes_locks[name] 84 | for curr in dir(ind): 85 | meth = getattr(ind, curr) 86 | if not curr.startswith('_') and isinstance(meth, MethodType): 87 | setattr(ind, curr, safe_wrapper(meth, lock)) 88 | stor = ind.storage 89 | for curr in dir(stor): 90 | meth = getattr(stor, curr) 91 | if not curr.startswith('_') and isinstance(meth, MethodType): 92 | setattr(stor, curr, safe_wrapper(meth, lock)) 93 | 94 | def __patch_index(self, name): 95 | self.__patch_index_methods(name) 96 | self.__patch_index_gens(name) 97 | 98 | def initialize(self, *args, **kwargs): 99 | with self.close_open_lock: 100 | self.close_open_lock.acquire() 101 | res = super(SafeDatabase, self).initialize(*args, **kwargs) 102 | for name in self.indexes_names.keys(): 103 | self.indexes_locks[name] = cdb_environment['rlock_obj']() 104 | return res 105 | 106 | def open(self, *args, **kwargs): 107 | with self.close_open_lock: 108 | res = super(SafeDatabase, self).open(*args, **kwargs) 109 | for name in self.indexes_names.keys(): 110 | self.indexes_locks[name] = cdb_environment['rlock_obj']() 111 | self.__patch_index(name) 112 | return res 113 | 114 | def create(self, *args, **kwargs): 115 | with self.close_open_lock: 116 | res = super(SafeDatabase, self).create(*args, **kwargs) 117 | for name in self.indexes_names.keys(): 118 | self.indexes_locks[name] = cdb_environment['rlock_obj']() 119 | self.__patch_index(name) 120 | return res 121 | 122 | def close(self): 123 | with self.close_open_lock: 124 | return super(SafeDatabase, self).close() 125 | 126 | def destroy(self): 127 | with self.close_open_lock: 128 | return super(SafeDatabase, self).destroy() 129 | 130 | def add_index(self, *args, **kwargs): 131 | with self.main_lock: 132 | res = super(SafeDatabase, self).add_index(*args, **kwargs) 133 | if self.opened: 134 | self.indexes_locks[res] = cdb_environment['rlock_obj']() 135 | self.__patch_index(res) 136 | return res 137 | 138 | def _single_update_index(self, index, data, db_data, doc_id): 139 | with self.indexes_locks[index.name]: 140 | super(SafeDatabase, self)._single_update_index( 141 | index, data, db_data, doc_id) 142 | 143 | def _single_delete_index(self, index, data, doc_id, old_data): 144 | with self.indexes_locks[index.name]: 145 | super(SafeDatabase, self)._single_delete_index( 146 | index, data, doc_id, old_data) 147 | 148 | def edit_index(self, *args, **kwargs): 149 | with self.main_lock: 150 | res = super(SafeDatabase, self).edit_index(*args, **kwargs) 151 | if self.opened: 152 | self.indexes_locks[res] = cdb_environment['rlock_obj']() 153 | self.__patch_index(res) 154 | return res 155 | 156 | def set_indexes(self, *args, **kwargs): 157 | try: 158 | self.main_lock.acquire() 159 | super(SafeDatabase, self).set_indexes(*args, **kwargs) 160 | finally: 161 | self.main_lock.release() 162 | 163 | def reindex_index(self, index, *args, **kwargs): 164 | if isinstance(index, six.string_types): 165 | if not index in self.indexes_names: 166 | raise PreconditionsException("No index named %s" % index) 167 | index = self.indexes_names[index] 168 | key = index.name + "reind" 169 | self.main_lock.acquire() 170 | if key in self.indexes_locks: 171 | lock = self.indexes_locks[index.name + "reind"] 172 | else: 173 | self.indexes_locks[index.name + 174 | "reind"] = cdb_environment['rlock_obj']() 175 | lock = self.indexes_locks[index.name + "reind"] 176 | self.main_lock.release() 177 | try: 178 | lock.acquire() 179 | super(SafeDatabase, self).reindex_index( 180 | index, *args, **kwargs) 181 | finally: 182 | lock.release() 183 | 184 | def flush(self): 185 | try: 186 | self.main_lock.acquire() 187 | super(SafeDatabase, self).flush() 188 | finally: 189 | self.main_lock.release() 190 | 191 | def fsync(self): 192 | try: 193 | self.main_lock.acquire() 194 | super(SafeDatabase, self).fsync() 195 | finally: 196 | self.main_lock.release() 197 | 198 | def _update_id_index(self, _rev, data): 199 | with self.indexes_locks['id']: 200 | return super(SafeDatabase, self)._update_id_index(_rev, data) 201 | 202 | def _delete_id_index(self, _id, _rev, data): 203 | with self.indexes_locks['id']: 204 | return super(SafeDatabase, self)._delete_id_index(_id, _rev, data) 205 | 206 | def _update_indexes(self, _rev, data): 207 | _id, new_rev, db_data = self._update_id_index(_rev, data) 208 | with self.main_lock: 209 | self.id_revs[_id] = new_rev 210 | for index in self.indexes[1:]: 211 | with self.main_lock: 212 | curr_rev = self.id_revs.get(_id) # get last _id, _rev 213 | if curr_rev != new_rev: 214 | break # new update on the way stop current 215 | self._single_update_index(index, data, db_data, _id) 216 | with self.main_lock: 217 | if self.id_revs[_id] == new_rev: 218 | del self.id_revs[_id] 219 | return _id, new_rev 220 | 221 | def _delete_indexes(self, _id, _rev, data): 222 | old_data = self.get('id', _id) 223 | if old_data['_rev'] != _rev: 224 | raise RevConflict() 225 | with self.main_lock: 226 | self.id_revs[_id] = _rev 227 | for index in self.indexes[1:]: 228 | self._single_delete_index(index, data, _id, old_data) 229 | self._delete_id_index(_id, _rev, data) 230 | with self.main_lock: 231 | if self.id_revs[_id] == _rev: 232 | del self.id_revs[_id] 233 | -------------------------------------------------------------------------------- /CodernityDB3/database_super_thread_safe.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from __future__ import absolute_import 20 | from threading import RLock 21 | 22 | from CodernityDB3.env import cdb_environment 23 | 24 | cdb_environment['mode'] = "threads" 25 | cdb_environment['rlock_obj'] = RLock 26 | 27 | from .database import Database 28 | 29 | from functools import wraps 30 | from types import FunctionType, MethodType 31 | 32 | from CodernityDB3.database_safe_shared import th_safe_gen 33 | 34 | 35 | class SuperLock(type): 36 | 37 | @staticmethod 38 | def wrapper(f): 39 | @wraps(f) 40 | def _inner(*args, **kwargs): 41 | db = args[0] 42 | with db.super_lock: 43 | # print('=>', f.__name__, repr(args[1:])) 44 | res = f(*args, **kwargs) 45 | # if db.opened: 46 | # db.flush() 47 | # print('<=', f.__name__, repr(args[1:])) 48 | return res 49 | return _inner 50 | 51 | def __new__(cls, classname, bases, attr): 52 | new_attr = {} 53 | for base in bases: 54 | for b_attr in dir(base): 55 | a = getattr(base, b_attr, None) 56 | if isinstance(a, MethodType) and not b_attr.startswith('_'): 57 | if b_attr == 'flush' or b_attr == 'flush_indexes': 58 | pass 59 | else: 60 | # setattr(base, b_attr, SuperLock.wrapper(a)) 61 | new_attr[b_attr] = SuperLock.wrapper(a) 62 | for attr_name, attr_value in attr.items(): 63 | if isinstance(attr_value, FunctionType) and not attr_name.startswith('_'): 64 | attr_value = SuperLock.wrapper(attr_value) 65 | new_attr[attr_name] = attr_value 66 | new_attr['super_lock'] = RLock() 67 | return type.__new__(cls, classname, bases, new_attr) 68 | 69 | 70 | class SuperThreadSafeDatabase(Database, metaclass=SuperLock): 71 | """ 72 | Thread safe version that always allows single thread to use db. 73 | It adds the same lock for all methods, so only one operation can be 74 | performed in given time. Completely different implementation 75 | than ThreadSafe version (without super word) 76 | """ 77 | 78 | def __init__(self, *args, **kwargs): 79 | super(SuperThreadSafeDatabase, self).__init__(*args, **kwargs) 80 | 81 | def __patch_index_gens(self, name): 82 | ind = self.indexes_names[name] 83 | for c in ('all', 'get_many'): 84 | m = getattr(ind, c) 85 | if getattr(ind, c + "_orig", None): 86 | return 87 | m_fixed = th_safe_gen.wrapper(m, name, c, self.super_lock) 88 | setattr(ind, c, m_fixed) 89 | setattr(ind, c + '_orig', m) 90 | 91 | def open(self, *args, **kwargs): 92 | res = super(SuperThreadSafeDatabase, self).open(*args, **kwargs) 93 | for name in self.indexes_names.keys(): 94 | self.__patch_index_gens(name) 95 | return res 96 | 97 | def create(self, *args, **kwargs): 98 | res = super(SuperThreadSafeDatabase, self).create(*args, **kwargs) 99 | for name in self.indexes_names.keys(): 100 | self.__patch_index_gens(name) 101 | return res 102 | 103 | def add_index(self, *args, **kwargs): 104 | res = super(SuperThreadSafeDatabase, self).add_index(*args, **kwargs) 105 | self.__patch_index_gens(res) 106 | return res 107 | 108 | def edit_index(self, *args, **kwargs): 109 | res = super(SuperThreadSafeDatabase, self).edit_index(*args, **kwargs) 110 | self.__patch_index_gens(res) 111 | return res 112 | -------------------------------------------------------------------------------- /CodernityDB3/database_thread_safe.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import absolute_import 19 | from threading import RLock 20 | 21 | from CodernityDB3.env import cdb_environment 22 | 23 | cdb_environment['mode'] = "threads" 24 | cdb_environment['rlock_obj'] = RLock 25 | 26 | 27 | from .database_safe_shared import SafeDatabase 28 | 29 | 30 | class ThreadSafeDatabase(SafeDatabase): 31 | """ 32 | Thread safe version of CodernityDB that uses several lock objects, 33 | on different methods / different indexes etc. It's completely different 34 | implementation of locking than SuperThreadSafe one. 35 | """ 36 | pass 37 | -------------------------------------------------------------------------------- /CodernityDB3/debug_stuff.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import absolute_import 19 | from __future__ import print_function 20 | from CodernityDB3.tree_index import TreeBasedIndex 21 | import struct 22 | import os 23 | 24 | import inspect 25 | from functools import wraps 26 | import json 27 | from six.moves import range 28 | from io import open 29 | import codecs 30 | 31 | 32 | class DebugTreeBasedIndex(TreeBasedIndex): 33 | 34 | def __init__(self, *args, **kwargs): 35 | super(DebugTreeBasedIndex, self).__init__(*args, **kwargs) 36 | 37 | def print_tree(self): 38 | print('-----CURRENT TREE-----') 39 | print((self.root_flag)) 40 | 41 | if self.root_flag == 'l': 42 | print('---ROOT---') 43 | self._print_leaf_data(self.data_start) 44 | return 45 | else: 46 | print('---ROOT---') 47 | self._print_node_data(self.data_start) 48 | nr_of_el, children_flag = self._read_node_nr_of_elements_and_children_flag( 49 | self.data_start) 50 | nodes = [] 51 | for index in range(nr_of_el): 52 | l_pointer, key, r_pointer = self._read_single_node_key( 53 | self.data_start, index) 54 | nodes.append(l_pointer) 55 | nodes.append(r_pointer) 56 | print(('ROOT NODES', nodes)) 57 | while children_flag == 'n': 58 | self._print_level(nodes, 'n') 59 | new_nodes = [] 60 | for node in nodes: 61 | nr_of_el, children_flag = \ 62 | self._read_node_nr_of_elements_and_children_flag(node) 63 | for index in range(nr_of_el): 64 | l_pointer, key, r_pointer = self._read_single_node_key( 65 | node, index) 66 | new_nodes.append(l_pointer) 67 | new_nodes.append(r_pointer) 68 | nodes = new_nodes 69 | self._print_level(nodes, 'l') 70 | 71 | def _print_level(self, nodes, flag): 72 | print('---NEXT LVL---') 73 | if flag == 'n': 74 | for node in nodes: 75 | self._print_node_data(node) 76 | elif flag == 'l': 77 | for node in nodes: 78 | self._print_leaf_data(node) 79 | 80 | def _print_leaf_data(self, leaf_start_position): 81 | print(('printing data of leaf at', leaf_start_position)) 82 | nr_of_elements = self._read_leaf_nr_of_elements(leaf_start_position) 83 | self.buckets.seek(leaf_start_position) 84 | data = self.buckets.read(self.leaf_heading_size + 85 | nr_of_elements * self.single_leaf_record_size) 86 | leaf = struct.unpack('<' + self.leaf_heading_format + 87 | nr_of_elements * self.single_leaf_record_format, data) 88 | print(leaf) 89 | print('') 90 | 91 | def _print_node_data(self, node_start_position): 92 | print(('printing data of node at', node_start_position)) 93 | nr_of_elements = self._read_node_nr_of_elements_and_children_flag( 94 | node_start_position)[0] 95 | self.buckets.seek(node_start_position) 96 | data = self.buckets.read(self.node_heading_size + self.pointer_size 97 | + nr_of_elements * (self.key_size + self.pointer_size)) 98 | node = struct.unpack('<' + self.node_heading_format + self.pointer_format 99 | + nr_of_elements * ( 100 | self.key_format + self.pointer_format), 101 | data) 102 | print(node) 103 | print('') 104 | # ------------------> 105 | 106 | 107 | def database_step_by_step(db_obj, path=None): 108 | 109 | if not path: 110 | # ugly for multiplatform support.... 111 | p = db_obj.path 112 | p1 = os.path.split(p) 113 | p2 = os.path.split(p1[0]) 114 | p3 = '_'.join([p2[1], 'operation_logger.log']) 115 | path = os.path.join(os.path.split(p2[0])[0], p3) 116 | f_obj = open(path, 'wb') 117 | 118 | __stack = [] # inspect.stack() is not working on pytest etc 119 | 120 | def remove_from_stack(name): 121 | for i in range(len(__stack)): 122 | if __stack[-i] == name: 123 | __stack.pop(-i) 124 | 125 | def __dumper(f): 126 | @wraps(f) 127 | def __inner(*args, **kwargs): 128 | funct_name = f.__name__ 129 | if funct_name == 'count': 130 | name = args[0].__name__ 131 | meth_args = (name,) + args[1:] 132 | elif funct_name in ('reindex_index', 'compact_index'): 133 | name = args[0].name 134 | meth_args = (name,) + args[1:] 135 | else: 136 | meth_args = args 137 | kwargs_copy = kwargs.copy() 138 | res = None 139 | __stack.append(funct_name) 140 | if funct_name == 'insert': 141 | try: 142 | res = f(*args, **kwargs) 143 | except: 144 | packed = json.dumps((funct_name, 145 | meth_args, kwargs_copy, None)) 146 | f_obj.write(codecs.encode('%s\n' % packed, 'utf-8')) 147 | f_obj.flush() 148 | raise 149 | else: 150 | packed = json.dumps((funct_name, 151 | meth_args, kwargs_copy, res)) 152 | f_obj.write(bytes('%s\n' % packed, 'utf-8')) 153 | f_obj.flush() 154 | else: 155 | if funct_name == 'get': 156 | for curr in __stack: 157 | if ('delete' in curr or 'update' in curr) and not curr.startswith('test'): 158 | remove_from_stack(funct_name) 159 | return f(*args, **kwargs) 160 | packed = json.dumps((funct_name, meth_args, kwargs_copy)) 161 | f_obj.write(bytes('%s\n' % packed, 'utf-8')) 162 | f_obj.flush() 163 | res = f(*args, **kwargs) 164 | remove_from_stack(funct_name) 165 | return res 166 | return __inner 167 | 168 | for meth_name, meth_f in inspect.getmembers(db_obj, predicate=inspect.ismethod): 169 | if not meth_name.startswith('_'): 170 | setattr(db_obj, meth_name, __dumper(meth_f)) 171 | 172 | setattr(db_obj, 'operation_logger', f_obj) 173 | 174 | 175 | def database_from_steps(db_obj, path): 176 | # db_obj.insert=lambda data : insert_for_debug(db_obj, data) 177 | with open(path, 'rb') as f_obj: 178 | for current in f_obj: 179 | line = json.loads(current[:-1]) 180 | if line[0] == 'count': 181 | obj = getattr(db_obj, line[1][0]) 182 | line[1] = [obj] + line[1][1:] 183 | name = line[0] 184 | if name == 'insert': 185 | try: 186 | line[1][0].pop('_rev') 187 | except: 188 | pass 189 | elif name in ('delete', 'update'): 190 | el = db_obj.get('id', line[1][0]['_id']) 191 | line[1][0]['_rev'] = el['_rev'] 192 | # print('FROM STEPS doing', line) 193 | meth = getattr(db_obj, line[0], None) 194 | if not meth: 195 | raise Exception("Method = `%s` not found" % line[0]) 196 | 197 | meth(*line[1], **line[2]) 198 | 199 | 200 | # def insert_for_debug(self, data): 201 | # 202 | # _rev = data['_rev'] 203 | # 204 | # if not '_id' in data: 205 | # _id = uuid4().hex 206 | # else: 207 | # _id = data['_id'] 208 | # data['_id'] = _id 209 | # try: 210 | # _id = bytes(_id) 211 | # except: 212 | # raise DatabaseException("`_id` must be valid bytes object") 213 | # self._insert_indexes(_id, _rev, data) 214 | # ret = {'_id': _id, '_rev': _rev} 215 | # data.update(ret) 216 | # return ret 217 | -------------------------------------------------------------------------------- /CodernityDB3/env.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | """ 19 | It's CodernityDB environment. 20 | Handles internal informations.' 21 | """ 22 | 23 | cdb_environment = { 24 | 'mode': 'normal' 25 | } 26 | -------------------------------------------------------------------------------- /CodernityDB3/index.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from __future__ import absolute_import 20 | import os 21 | import marshal 22 | 23 | import struct 24 | import shutil 25 | 26 | from CodernityDB3.storage import IU_Storage, DummyStorage 27 | 28 | try: 29 | from CodernityDB3 import __version__ 30 | except ImportError: 31 | from .__init__ import __version__ 32 | 33 | 34 | import io 35 | 36 | 37 | class IndexException(Exception): 38 | pass 39 | 40 | 41 | class IndexNotFoundException(IndexException): 42 | pass 43 | 44 | 45 | class ReindexException(IndexException): 46 | pass 47 | 48 | 49 | class TryReindexException(ReindexException): 50 | pass 51 | 52 | 53 | class ElemNotFound(IndexException): 54 | pass 55 | 56 | 57 | class DocIdNotFound(ElemNotFound): 58 | pass 59 | 60 | 61 | class IndexConflict(IndexException): 62 | pass 63 | 64 | 65 | class IndexPreconditionsException(IndexException): 66 | pass 67 | 68 | 69 | class Index(object): 70 | 71 | __version__ = __version__ 72 | 73 | custom_header = "" # : use it for imports required by your index 74 | 75 | def __init__(self, 76 | db_path, 77 | name): 78 | if isinstance(name, bytes): 79 | name = name.decode() 80 | if isinstance(db_path, bytes): 81 | db_path = db_path.decode() 82 | 83 | self.name = name 84 | self._start_ind = 500 85 | self.db_path = db_path 86 | 87 | def open_index(self): 88 | if not os.path.isfile(os.path.join(self.db_path, self.name + '_buck')): 89 | raise IndexException("Doesn't exists") 90 | self.buckets = io.open( 91 | os.path.join(self.db_path, self.name + "_buck"), 'r+b', buffering=0) 92 | self._fix_params() 93 | self._open_storage() 94 | 95 | def _close(self): 96 | self.buckets.close() 97 | self.storage.close() 98 | 99 | def close_index(self): 100 | self.flush() 101 | self.fsync() 102 | self._close() 103 | 104 | def create_index(self): 105 | raise NotImplementedError() 106 | 107 | def _fix_params(self): 108 | self.buckets.seek(0) 109 | props = marshal.loads(self.buckets.read(self._start_ind)) 110 | for k, v in props.items(): 111 | self.__dict__[k] = v 112 | self.buckets.seek(0, 2) 113 | 114 | def _save_params(self, in_params={}): 115 | self.buckets.seek(0) 116 | props = marshal.loads(self.buckets.read(self._start_ind)) 117 | props.update(in_params) 118 | self.buckets.seek(0) 119 | data = marshal.dumps(props) 120 | if len(data) > self._start_ind: 121 | raise IndexException("To big props") 122 | self.buckets.write(data) 123 | self.flush() 124 | self.buckets.seek(0, 2) 125 | self.__dict__.update(props) 126 | 127 | def _open_storage(self, *args, **kwargs): 128 | pass 129 | 130 | def _create_storage(self, *args, **kwargs): 131 | pass 132 | 133 | def _destroy_storage(self, *args, **kwargs): 134 | self.storage.destroy() 135 | 136 | def _find_key(self, key): 137 | raise NotImplementedError() 138 | 139 | def update(self, doc_id, key, start, size): 140 | raise NotImplementedError() 141 | 142 | def insert(self, doc_id, key, start, size): 143 | raise NotImplementedError() 144 | 145 | def get(self, key): 146 | raise NotImplementedError() 147 | 148 | def get_many(self, key, start_from=None, limit=0): 149 | raise NotImplementedError() 150 | 151 | def all(self, start_pos): 152 | raise NotImplementedError() 153 | 154 | def delete(self, key, start, size): 155 | raise NotImplementedError() 156 | 157 | def make_key_value(self, data): 158 | raise NotImplementedError() 159 | 160 | def make_key(self, data): 161 | raise NotImplementedError() 162 | 163 | def compact(self, *args, **kwargs): 164 | raise NotImplementedError() 165 | 166 | def destroy(self, *args, **kwargs): 167 | self._close() 168 | bucket_file = os.path.join(self.db_path, self.name + '_buck') 169 | os.unlink(bucket_file) 170 | self._destroy_storage() 171 | self._find_key.clear() 172 | 173 | def flush(self): 174 | try: 175 | self.buckets.flush() 176 | self.storage.flush() 177 | except: 178 | pass 179 | 180 | def fsync(self): 181 | try: 182 | os.fsync(self.buckets.fileno()) 183 | self.storage.fsync() 184 | except: 185 | pass 186 | 187 | def update_with_storage(self, doc_id, key, value): 188 | if value: 189 | start, size = self.storage.insert(value) 190 | else: 191 | start = 1 192 | size = 0 193 | return self.update(doc_id, key, start, size) 194 | 195 | def insert_with_storage(self, doc_id, key, value): 196 | if value is not None: 197 | start, size = self.storage.insert(value) 198 | else: 199 | start = 1 200 | size = 0 201 | return self.insert(doc_id, key, start, size) 202 | -------------------------------------------------------------------------------- /CodernityDB3/indexcreator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from __future__ import absolute_import 20 | import re 21 | import tokenize 22 | import token 23 | import uuid 24 | from six.moves import range 25 | 26 | 27 | class IndexCreatorException(Exception): 28 | def __init__(self, ex, line=None): 29 | self.ex = ex 30 | self.line = line 31 | 32 | def __str__(self): 33 | if self.line: 34 | return repr(self.ex + "(in line: %d)" % self.line) 35 | return repr(self.ex) 36 | 37 | 38 | class IndexCreatorFunctionException(IndexCreatorException): 39 | pass 40 | 41 | 42 | class IndexCreatorValueException(IndexCreatorException): 43 | pass 44 | 45 | 46 | class Parser(object): 47 | def __init__(self): 48 | pass 49 | 50 | def parse(self, data, name=None): 51 | if not name: 52 | self.name = "_" + uuid.uuid4().hex 53 | else: 54 | self.name = name 55 | 56 | self.ind = 0 57 | self.stage = 0 58 | self.logic = ['and', 'or', 'in'] 59 | self.logic2 = ['&', '|'] 60 | self.allowed_props = { 61 | 'TreeBasedIndex': ['type', 'name', 'key_format', 'node_capacity', 'pointer_format', 'meta_format'], 62 | 'HashIndex': ['type', 'name', 'key_format', 'hash_lim', 'entry_line_format'], 63 | 'MultiHashIndex': ['type', 'name', 'key_format', 'hash_lim', 'entry_line_format'], 64 | 'MultiTreeBasedIndex': ['type', 'name', 'key_format', 'node_capacity', 'pointer_format', 'meta_format'] 65 | } 66 | self.funcs = { 67 | 'md5': (['md5'], ['.digest()']), 68 | 'len': (['len'], []), 69 | 'str': (['str'], []), 70 | 'fix_r': (['self.fix_r'], []), 71 | 'prefix': (['self.prefix'], []), 72 | 'infix': (['self.infix'], []), 73 | 'suffix': (['self.suffix'], []) 74 | } 75 | self.handle_int_imports = {'infix': "\n"} 76 | 77 | self.funcs_with_body = { 78 | 'fix_r': ( 79 | """ def fix_r(self,s,l): 80 | e = len(s) 81 | if e == l: 82 | return s 83 | elif e > l: 84 | return s[:l] 85 | else: 86 | return s.rjust(l,b'_')\n""", False), 87 | 88 | 'prefix': ( 89 | """ def prefix(self,s,m,l,f): 90 | t = len(s) 91 | if m < 1: 92 | m = 1 93 | o = set() 94 | if t > l: 95 | s = s[:l] 96 | t = l 97 | while m <= t: 98 | o.add(s.rjust(f,'_')) 99 | s = s[:-1] 100 | t -= 1 101 | return o\n""", False), 102 | 'suffix': ( 103 | """ def suffix(self,s,m,l,f): 104 | t = len(s) 105 | if m < 1: 106 | m = 1 107 | o = set() 108 | if t > l: 109 | s = s[t-l:] 110 | t = len(s) 111 | while m <= t: 112 | o.add(s.rjust(f,'_')) 113 | s = s[1:] 114 | t -= 1 115 | return o\n""", False), 116 | 'infix': ( 117 | """ def infix(self,s,m,l,f): 118 | t = len(s) 119 | o = set() 120 | for x in range(m - 1, l): 121 | t = (s, ) 122 | for y in range(0, x): 123 | t += (s[y + 1:],) 124 | o.update(set(''.join(x).rjust(f, '_').lower() for x in zip(*t))) 125 | return o\n""", False)} 126 | self.none = ['None', 'none', 'null'] 127 | self.props_assign = ['=', ':'] 128 | self.all_adj_num_comp = {token.NUMBER: ( 129 | token.NUMBER, token.NAME, '-', '('), 130 | token.NAME: (token.NUMBER, token.NAME, '-', '('), 131 | ')': (token.NUMBER, token.NAME, '-', '(') 132 | } 133 | 134 | self.all_adj_num_op = {token.NUMBER: (token.NUMBER, token.NAME, '('), 135 | token.NAME: (token.NUMBER, token.NAME, '('), 136 | ')': (token.NUMBER, token.NAME, '(') 137 | } 138 | self.allowed_adjacent = { 139 | "<=": self.all_adj_num_comp, 140 | ">=": self.all_adj_num_comp, 141 | ">": self.all_adj_num_comp, 142 | "<": self.all_adj_num_comp, 143 | 144 | "==": {token.NUMBER: (token.NUMBER, token.NAME, '('), 145 | token.NAME: (token.NUMBER, token.NAME, token.STRING, '('), 146 | token.STRING: (token.NAME, token.STRING, '('), 147 | ')': (token.NUMBER, token.NAME, token.STRING, '('), 148 | ']': (token.NUMBER, token.NAME, token.STRING, '(') 149 | }, 150 | 151 | "+": {token.NUMBER: (token.NUMBER, token.NAME, '('), 152 | token.NAME: (token.NUMBER, token.NAME, token.STRING, '('), 153 | token.STRING: (token.NAME, token.STRING, '('), 154 | ')': (token.NUMBER, token.NAME, token.STRING, '('), 155 | ']': (token.NUMBER, token.NAME, token.STRING, '(') 156 | }, 157 | 158 | "-": {token.NUMBER: (token.NUMBER, token.NAME, '('), 159 | token.NAME: (token.NUMBER, token.NAME, '('), 160 | ')': (token.NUMBER, token.NAME, '('), 161 | '<': (token.NUMBER, token.NAME, '('), 162 | '>': (token.NUMBER, token.NAME, '('), 163 | '<=': (token.NUMBER, token.NAME, '('), 164 | '>=': (token.NUMBER, token.NAME, '('), 165 | '==': (token.NUMBER, token.NAME, '('), 166 | ']': (token.NUMBER, token.NAME, '(') 167 | }, 168 | "*": self.all_adj_num_op, 169 | "/": self.all_adj_num_op, 170 | "%": self.all_adj_num_op, 171 | ",": {token.NUMBER: (token.NUMBER, token.NAME, token.STRING, '{', '[', '('), 172 | token.NAME: (token.NUMBER, token.NAME, token.STRING, '(', '{', '['), 173 | token.STRING: (token.NAME, token.STRING, token.NUMBER, '(', '{', '['), 174 | ')': (token.NUMBER, token.NAME, token.STRING, '(', '{', '['), 175 | ']': (token.NUMBER, token.NAME, token.STRING, '(', '{', '['), 176 | '}': (token.NUMBER, token.NAME, token.STRING, '(', '{', '[') 177 | } 178 | } 179 | 180 | def is_num(s): 181 | m = re.search('[^0-9*()+\-\s/]+', s) 182 | return not m 183 | 184 | def is_string(s): 185 | m = re.search('\s*(?P[\'\"]+).*?(?P=a)\s*', s) 186 | return m 187 | 188 | data = re.split('make_key_value\:', data) 189 | 190 | if len(data) < 2: 191 | raise IndexCreatorFunctionException( 192 | "Couldn't find a definition of make_key_value function!\n") 193 | 194 | spl1 = re.split('make_key\:', data[0]) 195 | spl2 = re.split('make_key\:', data[1]) 196 | 197 | self.funcs_rev = False 198 | 199 | if len(spl1) > 1: 200 | data = [spl1[0]] + [data[1]] + [spl1[1]] 201 | self.funcs_rev = True 202 | elif len(spl2) > 1: 203 | data = [data[0]] + spl2 204 | else: 205 | data.append("key") 206 | 207 | if data[1] == re.search('\s*', data[1], re.S | re.M).group(0): 208 | raise IndexCreatorFunctionException( 209 | "Empty function body ", 210 | len(re.split('\n', data[0])) + (len(re.split('\n', data[2])) if self.funcs_rev else 1) - 1) 211 | if data[2] == re.search('\s*', data[2], re.S | re.M).group(0): 212 | raise IndexCreatorFunctionException( 213 | "Empty function body ", 214 | len(re.split('\n', data[0])) + (1 if self.funcs_rev else len(re.split('\n', data[1]))) - 1) 215 | if data[0] == re.search('\s*', data[0], re.S | re.M).group(0): 216 | raise IndexCreatorValueException("You didn't set any properity or you set them not at the begining of the code\n") 217 | 218 | data = [re.split( 219 | '\n', data[0]), re.split('\n', data[1]), re.split('\n', data[2])] 220 | self.cnt_lines = (len(data[0]), len(data[1]), len(data[2])) 221 | ind = 0 222 | self.predata = data 223 | self.data = [[], [], []] 224 | for i, v in enumerate(self.predata[0]): 225 | for k, w in enumerate(self.predata[0][i]): 226 | if self.predata[0][i][k] in self.props_assign: 227 | if not is_num(self.predata[0][i][k + 1:]) and self.predata[0][i].strip()[:4] != 'type' and self.predata[0][i].strip()[:4] != 'name': 228 | s = self.predata[0][i][k + 1:] 229 | self.predata[0][i] = self.predata[0][i][:k + 1] 230 | 231 | m = re.search('\s+', s.strip()) 232 | if not is_string(s) and not m: 233 | s = "'" + s.strip() + "'" 234 | self.predata[0][i] += s 235 | break 236 | 237 | for n, i in enumerate(self.predata): 238 | for k in i: 239 | k = k.strip() 240 | if k: 241 | self.data[ind].append(k) 242 | self.check_enclosures(k, n) 243 | ind += 1 244 | 245 | return self.parse_ex() 246 | 247 | def readline(self, stage): 248 | def foo(): 249 | if len(self.data[stage]) <= self.ind: 250 | self.ind = 0 251 | return "" 252 | else: 253 | self.ind += 1 254 | return self.data[stage][self.ind - 1] 255 | return foo 256 | 257 | def add(self, l, i): 258 | def add_aux(*args): 259 | # print(args,self.ind) 260 | if len(l[i]) < self.ind: 261 | l[i].append([]) 262 | l[i][self.ind - 1].append(args) 263 | return add_aux 264 | 265 | def parse_ex(self): 266 | self.index_name = "" 267 | self.index_type = "" 268 | self.curLine = -1 269 | self.con = -1 270 | self.brackets = -1 271 | self.curFunc = None 272 | self.colons = 0 273 | self.line_cons = ([], [], []) 274 | self.pre_tokens = ([], [], []) 275 | self.known_dicts_in_mkv = [] 276 | self.prop_name = True 277 | self.prop_assign = False 278 | self.is_one_arg_enough = False 279 | self.funcs_stack = [] 280 | self.last_line = [-1, -1, -1] 281 | self.props_set = [] 282 | self.custom_header = set() 283 | 284 | self.tokens = [] 285 | self.tokens_head = [ 286 | '# %s\n' % self.name, 287 | 'class %s(' % self.name, '):\n', 288 | ' def __init__(self, *args, **kwargs): '] 289 | 290 | for i in range(3): 291 | for tolkien in tokenize.generate_tokens(self.readline(i)): 292 | t = tuple(tolkien) 293 | self.add(self.pre_tokens, i)(*t) 294 | # tokenize treats some keyword not in the right way, thats why we 295 | # have to change some of them 296 | for nk, k in enumerate(self.pre_tokens[i]): 297 | for na, a in enumerate(k): 298 | if a[0] == token.NAME and a[1] in self.logic: 299 | self.pre_tokens[i][nk][na] = ( 300 | token.OP, a[1], a[2], a[3], a[4]) 301 | 302 | for i in self.pre_tokens[1]: 303 | self.line_cons[1].append(self.check_colons(i, 1)) 304 | self.check_adjacents(i, 1) 305 | if self.check_for_2nd_arg(i) == -1 and not self.is_one_arg_enough: 306 | raise IndexCreatorValueException( 307 | "No 2nd value to return (did u forget about ',None'?", 308 | self.cnt_line_nr(i[0][4], 1)) 309 | self.is_one_arg_enough = False 310 | 311 | for i in self.pre_tokens[2]: 312 | self.line_cons[2].append(self.check_colons(i, 2)) 313 | self.check_adjacents(i, 2) 314 | 315 | for i in self.pre_tokens[0]: 316 | self.handle_prop_line(i) 317 | 318 | self.cur_brackets = 0 319 | self.tokens += ['\n super(%s, self).__init__(*args, **kwargs)\n def make_key_value(self, data): ' % self.name] 320 | 321 | for i in self.pre_tokens[1]: 322 | for k in i: 323 | self.handle_make_value(*k) 324 | 325 | self.curLine = -1 326 | self.con = -1 327 | self.cur_brackets = 0 328 | self.tokens += ['\n def make_key(self, key):'] 329 | 330 | for i in self.pre_tokens[2]: 331 | for k in i: 332 | self.handle_make_key(*k) 333 | 334 | if self.index_type == "": 335 | raise IndexCreatorValueException("Missing index type definition\n") 336 | if self.index_name == "": 337 | raise IndexCreatorValueException("Missing index name\n") 338 | 339 | self.tokens_head[0] = "# " + self.index_name + "\n" + \ 340 | self.tokens_head[0] 341 | 342 | for i in self.funcs_with_body: 343 | if self.funcs_with_body[i][1]: 344 | self.tokens_head.insert(4, self.funcs_with_body[i][0]) 345 | 346 | if None in self.custom_header: 347 | self.custom_header.remove(None) 348 | if self.custom_header: 349 | s = ' custom_header = """' 350 | for i in self.custom_header: 351 | s += i 352 | s += '"""\n' 353 | self.tokens_head.insert(4, s) 354 | 355 | if self.index_type in self.allowed_props: 356 | for i in self.props_set: 357 | if i not in self.allowed_props[self.index_type]: 358 | raise IndexCreatorValueException("Properity %s is not allowed for index type: %s" % (i, self.index_type)) 359 | 360 | # print("".join(self.tokens_head)) 361 | # print("----------") 362 | # print(" ".join(self.tokens)) 363 | return "".join(self.custom_header), "".join(self.tokens_head) + (" ".join(self.tokens)) 364 | 365 | # has to be run BEFORE tokenize 366 | def check_enclosures(self, d, st): 367 | encs = [] 368 | contr = {'(': ')', '{': '}', '[': ']', "'": "'", '"': '"'} 369 | ends = [')', '}', ']', "'", '"'] 370 | for i in d: 371 | if len(encs) > 0 and encs[-1] in ['"', "'"]: 372 | if encs[-1] == i: 373 | del encs[-1] 374 | elif i in contr: 375 | encs += [i] 376 | elif i in ends: 377 | if len(encs) < 1 or contr[encs[-1]] != i: 378 | raise IndexCreatorValueException("Missing opening enclosure for \'%s\'" % i, self.cnt_line_nr(d, st)) 379 | del encs[-1] 380 | 381 | if len(encs) > 0: 382 | raise IndexCreatorValueException("Missing closing enclosure for \'%s\'" % encs[0], self.cnt_line_nr(d, st)) 383 | 384 | def check_adjacents(self, d, st): 385 | def std_check(d, n): 386 | if n == 0: 387 | prev = -1 388 | else: 389 | prev = d[n - 1][1] if d[n - 1][0] == token.OP else d[n - 1][0] 390 | 391 | cur = d[n][1] if d[n][0] == token.OP else d[n][0] 392 | 393 | # there always is an endmarker at the end, but this is a precaution 394 | if n + 2 > len(d): 395 | nex = -1 396 | else: 397 | nex = d[n + 1][1] if d[n + 1][0] == token.OP else d[n + 1][0] 398 | 399 | if prev not in self.allowed_adjacent[cur]: 400 | raise IndexCreatorValueException("Wrong left value of the %s" % cur, self.cnt_line_nr(line, st)) 401 | 402 | # there is an assumption that whole data always ends with 0 marker, the idea prolly needs a rewritting to allow more whitespaces 403 | # between tokens, so it will be handled anyway 404 | elif nex not in self.allowed_adjacent[cur][prev]: 405 | raise IndexCreatorValueException("Wrong right value of the %s" % cur, self.cnt_line_nr(line, st)) 406 | 407 | for n, (t, i, _, _, line) in enumerate(d): 408 | if t == token.NAME or t == token.STRING: 409 | if n + 1 < len(d) and d[n + 1][0] in [token.NAME, token.STRING]: 410 | raise IndexCreatorValueException("Did you forget about an operator in between?", self.cnt_line_nr(line, st)) 411 | elif i in self.allowed_adjacent: 412 | std_check(d, n) 413 | 414 | def check_colons(self, d, st): 415 | cnt = 0 416 | br = 0 417 | 418 | def check_ret_args_nr(a, s): 419 | c_b_cnt = 0 420 | s_b_cnt = 0 421 | n_b_cnt = 0 422 | comas_cnt = 0 423 | for _, i, _, _, line in a: 424 | 425 | if c_b_cnt == n_b_cnt == s_b_cnt == 0: 426 | if i == ',': 427 | comas_cnt += 1 428 | if (s == 1 and comas_cnt > 1) or (s == 2 and comas_cnt > 0): 429 | raise IndexCreatorFunctionException("Too much arguments to return", self.cnt_line_nr(line, st)) 430 | if s == 0 and comas_cnt > 0: 431 | raise IndexCreatorValueException("A coma here doesn't make any sense", self.cnt_line_nr(line, st)) 432 | 433 | elif i == ':': 434 | if s == 0: 435 | raise IndexCreatorValueException("A colon here doesn't make any sense", self.cnt_line_nr(line, st)) 436 | raise IndexCreatorFunctionException("Two colons don't make any sense", self.cnt_line_nr(line, st)) 437 | 438 | if i == '{': 439 | c_b_cnt += 1 440 | elif i == '}': 441 | c_b_cnt -= 1 442 | elif i == '(': 443 | n_b_cnt += 1 444 | elif i == ')': 445 | n_b_cnt -= 1 446 | elif i == '[': 447 | s_b_cnt += 1 448 | elif i == ']': 449 | s_b_cnt -= 1 450 | 451 | def check_if_empty(a): 452 | for i in a: 453 | if i not in [token.NEWLINE, token.INDENT, token.ENDMARKER]: 454 | return False 455 | return True 456 | if st == 0: 457 | check_ret_args_nr(d, st) 458 | return 459 | 460 | for n, i in enumerate(d): 461 | if i[1] == ':': 462 | if br == 0: 463 | if len(d) < n or check_if_empty(d[n + 1:]): 464 | raise IndexCreatorValueException( 465 | "Empty return value", self.cnt_line_nr(i[4], st)) 466 | elif len(d) >= n: 467 | check_ret_args_nr(d[n + 1:], st) 468 | return cnt 469 | else: 470 | cnt += 1 471 | elif i[1] == '{': 472 | br += 1 473 | elif i[1] == '}': 474 | br -= 1 475 | check_ret_args_nr(d, st) 476 | return -1 477 | 478 | def check_for_2nd_arg(self, d): 479 | c_b_cnt = 0 # curly brackets counter '{}' 480 | s_b_cnt = 0 # square brackets counter '[]' 481 | n_b_cnt = 0 # normal brackets counter '()' 482 | 483 | def check_2nd_arg(d, ind): 484 | d = d[ind[0]:] 485 | for t, i, (n, r), _, line in d: 486 | if i == '{' or i is None: 487 | return 0 488 | elif t == token.NAME: 489 | self.known_dicts_in_mkv.append((i, (n, r))) 490 | return 0 491 | elif t == token.STRING or t == token.NUMBER: 492 | raise IndexCreatorValueException("Second return value of make_key_value function has to be a dictionary!", self.cnt_line_nr(line, 1)) 493 | 494 | for ind in enumerate(d): 495 | t, i, _, _, _ = ind[1] 496 | if s_b_cnt == n_b_cnt == c_b_cnt == 0: 497 | if i == ',': 498 | return check_2nd_arg(d, ind) 499 | elif (t == token.NAME and i not in self.funcs) or i == '{': 500 | self.is_one_arg_enough = True 501 | 502 | if i == '{': 503 | c_b_cnt += 1 504 | self.is_one_arg_enough = True 505 | elif i == '}': 506 | c_b_cnt -= 1 507 | elif i == '(': 508 | n_b_cnt += 1 509 | elif i == ')': 510 | n_b_cnt -= 1 511 | elif i == '[': 512 | s_b_cnt += 1 513 | elif i == ']': 514 | s_b_cnt -= 1 515 | return -1 516 | 517 | def cnt_line_nr(self, l, stage): 518 | nr = -1 519 | for n, i in enumerate(self.predata[stage]): 520 | # print(i,"|||",i.strip(),"|||",l) 521 | if l == i.strip(): 522 | nr = n 523 | if nr == -1: 524 | return -1 525 | 526 | if stage == 0: 527 | return nr + 1 528 | elif stage == 1: 529 | return nr + self.cnt_lines[0] + (self.cnt_lines[2] - 1 if self.funcs_rev else 0) 530 | elif stage == 2: 531 | return nr + self.cnt_lines[0] + (self.cnt_lines[1] - 1 if not self.funcs_rev else 0) 532 | 533 | return -1 534 | 535 | def handle_prop_line(self, d): 536 | d_len = len(d) 537 | if d[d_len - 1][0] == token.ENDMARKER: 538 | d_len -= 1 539 | 540 | if d_len < 3: 541 | raise IndexCreatorValueException("Can't handle properity assingment ", self.cnt_line_nr(d[0][4], 0)) 542 | 543 | if not d[1][1] in self.props_assign: 544 | raise IndexCreatorValueException( 545 | "Did you forget : or =?", self.cnt_line_nr(d[0][4], 0)) 546 | 547 | if d[0][0] == token.NAME or d[0][0] == token.STRING: 548 | if d[0][1] in self.props_set: 549 | raise IndexCreatorValueException("Properity %s is set more than once" % d[0][1], self.cnt_line_nr(d[0][4], 0)) 550 | self.props_set += [d[0][1]] 551 | if d[0][1] == "type" or d[0][1] == "name": 552 | t, tk, _, _, line = d[2] 553 | 554 | if d_len > 3: 555 | raise IndexCreatorValueException( 556 | "Wrong value to assign", self.cnt_line_nr(line, 0)) 557 | 558 | if t == token.STRING: 559 | m = re.search('\s*(?P[\'\"]+)(.*?)(?P=a)\s*', tk) 560 | if m: 561 | tk = m.groups()[1] 562 | elif t != token.NAME: 563 | raise IndexCreatorValueException( 564 | "Wrong value to assign", self.cnt_line_nr(line, 0)) 565 | 566 | if d[0][1] == "type": 567 | if d[2][1] == "TreeBasedIndex": 568 | self.custom_header.add("from CodernityDB3.tree_index import TreeBasedIndex\n") 569 | elif d[2][1] == "MultiTreeBasedIndex": 570 | self.custom_header.add("from CodernityDB3.tree_index import MultiTreeBasedIndex\n") 571 | elif d[2][1] == "MultiHashIndex": 572 | self.custom_header.add("from CodernityDB3.hash_index import MultiHashIndex\n") 573 | self.tokens_head.insert(2, tk) 574 | self.index_type = tk 575 | else: 576 | self.index_name = tk 577 | return 578 | else: 579 | self.tokens += ['\n kwargs["' + d[0][1] + '"]'] 580 | else: 581 | raise IndexCreatorValueException("Can't handle properity assingment ", self.cnt_line_nr(d[0][4], 0)) 582 | 583 | self.tokens += ['='] 584 | 585 | self.check_adjacents(d[2:], 0) 586 | self.check_colons(d[2:], 0) 587 | 588 | for i in d[2:]: 589 | self.tokens += [i[1]] 590 | 591 | def generate_func(self, t, tk, pos_start, pos_end, line, hdata, stage): 592 | if self.last_line[stage] != -1 and pos_start[0] > self.last_line[stage] and line != '': 593 | raise IndexCreatorFunctionException("This line will never be executed!", self.cnt_line_nr(line, stage)) 594 | if t == 0: 595 | return 596 | 597 | if pos_start[1] == 0: 598 | if self.line_cons[stage][pos_start[0] - 1] == -1: 599 | self.tokens += ['\n return'] 600 | self.last_line[stage] = pos_start[0] 601 | else: 602 | self.tokens += ['\n if'] 603 | elif tk == ':' and self.line_cons[stage][pos_start[0] - 1] > -1: 604 | if self.line_cons[stage][pos_start[0] - 1] == 0: 605 | self.tokens += [':\n return'] 606 | return 607 | self.line_cons[stage][pos_start[0] - 1] -= 1 608 | 609 | if tk in self.logic2: 610 | # print(tk) 611 | if line[pos_start[1] - 1] != tk and line[pos_start[1] + 1] != tk: 612 | self.tokens += [tk] 613 | if line[pos_start[1] - 1] != tk and line[pos_start[1] + 1] == tk: 614 | if tk == '&': 615 | self.tokens += ['and'] 616 | else: 617 | self.tokens += ['or'] 618 | return 619 | 620 | if self.brackets != 0: 621 | def search_through_known_dicts(a): 622 | for i, (n, r) in self.known_dicts_in_mkv: 623 | if i == tk and r > pos_start[1] and n == pos_start[0] and hdata == 'data': 624 | return True 625 | return False 626 | 627 | if t == token.NAME and len(self.funcs_stack) > 0 and self.funcs_stack[-1][0] == 'md5' and search_through_known_dicts(tk): 628 | raise IndexCreatorValueException("Second value returned by make_key_value for sure isn't a dictionary ", self.cnt_line_nr(line, 1)) 629 | 630 | if tk == ')': 631 | self.cur_brackets -= 1 632 | if len(self.funcs_stack) > 0 and self.cur_brackets == self.funcs_stack[-1][1]: 633 | self.tokens += [tk] 634 | self.tokens += self.funcs[self.funcs_stack[-1][0]][1] 635 | del self.funcs_stack[-1] 636 | return 637 | if tk == '(': 638 | self.cur_brackets += 1 639 | 640 | if tk in self.none: 641 | self.tokens += ['None'] 642 | return 643 | 644 | if t == token.NAME and tk not in self.logic and tk != hdata: 645 | if tk not in self.funcs: 646 | self.tokens += [hdata + '["' + tk + '"]'] 647 | else: 648 | self.tokens += self.funcs[tk][0] 649 | if tk in self.funcs_with_body: 650 | self.funcs_with_body[tk] = ( 651 | self.funcs_with_body[tk][0], True) 652 | self.custom_header.add(self.handle_int_imports.get(tk)) 653 | self.funcs_stack += [(tk, self.cur_brackets)] 654 | else: 655 | self.tokens += [tk] 656 | 657 | def handle_make_value(self, t, tk, pos_start, pos_end, line): 658 | self.generate_func(t, tk, pos_start, pos_end, line, 'data', 1) 659 | 660 | def handle_make_key(self, t, tk, pos_start, pos_end, line): 661 | self.generate_func(t, tk, pos_start, pos_end, line, 'key', 2) 662 | -------------------------------------------------------------------------------- /CodernityDB3/lfu_cache.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from __future__ import absolute_import 20 | import six 21 | import functools 22 | from heapq import nsmallest 23 | from operator import itemgetter 24 | from collections import defaultdict 25 | 26 | try: 27 | from collections import Counter 28 | except ImportError: 29 | class Counter(dict): 30 | 'Mapping where default values are zero' 31 | def __missing__(self, key): 32 | return 0 33 | 34 | 35 | def cache1lvl(maxsize=100): 36 | """ 37 | modified version of http://code.activestate.com/recipes/498245/ 38 | """ 39 | def decorating_function(user_function): 40 | cache = {} 41 | use_count = Counter() 42 | 43 | @functools.wraps(user_function) 44 | def wrapper(key, *args, **kwargs): 45 | # print('lfu_cache1lvl.wrapper %r' % key) 46 | if isinstance(key, six.text_type): 47 | key = key.encode() 48 | try: 49 | result = cache[key] 50 | except KeyError: 51 | if len(cache) == maxsize: 52 | for k, _ in nsmallest(maxsize // 10 or 1, 53 | iter(use_count.items()), 54 | key=itemgetter(1)): 55 | del cache[k], use_count[k] 56 | cache[key] = user_function(key, *args, **kwargs) 57 | result = cache[key] 58 | # result = user_function(obj, key, *args, **kwargs) 59 | finally: 60 | use_count[key] += 1 61 | return result 62 | 63 | def clear(): 64 | cache.clear() 65 | use_count.clear() 66 | 67 | def delete(key): 68 | # print('lfu_cache1lvl.delete %r' % key) 69 | if isinstance(key, six.text_type): 70 | key = key.encode() 71 | try: 72 | del cache[key] 73 | del use_count[key] 74 | except KeyError: 75 | return False 76 | else: 77 | return True 78 | 79 | wrapper.clear = clear 80 | wrapper.cache = cache 81 | wrapper.delete = delete 82 | return wrapper 83 | return decorating_function 84 | 85 | 86 | def twolvl_iterator(dict): 87 | for k, v in dict.items(): 88 | for kk, vv in v.items(): 89 | yield k, kk, vv 90 | 91 | 92 | def cache2lvl(maxsize=100): 93 | """ 94 | modified version of http://code.activestate.com/recipes/498245/ 95 | """ 96 | def decorating_function(user_function): 97 | cache = {} 98 | use_count = defaultdict(Counter) 99 | 100 | @functools.wraps(user_function) 101 | def wrapper(*args, **kwargs): 102 | # return user_function(*args, **kwargs) 103 | key = args[0] 104 | # print('cache2lvl.wrapper %r' % key) 105 | if isinstance(key, six.text_type): 106 | key = key.encode() 107 | try: 108 | result = cache[key][args[1]] 109 | except KeyError: 110 | if wrapper.cache_size == maxsize: 111 | to_delete = maxsize // 10 or 1 112 | for k1, k2, v in nsmallest(to_delete, 113 | twolvl_iterator(use_count), 114 | key=itemgetter(2)): 115 | del cache[k1][k2], use_count[k1][k2] 116 | if not cache[k1]: 117 | del cache[k1] 118 | del use_count[k1] 119 | wrapper.cache_size -= to_delete 120 | result = user_function(*args, **kwargs) 121 | try: 122 | cache[key][args[1]] = result 123 | except KeyError: 124 | cache[key] = {args[1]: result} 125 | wrapper.cache_size += 1 126 | finally: 127 | use_count[key][args[1]] += 1 128 | return result 129 | 130 | def clear(): 131 | cache.clear() 132 | use_count.clear() 133 | 134 | def delete(key, inner_key=None): 135 | # print('cache2lvl.delete %r' % key) 136 | if isinstance(key, six.text_type): 137 | key = key.encode() 138 | if inner_key is not None: 139 | try: 140 | del cache[key][inner_key] 141 | del use_count[key][inner_key] 142 | if not cache[key]: 143 | del cache[key] 144 | del use_count[key] 145 | wrapper.cache_size -= 1 146 | except KeyError: 147 | return False 148 | else: 149 | return True 150 | else: 151 | try: 152 | wrapper.cache_size -= len(cache[key]) 153 | del cache[key] 154 | del use_count[key] 155 | except KeyError: 156 | return False 157 | else: 158 | return True 159 | 160 | wrapper.clear = clear 161 | wrapper.cache = cache 162 | wrapper.delete = delete 163 | wrapper.cache_size = 0 164 | return wrapper 165 | return decorating_function 166 | -------------------------------------------------------------------------------- /CodernityDB3/lfu_cache_with_lock.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from __future__ import absolute_import 20 | import six 21 | import functools 22 | from heapq import nsmallest 23 | from operator import itemgetter 24 | from collections import defaultdict 25 | 26 | 27 | try: 28 | from collections import Counter 29 | except ImportError: 30 | class Counter(dict): 31 | 'Mapping where default values are zero' 32 | def __missing__(self, key): 33 | return 0 34 | 35 | 36 | def twolvl_iterator(dict): 37 | for k, v in dict.items(): 38 | for kk, vv in v.items(): 39 | yield k, kk, vv 40 | 41 | 42 | def create_cache1lvl(lock_obj): 43 | def cache1lvl(maxsize=100): 44 | """ 45 | modified version of http://code.activestate.com/recipes/498245/ 46 | """ 47 | def decorating_function(user_function): 48 | cache = {} 49 | use_count = Counter() 50 | lock = lock_obj() 51 | 52 | @functools.wraps(user_function) 53 | def wrapper(key, *args, **kwargs): 54 | # print('lfu_cache_with_lock.1lvl.wrapper %r' % key) 55 | if isinstance(key, six.text_type): 56 | key = key.encode() 57 | try: 58 | result = cache[key] 59 | except KeyError: 60 | with lock: 61 | if len(cache) == maxsize: 62 | for k, _ in nsmallest(maxsize // 10 or 1, 63 | iter(use_count.items()), 64 | key=itemgetter(1)): 65 | del cache[k], use_count[k] 66 | cache[key] = user_function(key, *args, **kwargs) 67 | result = cache[key] 68 | use_count[key] += 1 69 | else: 70 | with lock: 71 | use_count[key] += 1 72 | return result 73 | 74 | def clear(): 75 | cache.clear() 76 | use_count.clear() 77 | 78 | def delete(key): 79 | # print('lfu_cache_with_lock.1lvl.delete %r' % key) 80 | if isinstance(key, six.text_type): 81 | key = key.encode() 82 | try: 83 | del cache[key] 84 | del use_count[key] 85 | return True 86 | except KeyError: 87 | return False 88 | 89 | wrapper.clear = clear 90 | wrapper.cache = cache 91 | wrapper.delete = delete 92 | return wrapper 93 | return decorating_function 94 | return cache1lvl 95 | 96 | 97 | def create_cache2lvl(lock_obj): 98 | def cache2lvl(maxsize=100): 99 | """ 100 | modified version of http://code.activestate.com/recipes/498245/ 101 | """ 102 | def decorating_function(user_function): 103 | cache = {} 104 | use_count = defaultdict(Counter) 105 | lock = lock_obj() 106 | 107 | @functools.wraps(user_function) 108 | def wrapper(*args, **kwargs): 109 | key = args[0] 110 | # print('lfu_cache_with_lock.2lvl.delete %r' % key) 111 | if isinstance(key, six.text_type): 112 | key = key.encode() 113 | try: 114 | result = cache[key][args[1]] 115 | except KeyError: 116 | with lock: 117 | if wrapper.cache_size == maxsize: 118 | to_delete = maxsize / 10 or 1 119 | for k1, k2, v in nsmallest(to_delete, 120 | twolvl_iterator( 121 | use_count), 122 | key=itemgetter(2)): 123 | del cache[k1][k2], use_count[k1][k2] 124 | if not cache[k1]: 125 | del cache[k1] 126 | del use_count[k1] 127 | wrapper.cache_size -= to_delete 128 | result = user_function(*args, **kwargs) 129 | try: 130 | cache[key][args[1]] = result 131 | except KeyError: 132 | cache[key] = {args[1]: result} 133 | use_count[key][args[1]] += 1 134 | wrapper.cache_size += 1 135 | else: 136 | use_count[key][args[1]] += 1 137 | return result 138 | 139 | def clear(): 140 | cache.clear() 141 | use_count.clear() 142 | 143 | def delete(key, *args): 144 | # print('lfu_cache_with_lock.2lvl.delete %r' % key) 145 | if isinstance(key, six.text_type): 146 | key = key.encode() 147 | if args: 148 | try: 149 | del cache[key][args[0]] 150 | del use_count[key][args[0]] 151 | if not cache[key]: 152 | del cache[key] 153 | del use_count[key] 154 | wrapper.cache_size -= 1 155 | return True 156 | except KeyError: 157 | return False 158 | else: 159 | try: 160 | wrapper.cache_size -= len(cache[key]) 161 | del cache[key] 162 | del use_count[key] 163 | return True 164 | except KeyError: 165 | return False 166 | 167 | wrapper.clear = clear 168 | wrapper.cache = cache 169 | wrapper.delete = delete 170 | wrapper.cache_size = 0 171 | return wrapper 172 | return decorating_function 173 | return cache2lvl 174 | -------------------------------------------------------------------------------- /CodernityDB3/migrate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import absolute_import 19 | from CodernityDB3.database import Database 20 | import shutil 21 | import os 22 | 23 | 24 | def migrate(source, destination): 25 | """ 26 | Very basic for now 27 | """ 28 | dbs = Database(source) 29 | dbt = Database(destination) 30 | dbs.open() 31 | dbt.create() 32 | dbt.close() 33 | for curr in os.listdir(os.path.join(dbs.path, '_indexes')): 34 | if curr != '00id.py': 35 | shutil.copyfile(os.path.join(dbs.path, '_indexes', curr), 36 | os.path.join(dbt.path, '_indexes', curr)) 37 | dbt.open() 38 | for c in dbs.all('id'): 39 | del c['_rev'] 40 | dbt.insert(c) 41 | return True 42 | 43 | 44 | if __name__ == '__main__': 45 | import sys 46 | migrate(sys.argv[1], sys.argv[2]) 47 | -------------------------------------------------------------------------------- /CodernityDB3/misc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import absolute_import 19 | from random import getrandbits, randrange 20 | import uuid 21 | 22 | 23 | class NONE: 24 | """ 25 | It's inteded to be None but different, 26 | for internal use only! 27 | """ 28 | pass 29 | 30 | 31 | def random_hex_32(): 32 | return uuid.UUID(int=getrandbits(128), version=4).hex 33 | 34 | 35 | def random_hex_4(*args, **kwargs): 36 | return '%04x' % randrange(256 ** 2) 37 | -------------------------------------------------------------------------------- /CodernityDB3/patch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from __future__ import absolute_import 20 | from CodernityDB3.misc import NONE 21 | 22 | 23 | def __patch(obj, name, new): 24 | n = NONE() 25 | orig = getattr(obj, name, n) 26 | if orig is not n: 27 | if orig == new: 28 | raise Exception("Shouldn't happen, new and orig are the same") 29 | setattr(obj, name, new) 30 | return 31 | 32 | 33 | def patch_cache_lfu(lock_obj): 34 | """ 35 | Patnches cache mechanizm to be thread safe (gevent ones also) 36 | 37 | .. note:: 38 | 39 | It's internal CodernityDB mechanizm, it will be called when needed 40 | 41 | """ 42 | from . import lfu_cache 43 | from . import lfu_cache_with_lock 44 | lfu_lock1lvl = lfu_cache_with_lock.create_cache1lvl(lock_obj) 45 | lfu_lock2lvl = lfu_cache_with_lock.create_cache2lvl(lock_obj) 46 | __patch(lfu_cache, 'cache1lvl', lfu_lock1lvl) 47 | __patch(lfu_cache, 'cache2lvl', lfu_lock2lvl) 48 | 49 | 50 | def patch_cache_rr(lock_obj): 51 | """ 52 | Patches cache mechanizm to be thread safe (gevent ones also) 53 | 54 | .. note:: 55 | 56 | It's internal CodernityDB mechanizm, it will be called when needed 57 | 58 | """ 59 | from . import rr_cache 60 | from . import rr_cache_with_lock 61 | rr_lock1lvl = rr_cache_with_lock.create_cache1lvl(lock_obj) 62 | rr_lock2lvl = rr_cache_with_lock.create_cache2lvl(lock_obj) 63 | __patch(rr_cache, 'cache1lvl', rr_lock1lvl) 64 | __patch(rr_cache, 'cache2lvl', rr_lock2lvl) 65 | 66 | 67 | def patch_flush_fsync(db_obj): 68 | """ 69 | Will always execute index.fsync after index.flush. 70 | 71 | .. note:: 72 | 73 | It's for advanced users, use when you understand difference between `flush` and `fsync`, and when you definitely need that. 74 | 75 | It's important to call it **AFTER** database has all indexes etc (after db.create or db.open) 76 | 77 | Example usage:: 78 | 79 | ... 80 | db = Database('/tmp/patch_demo') 81 | db.create() 82 | patch_flush_fsync(db) 83 | ... 84 | 85 | """ 86 | 87 | def always_fsync(ind_obj): 88 | def _inner(): 89 | ind_obj.orig_flush() 90 | ind_obj.fsync() 91 | return _inner 92 | 93 | for index in db_obj.indexes: 94 | setattr(index, 'orig_flush', index.flush) 95 | setattr(index, 'flush', always_fsync(index)) 96 | 97 | setattr(db_obj, 'orig_flush', db_obj.flush) 98 | setattr(db_obj, 'flush', always_fsync(db_obj)) 99 | 100 | return 101 | -------------------------------------------------------------------------------- /CodernityDB3/rr_cache.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import absolute_import 19 | import six 20 | import functools 21 | from random import choice 22 | from six.moves import range 23 | 24 | 25 | def cache1lvl(maxsize=100): 26 | def decorating_function(user_function): 27 | cache1lvl = {} 28 | 29 | @functools.wraps(user_function) 30 | def wrapper(key, *args, **kwargs): 31 | # print('rr_cache.1lvl.wrapper %r %r' % (user_function.__name__, key)) 32 | if isinstance(key, six.text_type): 33 | key = key.encode() 34 | try: 35 | result = cache1lvl[key] 36 | except KeyError: 37 | if len(cache1lvl) == maxsize: 38 | for i in range(maxsize // 10 or 1): 39 | del cache1lvl[choice(list(cache1lvl.keys()))] 40 | cache1lvl[key] = user_function(key, *args, **kwargs) 41 | result = cache1lvl[key] 42 | return result 43 | 44 | def clear(): 45 | cache1lvl.clear() 46 | 47 | def delete(key): 48 | # print('rr_cache.1lvl.delete %r %r' % (user_function.__name__, key)) 49 | if isinstance(key, six.text_type): 50 | key = key.encode() 51 | try: 52 | del cache1lvl[key] 53 | return True 54 | except KeyError: 55 | return False 56 | 57 | wrapper.clear = clear 58 | wrapper.cache = cache1lvl 59 | wrapper.delete = delete 60 | return wrapper 61 | return decorating_function 62 | 63 | 64 | def cache2lvl(maxsize=100): 65 | def decorating_function(user_function): 66 | cache = {} 67 | 68 | @functools.wraps(user_function) 69 | def wrapper(*args, **kwargs): 70 | # return user_function(*args, **kwargs) 71 | key = args[0] 72 | # print('rr_cache.2lvl.wrapper %r %r' % (user_function.__name__, key)) 73 | if isinstance(key, six.text_type): 74 | key = key.encode() 75 | try: 76 | result = cache[key][args[1]] 77 | except KeyError: 78 | if wrapper.cache_size == maxsize: 79 | to_delete = maxsize // 10 or 1 80 | for i in range(to_delete): 81 | key1 = choice(list(cache.keys())) 82 | key2 = choice(list(cache[key1].keys())) 83 | del cache[key1][key2] 84 | if not cache[key1]: 85 | del cache[key1] 86 | wrapper.cache_size -= to_delete 87 | result = user_function(*args, **kwargs) 88 | try: 89 | cache[key][args[1]] = result 90 | except KeyError: 91 | cache[key] = {args[1]: result} 92 | wrapper.cache_size += 1 93 | return result 94 | 95 | def clear(): 96 | cache.clear() 97 | wrapper.cache_size = 0 98 | 99 | def delete(key, inner_key=None): 100 | # print('rr_cache.2lvl.delete %r %r' % (user_function.__name__, key)) 101 | if isinstance(key, six.text_type): 102 | key = key.encode() 103 | if inner_key: 104 | try: 105 | del cache[key][inner_key] 106 | if not cache[key]: 107 | del cache[key] 108 | wrapper.cache_size -= 1 109 | return True 110 | except KeyError: 111 | return False 112 | else: 113 | try: 114 | wrapper.cache_size -= len(cache[key]) 115 | del cache[key] 116 | return True 117 | except KeyError: 118 | return False 119 | 120 | wrapper.clear = clear 121 | wrapper.cache = cache 122 | wrapper.delete = delete 123 | wrapper.cache_size = 0 124 | return wrapper 125 | return decorating_function 126 | -------------------------------------------------------------------------------- /CodernityDB3/rr_cache_with_lock.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from __future__ import absolute_import 20 | import six 21 | import functools 22 | from random import choice 23 | from six.moves import range 24 | 25 | 26 | def create_cache1lvl(lock_obj): 27 | def cache1lvl(maxsize=100): 28 | def decorating_function(user_function): 29 | cache = {} 30 | lock = lock_obj() 31 | 32 | @functools.wraps(user_function) 33 | def wrapper(key, *args, **kwargs): 34 | if isinstance(key, six.text_type): 35 | key = key.encode() 36 | try: 37 | result = cache[key] 38 | except KeyError: 39 | with lock: 40 | if len(cache) == maxsize: 41 | for i in range(maxsize // 10 or 1): 42 | del cache[choice(list(cache.keys()))] 43 | cache[key] = user_function(key, *args, **kwargs) 44 | result = cache[key] 45 | return result 46 | 47 | def clear(): 48 | cache.clear() 49 | 50 | def delete(key): 51 | if isinstance(key, six.text_type): 52 | key = key.encode() 53 | try: 54 | del cache[key] 55 | return True 56 | except KeyError: 57 | return False 58 | 59 | wrapper.clear = clear 60 | wrapper.cache = cache 61 | wrapper.delete = delete 62 | return wrapper 63 | return decorating_function 64 | return cache1lvl 65 | 66 | 67 | def create_cache2lvl(lock_obj): 68 | def cache2lvl(maxsize=100): 69 | def decorating_function(user_function): 70 | cache = {} 71 | lock = lock_obj() 72 | 73 | @functools.wraps(user_function) 74 | def wrapper(*args, **kwargs): 75 | key = args[0] 76 | # print('rr_cache_with_lock.2lvl.wrapper %r' % key) 77 | if isinstance(key, six.text_type): 78 | key = key.encode() 79 | try: 80 | result = cache[key][args[1]] 81 | except KeyError: 82 | with lock: 83 | if wrapper.cache_size == maxsize: 84 | to_delete = maxsize // 10 or 1 85 | for i in range(to_delete): 86 | key1 = choice(list(cache.keys())) 87 | key2 = choice(list(cache[key1].keys())) 88 | del cache[key1][key2] 89 | if not cache[key1]: 90 | del cache[key1] 91 | wrapper.cache_size -= to_delete 92 | result = user_function(*args, **kwargs) 93 | try: 94 | cache[key][args[1]] = result 95 | except KeyError: 96 | cache[key] = {args[1]: result} 97 | wrapper.cache_size += 1 98 | return result 99 | 100 | def clear(): 101 | cache.clear() 102 | wrapper.cache_size = 0 103 | 104 | def delete(key, *args): 105 | # print('rr_cache_with_lock.2lvl.delete %r' % key) 106 | if isinstance(key, six.text_type): 107 | key = key.encode() 108 | if args: 109 | try: 110 | del cache[key][args[0]] 111 | if not cache[key]: 112 | del cache[key] 113 | wrapper.cache_size -= 1 114 | return True 115 | except KeyError: 116 | return False 117 | else: 118 | try: 119 | wrapper.cache_size -= len(cache[key]) 120 | del cache[key] 121 | return True 122 | except KeyError: 123 | return False 124 | 125 | wrapper.clear = clear 126 | wrapper.cache = cache 127 | wrapper.delete = delete 128 | wrapper.cache_size = 0 129 | return wrapper 130 | return decorating_function 131 | return cache2lvl 132 | -------------------------------------------------------------------------------- /CodernityDB3/sharded_hash.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from __future__ import absolute_import 20 | import six 21 | 22 | from CodernityDB3.hash_index import UniqueHashIndex, HashIndex 23 | from CodernityDB3.sharded_index import ShardedIndex 24 | from CodernityDB3.index import IndexPreconditionsException 25 | 26 | from random import getrandbits 27 | import uuid 28 | 29 | 30 | class IU_ShardedUniqueHashIndex(ShardedIndex): 31 | 32 | custom_header = """import uuid 33 | from random import getrandbits 34 | from CodernityDB3.sharded_index import ShardedIndex 35 | """ 36 | 37 | def __init__(self, db_path, name, *args, **kwargs): 38 | if kwargs.get('sh_nums', 0) > 255: 39 | raise IndexPreconditionsException("Too many shards") 40 | kwargs['ind_class'] = UniqueHashIndex 41 | super(IU_ShardedUniqueHashIndex, self).__init__(db_path, 42 | name, *args, **kwargs) 43 | self.patchers.append(self.wrap_insert_id_index) 44 | 45 | @staticmethod 46 | def wrap_insert_id_index(db_obj, clean=False): 47 | def _insert_id_index(_rev, data): 48 | """ 49 | Performs insert on **id** index. 50 | """ 51 | _id, value = db_obj.id_ind.make_key_value(data) # may be improved 52 | trg_shard = _id[:2] 53 | if isinstance(trg_shard, six.binary_type): 54 | trg_shard = trg_shard.decode() 55 | storage = db_obj.id_ind.shards_r[trg_shard].storage 56 | start, size = storage.insert(value) 57 | db_obj.id_ind.insert(_id, _rev, start, size) 58 | return _id 59 | if not clean: 60 | if hasattr(db_obj, '_insert_id_index_orig'): 61 | raise IndexPreconditionsException( 62 | "Already patched, something went wrong") 63 | setattr(db_obj, "_insert_id_index_orig", db_obj._insert_id_index) 64 | setattr(db_obj, "_insert_id_index", _insert_id_index) 65 | else: 66 | setattr(db_obj, "_insert_id_index", db_obj._insert_id_index_orig) 67 | delattr(db_obj, "_insert_id_index_orig") 68 | 69 | def create_key(self): 70 | h = uuid.UUID(int=getrandbits(128), version=4).hex 71 | trg = self.last_used + 1 72 | if trg >= self.sh_nums: 73 | trg = 0 74 | self.last_used = trg 75 | h = '%02x%30s' % (trg, h[2:]) 76 | return h 77 | 78 | def delete(self, key, *args, **kwargs): 79 | trg_shard = key[:2] 80 | if isinstance(trg_shard, six.binary_type): 81 | trg_shard = trg_shard.decode() 82 | op = self.shards_r[trg_shard] 83 | return op.delete(key, *args, **kwargs) 84 | 85 | def update(self, key, *args, **kwargs): 86 | trg_shard = key[:2] 87 | if isinstance(trg_shard, six.binary_type): 88 | trg_shard = trg_shard.decode() 89 | self.last_used = int(trg_shard, 16) 90 | op = self.shards_r[trg_shard] 91 | return op.update(key, *args, **kwargs) 92 | 93 | def insert(self, key, *args, **kwargs): 94 | trg_shard = key[:2] # in most cases it's in create_key BUT not always 95 | if isinstance(trg_shard, six.binary_type): 96 | trg_shard = trg_shard.decode() 97 | self.last_used = int(key[:2], 16) 98 | op = self.shards_r[trg_shard] 99 | return op.insert(key, *args, **kwargs) 100 | 101 | def get(self, key, *args, **kwargs): 102 | trg_shard = key[:2] 103 | if isinstance(trg_shard, six.binary_type): 104 | trg_shard = trg_shard.decode() 105 | self.last_used = int(trg_shard, 16) 106 | op = self.shards_r[trg_shard] 107 | return op.get(key, *args, **kwargs) 108 | 109 | 110 | class ShardedUniqueHashIndex(IU_ShardedUniqueHashIndex): 111 | 112 | # allow unique hash to be used directly 113 | custom_header = 'from CodernityDB3.sharded_hash import IU_ShardedUniqueHashIndex' 114 | 115 | pass 116 | 117 | 118 | class IU_ShardedHashIndex(ShardedIndex): 119 | 120 | custom_header = """from CodernityDB3.sharded_index import ShardedIndex""" 121 | 122 | def __init__(self, db_path, name, *args, **kwargs): 123 | kwargs['ind_class'] = HashIndex 124 | super(IU_ShardedHashIndex, self).__init__(db_path, name, * 125 | args, **kwargs) 126 | 127 | def calculate_shard(self, key): 128 | """ 129 | Must be implemented. It has to return shard to be used by key 130 | 131 | :param key: key 132 | :returns: target shard 133 | :rtype: int 134 | """ 135 | raise NotImplementedError() 136 | 137 | def delete(self, doc_id, key, *args, **kwargs): 138 | trg_shard = self.calculate_shard(key) 139 | if isinstance(trg_shard, six.binary_type): 140 | trg_shard = trg_shard.decode() 141 | op = self.shards_r[trg_shard] 142 | return op.delete(doc_id, key, *args, **kwargs) 143 | 144 | def insert(self, doc_id, key, *args, **kwargs): 145 | trg_shard = self.calculate_shard(key) 146 | if isinstance(trg_shard, six.binary_type): 147 | trg_shard = trg_shard.decode() 148 | op = self.shards_r[trg_shard] 149 | return op.insert(doc_id, key, *args, **kwargs) 150 | 151 | def update(self, doc_id, key, *args, **kwargs): 152 | trg_shard = self.calculate_shard(key) 153 | if isinstance(trg_shard, six.binary_type): 154 | trg_shard = trg_shard.decode() 155 | op = self.shards_r[trg_shard] 156 | return op.insert(doc_id, key, *args, **kwargs) 157 | 158 | def get(self, key, *args, **kwargs): 159 | trg_shard = self.calculate_shard(key) 160 | if isinstance(trg_shard, six.binary_type): 161 | trg_shard = trg_shard.decode() 162 | op = self.shards_r[trg_shard] 163 | return op.get(key, *args, **kwargs) 164 | 165 | 166 | class ShardedHashIndex(IU_ShardedHashIndex): 167 | pass 168 | -------------------------------------------------------------------------------- /CodernityDB3/sharded_index.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from __future__ import absolute_import 20 | from CodernityDB3.index import Index 21 | from six.moves import range 22 | # from CodernityDB3.env import cdb_environment 23 | # import warnings 24 | 25 | 26 | class ShardedIndex(Index): 27 | 28 | def __init__(self, db_path, name, *args, **kwargs): 29 | """ 30 | There are 3 additional parameters. You have to hardcode them in your custom class. **NEVER** use directly 31 | 32 | :param int sh_nums: how many shards should be 33 | :param class ind_class: Index class to use (HashIndex or your custom one) 34 | :param bool use_make_keys: if True, `make_key`, and `make_key_value` will be overriden with those from first shard 35 | 36 | The rest parameters are passed straight to `ind_class` shards. 37 | 38 | """ 39 | super(ShardedIndex, self).__init__(db_path, name) 40 | try: 41 | self.sh_nums = kwargs.pop('sh_nums') 42 | except KeyError: 43 | self.sh_nums = 5 44 | try: 45 | ind_class = kwargs.pop('ind_class') 46 | except KeyError: 47 | raise Exception("ind_class must be given") 48 | else: 49 | self.ind_class = ind_class 50 | if 'use_make_keys' in kwargs: 51 | self.use_make_keys = kwargs.pop('use_make_keys') 52 | else: 53 | self.use_make_keys = False 54 | self._set_shard_datas(*args, **kwargs) 55 | self.patchers = [] # database object patchers 56 | 57 | def _set_shard_datas(self, *args, **kwargs): 58 | self.shards = {} 59 | self.shards_r = {} 60 | # ind_class = globals()[self.ind_class] 61 | ind_class = self.ind_class 62 | i = 0 63 | for sh_name in [self.name + str(x) for x in range(self.sh_nums)]: 64 | # dict is better than list in that case 65 | self.shards[i] = ind_class(self.db_path, sh_name, *args, **kwargs) 66 | self.shards_r['%02x' % i] = self.shards[i] 67 | self.shards_r[i] = self.shards[i] 68 | i += 1 69 | 70 | if not self.use_make_keys: 71 | self.make_key = self.shards[0].make_key 72 | self.make_key_value = self.shards[0].make_key_value 73 | 74 | self.last_used = 0 75 | 76 | @property 77 | def storage(self): 78 | st = self.shards[self.last_used].storage 79 | return st 80 | 81 | def __getattr__(self, name): 82 | return getattr(self.shards[self.last_used], name) 83 | 84 | def open_index(self): 85 | for curr in self.shards.values(): 86 | curr.open_index() 87 | 88 | def create_index(self): 89 | for curr in self.shards.values(): 90 | curr.create_index() 91 | 92 | def destroy(self): 93 | for curr in self.shards.values(): 94 | curr.destroy() 95 | 96 | def compact(self): 97 | for curr in self.shards.values(): 98 | curr.compact() 99 | 100 | def reindex(self): 101 | for curr in self.shards.values(): 102 | curr.reindex() 103 | 104 | def all(self, *args, **kwargs): 105 | for curr in self.shards.values(): 106 | for now in curr.all(*args, **kwargs): 107 | yield now 108 | 109 | def get_many(self, *args, **kwargs): 110 | for curr in self.shards.values(): 111 | for now in curr.get_many(*args, **kwargs): 112 | yield now 113 | -------------------------------------------------------------------------------- /CodernityDB3/storage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import absolute_import 19 | import six 20 | import os 21 | import struct 22 | import shutil 23 | import marshal 24 | import io 25 | 26 | 27 | try: 28 | from CodernityDB3 import __version__ 29 | except ImportError: 30 | from .__init__ import __version__ 31 | 32 | 33 | class StorageException(Exception): 34 | pass 35 | 36 | 37 | class DummyStorage(object): 38 | """ 39 | Storage mostly used to fake real storage 40 | """ 41 | 42 | def create(self, *args, **kwargs): 43 | pass 44 | 45 | def open(self, *args, **kwargs): 46 | pass 47 | 48 | def close(self, *args, **kwargs): 49 | pass 50 | 51 | def data_from(self, *args, **kwargs): 52 | pass 53 | 54 | def data_to(self, *args, **kwargs): 55 | pass 56 | 57 | def save(self, *args, **kwargs): 58 | return 0, 0 59 | 60 | def insert(self, *args, **kwargs): 61 | return self.save(*args, **kwargs) 62 | 63 | def update(self, *args, **kwargs): 64 | return 0, 0 65 | 66 | def get(self, *args, **kwargs): 67 | return None 68 | 69 | # def compact(self, *args, **kwargs): 70 | # pass 71 | 72 | def fsync(self, *args, **kwargs): 73 | pass 74 | 75 | def flush(self, *args, **kwargs): 76 | pass 77 | 78 | 79 | class IU_Storage(object): 80 | 81 | __version__ = __version__ 82 | 83 | def __init__(self, db_path, name='main'): 84 | if isinstance(db_path, bytes): 85 | db_path = db_path.decode() 86 | if isinstance(name, bytes): 87 | name = name.decode() 88 | 89 | self.db_path = db_path 90 | self.name = name 91 | self._header_size = 100 92 | 93 | def create(self): 94 | if os.path.exists(os.path.join(self.db_path, self.name + "_stor")): 95 | raise IOError("Storage already exists!") 96 | with io.open(os.path.join(self.db_path, self.name + "_stor"), 'wb') as f: 97 | if isinstance(self.__version__, six.text_type): 98 | new_version = self.__version__.encode() 99 | else: 100 | new_version = self.__version__ 101 | f.write(struct.pack(b'10s90s', new_version, b'|||||')) 102 | f.close() 103 | self._f = io.open(os.path.join( 104 | self.db_path, self.name + "_stor"), 'r+b', buffering=0) 105 | self.flush() 106 | self._f.seek(0, 2) 107 | 108 | def open(self): 109 | if not os.path.exists(os.path.join(self.db_path, self.name + "_stor")): 110 | raise IOError("Storage doesn't exists!") 111 | self._f = io.open(os.path.join( 112 | self.db_path, self.name + "_stor"), 'r+b', buffering=0) 113 | self.flush() 114 | self._f.seek(0, 2) 115 | 116 | def destroy(self): 117 | os.unlink(os.path.join(self.db_path, self.name + '_stor')) 118 | 119 | def close(self): 120 | self._f.close() 121 | # self.flush() 122 | # self.fsync() 123 | 124 | def data_from(self, data): 125 | return marshal.loads(data) 126 | 127 | def data_to(self, data): 128 | return marshal.dumps(data) 129 | 130 | def save(self, data): 131 | s_data = self.data_to(data) 132 | self._f.seek(0, 2) 133 | start = self._f.tell() 134 | size = len(s_data) 135 | self._f.write(s_data) 136 | self.flush() 137 | return start, size 138 | 139 | def insert(self, data): 140 | return self.save(data) 141 | 142 | def update(self, data): 143 | return self.save(data) 144 | 145 | def get(self, start, size, status='c'): 146 | if status == 'd' or status == b'd': 147 | return None 148 | else: 149 | self._f.seek(start) 150 | return self.data_from(self._f.read(size)) 151 | 152 | def flush(self): 153 | self._f.flush() 154 | 155 | def fsync(self): 156 | os.fsync(self._f.fileno()) 157 | 158 | 159 | # classes for public use, done in this way because of 160 | # generation static files with indexes (_index directory) 161 | 162 | 163 | class Storage(IU_Storage): 164 | pass 165 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | prune docs -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | This repository 2 | =============== 3 | 4 | 5 | This is an intent **to port CodernityDB to Python 3**, from de [original source](http://labs.codernity.com/codernitydb), written for Python 2.x, 6 | 7 | CodernityDB is opensource, pure python (no 3rd party dependency), fast (really fast check Speed in documentation if you don't believe in words), multiplatform, schema-less, NoSQL_ database. It has optional support for HTTP server version (CodernityDB-HTTP), and also Python client library (CodernityDB-PyClient) that aims to be 100% compatible with embeded version. 8 | 9 | **Although this port is a beta** yet, it works in the basic usage cases. 10 | 11 | Calling for help 12 | ================ 13 | 14 | Any help to port CodernityDB to Python 3 is wellcome. It's a hard works. 15 | 16 | Feel free to clone the repo an send any patch. 17 | 18 | Status 19 | ====== 20 | 21 | Following the official examples, I was able to determinate: 22 | 23 | - Insert/Save: Works oks! 24 | - Get query: Works oks! 25 | - Duplicates: Works oks! 26 | - Update/delete: Doesn't works yet. 27 | - Ordered data: Doesn't works yet. 28 | 29 | 30 | Ported examples 31 | =============== 32 | 33 | There he ported examples: 34 | 35 | Insert/Save 36 | ----------- 37 | 38 | .. code-block:: python 39 | 40 | from CodernityDB3.database import Database 41 | 42 | def main(): 43 | db = Database('/tmp/tut1') 44 | db.create() 45 | for x in range(100): 46 | print(db.insert(dict(x=x))) 47 | for curr in db.all('id'): 48 | print(curr) 49 | 50 | main() 51 | 52 | 53 | Get query 54 | --------- 55 | 56 | .. code-block:: python 57 | 58 | from CodernityDB3.database import Database 59 | from CodernityDB3.hash_index import HashIndex 60 | 61 | 62 | class WithXIndex(HashIndex): 63 | 64 | def __init__(self, *args, **kwargs): 65 | kwargs['key_format'] = 'I' 66 | super(WithXIndex, self).__init__(*args, **kwargs) 67 | 68 | def make_key_value(self, data): 69 | a_val = data.get("x") 70 | if a_val is not None: 71 | return a_val, None 72 | return None 73 | 74 | def make_key(self, key): 75 | return key 76 | 77 | 78 | def main(): 79 | db = Database('/tmp/tut2') 80 | db.create() 81 | x_ind = WithXIndex(db.path, 'x') 82 | db.add_index(x_ind) 83 | 84 | for x in range(100): 85 | db.insert(dict(x=x)) 86 | 87 | for y in range(100): 88 | db.insert(dict(y=y)) 89 | 90 | print(db.get('x', 10, with_doc=True)) 91 | 92 | if __name__ == '__main__': 93 | main() 94 | 95 | 96 | Duplicates 97 | ---------- 98 | 99 | .. code-block:: python 100 | 101 | from CodernityDB3.database import Database 102 | from CodernityDB3.hash_index import HashIndex 103 | 104 | 105 | class WithXIndex(HashIndex): 106 | 107 | def __init__(self, *args, **kwargs): 108 | kwargs['key_format'] = 'I' 109 | super(WithXIndex, self).__init__(*args, **kwargs) 110 | 111 | def make_key_value(self, data): 112 | a_val = data.get("x") 113 | if a_val is not None: 114 | return a_val, None 115 | return None 116 | 117 | def make_key(self, key): 118 | return key 119 | 120 | 121 | def main(): 122 | db = Database('/tmp/tut3') 123 | db.create() 124 | x_ind = WithXIndex(db.path, 'x') 125 | db.add_index(x_ind) 126 | 127 | for x in range(100): 128 | db.insert(dict(x=x)) 129 | 130 | for x in range(100): 131 | db.insert(dict(x=x)) 132 | 133 | for y in range(100): 134 | db.insert(dict(y=y)) 135 | 136 | print(db.get('x', 10, with_doc=True)) 137 | for curr in db.get_many('x', 10, limit=-1, with_doc=True): 138 | print(curr) 139 | 140 | if __name__ == '__main__': 141 | main() 142 | 143 | 144 | 145 | Update/delete 146 | ------------- 147 | 148 | .. code-block:: python 149 | 150 | from CodernityDB3.database import Database 151 | from CodernityDB3.tree_index import TreeBasedIndex 152 | 153 | 154 | class WithXIndex(TreeBasedIndex): 155 | 156 | def __init__(self, *args, **kwargs): 157 | kwargs['node_capacity'] = 10 158 | kwargs['key_format'] = 'I' 159 | super(WithXIndex, self).__init__(*args, **kwargs) 160 | 161 | def make_key_value(self, data): 162 | t_val = data.get('x') 163 | if t_val is not None: 164 | return t_val, None 165 | return None 166 | 167 | def make_key(self, key): 168 | return key 169 | 170 | 171 | def main(): 172 | db = Database('/tmp/tut_update') 173 | db.create() 174 | x_ind = WithXIndex(db.path, 'x') 175 | db.add_index(x_ind) 176 | 177 | # full examples so we had to add first the data 178 | # the same code as in previous step 179 | 180 | for x in range(100): 181 | db.insert(dict(x=x)) 182 | 183 | for y in range(100): 184 | db.insert(dict(y=y)) 185 | 186 | # end of insert part 187 | 188 | print(db.count(db.all, 'x')) 189 | 190 | for curr in db.all('x', with_doc=True): 191 | doc = curr['doc'] 192 | if curr['key'] % 7 == 0: 193 | db.delete(doc) 194 | elif curr['key'] % 5 == 0: 195 | doc['updated'] = True 196 | db.update(doc) 197 | 198 | print(db.count(db.all, 'x')) 199 | 200 | for curr in db.all('x', with_doc=True): 201 | print(curr) 202 | 203 | if __name__ == '__main__': 204 | main() 205 | 206 | 207 | Ordered 208 | ------- 209 | 210 | .. code-block:: python 211 | 212 | from CodernityDB3.database import Database 213 | from CodernityDB3.tree_index import TreeBasedIndex 214 | 215 | 216 | class WithXIndex(TreeBasedIndex): 217 | 218 | def __init__(self, *args, **kwargs): 219 | kwargs['node_capacity'] = 10 220 | kwargs['key_format'] = 'I' 221 | super(WithXXIndex, self).__init__(*args, **kwargs) 222 | 223 | def make_key_value(self, data): 224 | t_val = data.get('x') 225 | if t_val is not None: 226 | return t_val, data 227 | return None 228 | 229 | def make_key(self, key): 230 | return key 231 | 232 | 233 | def main(): 234 | db = Database('/tmp/tut4') 235 | db.create() 236 | x_ind = WithXIndex(db.path, 'x') 237 | db.add_index(x_ind) 238 | 239 | for x in range(11): 240 | db.insert(dict(x=x)) 241 | 242 | for y in range(11): 243 | db.insert(dict(y=y)) 244 | 245 | print(db.get('x', 10, with_doc=True)) 246 | 247 | for curr in db.get_many('x', start=15, end=25, limit=-1, with_doc=True): 248 | print(curr) 249 | 250 | 251 | if __name__ == '__main__': 252 | main() 253 | 254 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = -l 3 | norecursedirs = * # had to do it because of monkey_patching, etc. run each file separately from tests directory -------------------------------------------------------------------------------- /python3_codernity_duplicates.py: -------------------------------------------------------------------------------- 1 | from CodernityDB3.database import Database 2 | from CodernityDB3.hash_index import HashIndex 3 | 4 | 5 | class WithXIndex(HashIndex): 6 | 7 | def __init__(self, *args, **kwargs): 8 | kwargs['key_format'] = 'I' 9 | super(WithXIndex, self).__init__(*args, **kwargs) 10 | 11 | def make_key_value(self, data): 12 | a_val = data.get("x") 13 | if a_val is not None: 14 | return a_val, None 15 | return None 16 | 17 | def make_key(self, key): 18 | return key 19 | 20 | 21 | def main(): 22 | db = Database('/tmp/tut3') 23 | db.create() 24 | x_ind = WithXIndex(db.path, 'x') 25 | db.add_index(x_ind) 26 | 27 | for x in range(100): 28 | db.insert(dict(x=x)) 29 | 30 | for x in range(100): 31 | db.insert(dict(x=x)) 32 | 33 | for y in range(100): 34 | db.insert(dict(y=y)) 35 | 36 | print(db.get('x', 10, with_doc=True)) 37 | for curr in db.get_many('x', 10, limit=-1, with_doc=True): 38 | print(curr) 39 | 40 | if __name__ == '__main__': 41 | main() -------------------------------------------------------------------------------- /python3_codernity_get_query.py: -------------------------------------------------------------------------------- 1 | from CodernityDB3.database import Database 2 | from CodernityDB3.hash_index import HashIndex 3 | 4 | 5 | class WithXIndex(HashIndex): 6 | 7 | def __init__(self, *args, **kwargs): 8 | kwargs['key_format'] = 'I' 9 | super(WithXIndex, self).__init__(*args, **kwargs) 10 | 11 | def make_key_value(self, data): 12 | a_val = data.get("x") 13 | if a_val is not None: 14 | return a_val, None 15 | return None 16 | 17 | def make_key(self, key): 18 | return key 19 | 20 | 21 | def main(): 22 | db = Database('/tmp/tut2') 23 | db.create() 24 | x_ind = WithXIndex(db.path, 'x') 25 | db.add_index(x_ind) 26 | 27 | for x in range(100): 28 | db.insert(dict(x=x)) 29 | 30 | for y in range(100): 31 | db.insert(dict(y=y)) 32 | 33 | print(db.get('x', 10, with_doc=True)) 34 | 35 | 36 | if __name__ == '__main__': 37 | main() -------------------------------------------------------------------------------- /python3_codernity_insert_save_store.py: -------------------------------------------------------------------------------- 1 | from CodernityDB3.database import Database 2 | 3 | 4 | def main(): 5 | db = Database('/tmp/tut1') 6 | db.create() 7 | for x in range(100): 8 | print(db.insert(dict(x=x))) 9 | for curr in db.all('id'): 10 | print(curr) 11 | 12 | main() 13 | -------------------------------------------------------------------------------- /python3_codernity_ordered.py: -------------------------------------------------------------------------------- 1 | from CodernityDB3.database import Database 2 | from CodernityDB3.tree_index import TreeBasedIndex 3 | 4 | 5 | class WithXIndex(TreeBasedIndex): 6 | 7 | def __init__(self, *args, **kwargs): 8 | kwargs['node_capacity'] = 10 9 | kwargs['key_format'] = 'I' 10 | super(WithXXIndex, self).__init__(*args, **kwargs) 11 | 12 | def make_key_value(self, data): 13 | t_val = data.get('x') 14 | if t_val is not None: 15 | return t_val, data 16 | return None 17 | 18 | def make_key(self, key): 19 | return key 20 | 21 | 22 | def main(): 23 | db = Database('/tmp/tut4') 24 | db.create() 25 | x_ind = WithXIndex(db.path, 'x') 26 | db.add_index(x_ind) 27 | 28 | for x in range(11): 29 | db.insert(dict(x=x)) 30 | 31 | for y in range(11): 32 | db.insert(dict(y=y)) 33 | 34 | print(db.get('x', 10, with_doc=True)) 35 | 36 | for curr in db.get_many('x', start=15, end=25, limit=-1, with_doc=True): 37 | print(curr) 38 | 39 | 40 | if __name__ == '__main__': 41 | main() -------------------------------------------------------------------------------- /python3_codernity_update_delete.py: -------------------------------------------------------------------------------- 1 | from CodernityDB3.database import Database 2 | from CodernityDB3.tree_index import TreeBasedIndex 3 | 4 | 5 | class WithXIndex(TreeBasedIndex): 6 | 7 | def __init__(self, *args, **kwargs): 8 | kwargs['node_capacity'] = 10 9 | kwargs['key_format'] = 'I' 10 | super(WithXIndex, self).__init__(*args, **kwargs) 11 | 12 | def make_key_value(self, data): 13 | t_val = data.get('x') 14 | if t_val is not None: 15 | return t_val, None 16 | return None 17 | 18 | def make_key(self, key): 19 | return key 20 | 21 | 22 | def main(): 23 | db = Database('/tmp/tut_update') 24 | db.create() 25 | x_ind = WithXIndex(db.path, 'x') 26 | db.add_index(x_ind) 27 | 28 | # full examples so we had to add first the data 29 | # the same code as in previous step 30 | 31 | for x in range(100): 32 | db.insert(dict(x=x)) 33 | 34 | for y in range(100): 35 | db.insert(dict(y=y)) 36 | 37 | # end of insert part 38 | 39 | print(db.count(db.all, 'x')) 40 | 41 | for curr in db.all('x', with_doc=True): 42 | doc = curr['doc'] 43 | if curr['key'] % 7 == 0: 44 | db.delete(doc) 45 | elif curr['key'] % 5 == 0: 46 | doc['updated'] = True 47 | db.update(doc) 48 | 49 | print(db.count(db.all, 'x')) 50 | 51 | for curr in db.all('x', with_doc=True): 52 | print(curr) 53 | 54 | if __name__ == '__main__': 55 | main() -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cov-core 2 | coverage 3 | execnet 4 | py 5 | pytest 6 | pytest-cov 7 | pytest-xdist 8 | six 9 | # must be already in Python3 10 | # wsgiref 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from setuptools import setup 19 | import os 20 | 21 | 22 | def get_meta(inc, name): 23 | import re 24 | return eval(re.search(r'(?:%s)\s*=\s*(.*)' % name, inc).group(1)) 25 | 26 | 27 | with open(os.path.join("CodernityDB3", '__init__.py'), 'r') as _init: 28 | _init_d = _init.read() 29 | 30 | __version__ = get_meta(_init_d, '__version__') 31 | __license__ = get_meta(_init_d, '__license__') 32 | 33 | 34 | with open('README.rst') as f: 35 | L_DESCR = f.read() 36 | 37 | requires = [ 38 | ] 39 | 40 | keywords = ' '.join(('database', 'python', 'nosql', 'key-value', 'key/value', 'db')) 41 | 42 | setup(name='CodernityDB3', 43 | version=__version__, 44 | description="Python 3 port of pure python, fast, schema-less, NoSQL database", 45 | long_description=L_DESCR, 46 | keywords=keywords, 47 | author='cr0hn', 48 | author_email='cr0hn@cr0hn.com', 49 | url='http://labs.codernity.com/codernitydb', 50 | packages=['CodernityDB3'], 51 | platforms='any', 52 | license=__license__, 53 | install_requires=requires, 54 | classifiers=[ 55 | "License :: OSI Approved :: Apache Software License", 56 | "Programming Language :: Python", 57 | "Programming Language :: Python :: 3.2", 58 | "Programming Language :: Python :: 3.4", 59 | "Operating System :: OS Independent", 60 | "Topic :: Internet", 61 | "Topic :: Database", 62 | "Topic :: Software Development", 63 | "Intended Audience :: Developers", 64 | "Development Status :: 4 - Beta"]) 65 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | def pytest_addoption(parser): 20 | parser.addoption("--inserts", type="int", 21 | help="how many inserts", default=20) 22 | 23 | parser.addoption("--operations", type="int", 24 | help="how many operations", default=50) 25 | 26 | 27 | def pytest_generate_tests(metafunc): 28 | if "inserts" in metafunc.funcargnames: 29 | metafunc.parametrize("inserts", [metafunc.config.option.inserts, ]) 30 | if "operations" in metafunc.funcargnames: 31 | metafunc.parametrize("operations", [metafunc.config.option.operations, ]) 32 | -------------------------------------------------------------------------------- /tests/hash_tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from CodernityDB3.database import Database, RecordDeleted, RecordNotFound 19 | from CodernityDB3.database import DatabaseException 20 | 21 | from CodernityDB3.hash_index import HashIndex, UniqueHashIndex 22 | from CodernityDB3.index import IndexException 23 | from CodernityDB3.misc import random_hex_32 24 | 25 | from CodernityDB3 import rr_cache 26 | 27 | import pytest 28 | import os 29 | import random 30 | from hashlib import md5 31 | 32 | try: 33 | from collections import Counter 34 | except ImportError: 35 | class Counter(dict): 36 | 'Mapping where default values are zero' 37 | def __missing__(self, key): 38 | return 0 39 | 40 | 41 | class CustomHashIndex(HashIndex): 42 | 43 | def __init__(self, *args, **kwargs): 44 | kwargs['key_format'] = 'I' 45 | kwargs['hash_lim'] = 1 46 | super(CustomHashIndex, self).__init__(*args, **kwargs) 47 | 48 | def make_key_value(self, data): 49 | d = data.get('test') 50 | if d is None: 51 | return None 52 | if d > 5: 53 | k = 1 54 | else: 55 | k = 0 56 | return k, dict(test=d) 57 | 58 | def make_key(self, key): 59 | return key 60 | 61 | 62 | class Md5Index(HashIndex): 63 | 64 | def __init__(self, *args, **kwargs): 65 | kwargs['key_format'] = '16s' 66 | kwargs['hash_lim'] = 4 * 1024 67 | super(Md5Index, self).__init__(*args, **kwargs) 68 | 69 | def make_key_value(self, data): 70 | key = data['name'] 71 | if not isinstance(key, bytes): 72 | key = bytes(key, 'utf-8') 73 | return md5(key).digest(), {} 74 | 75 | def make_key(self, key): 76 | if not isinstance(key, bytes): 77 | key = bytes(key, 'utf-8') 78 | return md5(key).digest() 79 | 80 | 81 | class WithAIndex(HashIndex): 82 | 83 | def __init__(self, *args, **kwargs): 84 | kwargs['key_format'] = '16s' 85 | kwargs['hash_lim'] = 4 * 1024 86 | super(WithAIndex, self).__init__(*args, **kwargs) 87 | 88 | def make_key_value(self, data): 89 | a_val = data.get("a") 90 | if a_val: 91 | if isinstance(a_val, int): 92 | a_val = str(a_val) 93 | if not isinstance(a_val, bytes): 94 | a_val = bytes(a_val, 'utf-8') 95 | return md5(a_val).digest(), {} 96 | return None 97 | 98 | def make_key(self, key): 99 | if isinstance(key, int): 100 | key = str(key) 101 | if not isinstance(key, bytes): 102 | key = bytes(key, 'utf-8') 103 | return md5(key).digest() 104 | 105 | 106 | class HashIndexTests: 107 | 108 | def setup_method(self, method): 109 | self.counter = Counter() 110 | 111 | def test_simple(self, tmpdir): 112 | db = self._db(os.path.join(str(tmpdir), 'db')) 113 | # db.set_indexes([UniqueHashIndex(db.path, 'id')]) 114 | # db.initialize() 115 | db.add_index(UniqueHashIndex(db.path, 'id'), False) 116 | db.create() 117 | 118 | with pytest.raises(RecordNotFound): 119 | db.get("id", "not_existing") 120 | a1 = dict(a=1) 121 | db.insert(a1) 122 | test = db.get('id', a1['_id']) 123 | assert test['_id'] == a1['_id'] 124 | a_id = a1['_id'] 125 | assert test == a1 126 | a1['x'] = 'x' 127 | db.update(a1) 128 | test2 = db.get('id', a_id) 129 | assert test2['x'] == 'x' 130 | assert test2 != test 131 | db.delete(a1) 132 | with pytest.raises(RecordDeleted): 133 | db.get('id', a_id) 134 | db.close() 135 | 136 | def test_insert_with_id(self, tmpdir): 137 | db = self._db(os.path.join(str(tmpdir), 'db')) 138 | # db.set_indexes([UniqueHashIndex(db.path, 'id')]) 139 | # db.initialize() 140 | db.add_index(UniqueHashIndex(db.path, 'id'), False) 141 | db.create() 142 | 143 | doc = dict(a=1, _id=random_hex_32()) 144 | ins = db.insert(doc) 145 | assert ins['_id'] == doc["_id"] 146 | db.close() 147 | 148 | def test_delete(self, tmpdir, inserts): 149 | db = self._db(os.path.join(str(tmpdir), 'db')) 150 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 151 | db.create() 152 | 153 | ins = [] 154 | for x in range(inserts): 155 | doc = dict(x=x) 156 | db.insert(doc) 157 | ins.append(doc) 158 | self.counter['ins'] += 1 159 | 160 | for i in range(inserts // 10): 161 | curr = ins.pop(random.randint(0, len(ins) - 1)) 162 | db.delete(curr) 163 | 164 | assert len(ins) == db.count(db.all, 'id') 165 | 166 | for x in range(inserts): 167 | doc = dict(x=x) 168 | db.insert(doc) 169 | ins.append(doc) 170 | self.counter['ins'] += 1 171 | 172 | for i in range(inserts // 10): 173 | curr = ins.pop(random.randint(0, len(ins) - 1)) 174 | db.delete(curr) 175 | 176 | assert len(ins) == db.count(db.all, 'id') 177 | 178 | def test_get_after_delete(self, tmpdir): 179 | db = self._db(os.path.join(str(tmpdir), 'db')) 180 | db.set_indexes([UniqueHashIndex(db.path, 'id'), 181 | CustomHashIndex(db.path, 'custom')]) 182 | db.create() 183 | 184 | for x in range(100): 185 | doc = dict(test=6) 186 | doc.update(db.insert(doc)) 187 | if doc['test'] > 5: 188 | self.counter['r'] += 1 189 | else: 190 | self.counter['l'] += 1 191 | 192 | counted_bef = db.count(db.all, 'custom') 193 | elem = db.all('custom', with_doc=True, limit=1) 194 | 195 | doc = next(elem)['doc'] 196 | assert db.delete(doc) == True 197 | 198 | counted_aft = db.count(db.all, 'custom') 199 | assert counted_bef - 1 == counted_aft 200 | 201 | from_ind = db.get('custom', 1, with_doc=True) 202 | assert from_ind['doc']['_id'] != doc['_id'] 203 | 204 | alls = db.all('custom', with_doc=True, limit=90) 205 | for curr in alls: 206 | assert db.delete(curr['doc']) == True 207 | 208 | from_ind = db.get('custom', 1, with_doc=True) 209 | assert from_ind['doc'] != {} 210 | 211 | def test_delete_with_id_and_rev_only(self, tmpdir, inserts): 212 | db = self._db(os.path.join(str(tmpdir), 'db')) 213 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 214 | db.create() 215 | 216 | ins = [] 217 | for x in range(inserts): 218 | doc = dict(x=x) 219 | db.insert(doc) 220 | ins.append(doc) 221 | self.counter['ins'] += 1 222 | 223 | for i in range(inserts // 10): 224 | curr = ins.pop(random.randint(0, len(ins) - 1)) 225 | d = {"_id": curr['_id'], '_rev': curr['_rev']} 226 | db.delete(d) 227 | 228 | assert len(ins) == db.count(db.all, 'id') 229 | 230 | for x in range(inserts): 231 | doc = dict(x=x) 232 | db.insert(doc) 233 | ins.append(doc) 234 | self.counter['ins'] += 1 235 | 236 | for i in range(inserts // 10): 237 | curr = ins.pop(random.randint(0, len(ins) - 1)) 238 | db.delete(curr) 239 | 240 | assert len(ins) == db.count(db.all, 'id') 241 | 242 | def test_update_custom_unique(self, tmpdir, inserts): 243 | db = self._db(os.path.join(str(tmpdir), 'db')) 244 | db.set_indexes([UniqueHashIndex(db.path, 'id'), 245 | CustomHashIndex(db.path, 'custom')]) 246 | db.create() 247 | 248 | ins = [] 249 | 250 | for x in range(inserts): 251 | doc = dict(test=1) 252 | db.insert(doc) 253 | ins.append(doc) 254 | self.counter['ins'] += 1 255 | 256 | assert len(ins) == db.count(db.all, 'id') 257 | assert len(ins) == db.count(db.all, 'custom') 258 | assert len(ins) == db.count( 259 | db.get_many, 'custom', key=0, limit=inserts + 1) 260 | assert 0 == db.count( 261 | db.get_many, 'custom', key=1, limit=inserts + 1) 262 | 263 | sample = random.sample(ins, inserts // 10) 264 | for curr in sample: 265 | curr['test'] = 10 266 | db.update(curr) 267 | self.counter['upd'] += 1 268 | 269 | assert self.counter['ins'] == db.count(db.all, 'id') 270 | assert self.counter['ins'] == db.count(db.all, 'custom') 271 | assert self.counter['upd'] == db.count( 272 | db.get_many, 'custom', key=1, limit=inserts + 1) 273 | assert self.counter['ins'] - self.counter['upd'] == db.count( 274 | db.get_many, 'custom', key=0, limit=inserts + 1) 275 | 276 | def test_update_custom_same_key_new_value(self, tmpdir, inserts): 277 | db = self._db(os.path.join(str(tmpdir), 'db')) 278 | db.set_indexes([UniqueHashIndex(db.path, 'id'), 279 | CustomHashIndex(db.path, 'custom')]) 280 | db.create() 281 | 282 | inserted = [] 283 | for x in range(inserts): 284 | inserted.append(db.insert(dict(test=x))) 285 | inserted[-1]['test'] = x 286 | for el in inserted[::20]: 287 | for i in range(4): 288 | curr = db.get('id', el['_id'], with_storage=True) 289 | assert el['test'] == curr['test'] 290 | el['test'] += random.randint(1, 3) 291 | db.update(el) 292 | assert len(inserted) == db.count(db.all, 'custom') 293 | 294 | db.close() 295 | 296 | def test_double_insert(self, tmpdir): 297 | db = self._db(os.path.join(str(tmpdir), 'db')) 298 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 299 | db.create() 300 | 301 | a_id = '54bee5c4628648b5a742379a1de89b2d' 302 | a1 = dict(a=1, _id=a_id) 303 | db.insert(a1) 304 | a2 = dict(a=2, _id=a_id) 305 | with pytest.raises(IndexException): 306 | db.insert(a2) 307 | 308 | def test_adv1(self, tmpdir, inserts): 309 | db = self._db(os.path.join(str(tmpdir), 'db')) 310 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 311 | db.create() 312 | l = [] 313 | for i in range(inserts): 314 | c = dict(i=i) 315 | db.insert(c) 316 | l.append(c) 317 | for j in range(inserts): 318 | curr = l[j] 319 | c = db.get('id', curr['_id']) 320 | assert c['_id'] == curr['_id'] 321 | assert c['i'] == j 322 | for i in range(inserts): 323 | curr = l[i] 324 | c = db.get("id", curr['_id']) 325 | c['update'] = True 326 | db.update(c) 327 | 328 | for j in range(inserts): 329 | curr = l[i] 330 | c = db.get('id', curr['_id']) 331 | assert c['update'] == True 332 | 333 | for j in range(inserts): 334 | curr = l[j] 335 | c = db.get('id', curr['_id']) 336 | assert db.delete(c) == True 337 | 338 | for j in range(inserts): 339 | with pytest.raises(RecordDeleted): 340 | db.get('id', l[j]['_id']) 341 | 342 | db.close() 343 | 344 | def test_all(self, tmpdir, inserts): 345 | db = self._db(os.path.join(str(tmpdir), 'db')) 346 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 347 | db.create() 348 | l = [] 349 | for i in range(inserts): 350 | c = dict(i=i) 351 | db.insert(c) 352 | l.append(c) 353 | 354 | assert db.count(db.all, 'id') == inserts 355 | 356 | to_delete = random.randint(0, inserts - 1) 357 | db.delete(l[to_delete]) 358 | 359 | assert db.count(db.all, 'id') == inserts - 1 360 | 361 | def test_compact(self, tmpdir): 362 | db = self._db(os.path.join(str(tmpdir), 'db')) 363 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 364 | db.create() 365 | l = [] 366 | for i in range(10): 367 | c = dict(i=i) 368 | db.insert(c) 369 | l.append(c) 370 | 371 | for i in range(10): 372 | curr = l[i] 373 | c = db.get("id", curr['_id']) 374 | c['update'] = True 375 | c.update(db.update(c)) 376 | 377 | db.compact() 378 | 379 | for j in range(10): 380 | curr = l[j] 381 | c = db.get('id', curr['_id']) 382 | assert c['_id'] == curr['_id'] 383 | assert c['i'] == j 384 | 385 | db.close() 386 | 387 | def test_compact2(self, tmpdir): 388 | db = self._db(os.path.join(str(tmpdir), 'db')) 389 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 390 | db.create() 391 | l = [] 392 | for i in range(10): 393 | if i % 2 == 0: 394 | c = dict(i=i, even=True) 395 | else: 396 | c = dict(i=i) 397 | c.update(db.insert(c)) 398 | l.append(c) 399 | 400 | for i, curr in enumerate(db.all('id')): 401 | if i % 2: 402 | db.delete(curr) 403 | 404 | db.compact() 405 | db.compact() 406 | db.compact() 407 | assert len(list(db.all('id', with_doc=True))) == 5 408 | 409 | for curr in db.all('id'): 410 | db.delete(curr) 411 | 412 | assert len(list(db.all('id', with_doc=True))) == 0 413 | 414 | def test_similar(self, tmpdir): 415 | db = self._db(os.path.join(str(tmpdir), 'db')) 416 | db.set_indexes( 417 | [UniqueHashIndex(db.path, 'id'), Md5Index(db.path, 'md5')]) 418 | db.create() 419 | 420 | a = dict(name='pigmej') 421 | db.insert(a) 422 | db.get('md5', 'pigmej') 423 | 424 | with pytest.raises(RecordNotFound): 425 | db.get("md5", 'pigme') 426 | 427 | def test_custom_index(self, tmpdir): 428 | 429 | db = self._db(os.path.join(str(tmpdir), 'db')) 430 | db.set_indexes([UniqueHashIndex( 431 | db.path, 'id'), CustomHashIndex(db.path, 'custom')]) 432 | db.create() 433 | 434 | l_1 = [] 435 | l_0 = [] 436 | for i in range(10): 437 | c = dict(test=i) 438 | db.insert(c) 439 | if i > 5: 440 | l_1.append(c) 441 | else: 442 | l_0.append(c) 443 | 444 | i = 0 445 | for curr in db.all('custom', with_doc=False): 446 | i += 1 447 | 448 | assert i == len(l_1) + len(l_0) 449 | 450 | db.compact() 451 | 452 | gen = db.get_many('custom', key=1, limit=10, offset=0, with_doc=True) 453 | got = [] 454 | while True: 455 | try: 456 | d = next(gen) 457 | except StopIteration: 458 | break 459 | else: 460 | got.append(d) 461 | assert len(l_1) == len(got) 462 | 463 | for doc in l_1: 464 | doc['test'] = 0 465 | db.update(doc) 466 | 467 | gen = db.get_many('custom', key=0, limit=100, offset=0, with_doc=False) 468 | got = [] 469 | while True: 470 | try: 471 | d = next(gen) 472 | except StopIteration: 473 | break 474 | else: 475 | got.append(d) 476 | 477 | assert len(l_1) + len(l_0) == len(got) 478 | 479 | db.compact() 480 | 481 | gen = db.get_many('custom', key=0, limit=100, offset=0, with_doc=False) 482 | got = [] 483 | while True: 484 | try: 485 | d = next(gen) 486 | except StopIteration: 487 | break 488 | else: 489 | got.append(d) 490 | 491 | assert len(l_1) + len(l_0) == len(got) 492 | 493 | def test_custom_index_2(self, tmpdir): 494 | 495 | db = self._db(os.path.join(str(tmpdir), 'db')) 496 | db.set_indexes( 497 | [UniqueHashIndex(db.path, 'id'), WithAIndex(db.path, 'with_a')]) 498 | db.create() 499 | 500 | all_ins = [] 501 | for i in range(10): 502 | curr = dict(something='other') 503 | if i % 2 == 0: 504 | curr['a'] = str(i) 505 | all_ins.append(curr) 506 | db.insert(curr) 507 | 508 | l_0 = len(list(db.all('id'))) 509 | l_1 = len(list(db.all('with_a'))) 510 | assert l_1 != l_0 511 | 512 | all_a = list(db.all('with_a', with_doc=True)) 513 | curr_a = all_a[0]['doc'] 514 | del curr_a['a'] 515 | db.update(curr_a) 516 | l_2 = len(list(db.all('with_a'))) 517 | assert l_2 + 1 == l_1 518 | 519 | curr_a = all_a[-1]['doc'] 520 | db.delete(curr_a) 521 | 522 | l_3 = len(list(db.all('with_a'))) 523 | assert l_3 + 2 == l_1 524 | 525 | def test_insert_delete_compact_get_huge(self, tmpdir, inserts): 526 | inserts *= 10 527 | db = self._db(os.path.join(str(tmpdir), 'db')) 528 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 529 | db.create() 530 | l = [] 531 | for i in range(inserts): 532 | c = dict(i=i) 533 | db.insert(c) 534 | l.append(c) 535 | 536 | assert db.count(db.all, 'id') == inserts 537 | 538 | will_delete = random.randint(0, int(inserts / 15)) 539 | for x in range(will_delete): 540 | to_delete = random.randint(0, len(l) - 1) 541 | db.delete(l.pop(to_delete)) 542 | 543 | assert db.count(db.all, 'id') == inserts - will_delete 544 | 545 | db.compact() 546 | 547 | assert db.count(db.all, 'id') == inserts - will_delete 548 | 549 | def test_all_same_keys(self, tmpdir, inserts): 550 | 551 | db = self._db(os.path.join(str(tmpdir), 'db')) 552 | db.set_indexes([UniqueHashIndex(db.path, 'id'), 553 | WithAIndex(db.path, 'with_a')]) 554 | 555 | db.create() 556 | l = 0 557 | r = 0 558 | z = 0 559 | data = [] 560 | for i in range(inserts): 561 | a = random.randint(0, 10) 562 | if a > 5: 563 | r += 1 564 | elif a == 0: 565 | z += 1 566 | else: 567 | l += 1 568 | c = dict(a=a) 569 | db.insert(c) 570 | data.append(c) 571 | 572 | assert l + r + z == db.count(db.all, "id") 573 | 574 | assert l + r == db.count(db.all, "with_a") 575 | 576 | def test_update_same(self, tmpdir, inserts): 577 | db = self._db(os.path.join(str(tmpdir), 'db')) 578 | db.set_indexes([UniqueHashIndex(db.path, 'id'), 579 | WithAIndex(db.path, 'with_a'), 580 | CustomHashIndex(db.path, 'custom')]) 581 | db.create() 582 | 583 | inserted = {} 584 | 585 | for i in range(inserts): 586 | a = random.randint(1, 9) 587 | # a = 10 588 | if a > 5: 589 | self.counter['r'] += 1 590 | else: 591 | self.counter['l'] += 1 592 | c = dict(test=a) 593 | db.insert(c) 594 | inserted[c['_id']] = c 595 | 596 | def _check(): 597 | 598 | assert self.counter['l'] == db.count(db.get_many, 599 | 'custom', key=0, limit=inserts) 600 | assert self.counter['r'] == db.count(db.get_many, 601 | 'custom', key=1, limit=inserts) 602 | assert self.counter['r'] + self.counter[ 603 | 'l'] == db.count(db.all, "custom") 604 | 605 | _check() 606 | 607 | d = list(inserted.values())[int(inserts / 2)] 608 | for curr_u in range(20): 609 | last_test = d['test'] 610 | if last_test > 5: 611 | d['test'] = 1 612 | self.counter['r'] -= 1 613 | self.counter['l'] += 1 614 | else: 615 | d['test'] = 6 616 | self.counter['l'] -= 1 617 | self.counter['r'] += 1 618 | d['upd'] = curr_u 619 | db.update(d) 620 | _check() 621 | 622 | num = inserts // 10 623 | num = num if num < 20 else 20 624 | 625 | to_update = random.sample(list(inserted.values()), num) 626 | for d in to_update: 627 | for curr_u in range(10): 628 | last_test = d['test'] 629 | if last_test > 5: 630 | d['test'] = 1 631 | self.counter['r'] -= 1 632 | self.counter['l'] += 1 633 | else: 634 | d['test'] = 6 635 | self.counter['l'] -= 1 636 | self.counter['r'] += 1 637 | d['upd'] = curr_u 638 | db.update(d) 639 | _check() 640 | 641 | to_update = random.sample(list(inserted.values()), num) 642 | for curr_u in range(10): 643 | for d in to_update: 644 | last_test = d['test'] 645 | if last_test > 5: 646 | d['test'] = 1 647 | self.counter['r'] -= 1 648 | self.counter['l'] += 1 649 | else: 650 | d['test'] = 6 651 | self.counter['l'] -= 1 652 | self.counter['r'] += 1 653 | d['upd'] = curr_u 654 | db.update(d) 655 | _check() 656 | 657 | db.close() 658 | 659 | def test_insert_on_deleted(self, tmpdir): 660 | db = self._db(os.path.join(str(tmpdir), 'db')) 661 | db.create() 662 | a = db.insert(dict(_id='dd0604e2110d41e9a2ec630461ffd067')) 663 | db.insert(dict(_id='892508163deb4da0b44e8a00802dc75a')) 664 | db.delete(a) 665 | db.insert(dict(_id='d0a6e4e1aa74476a9012f9b8d7181a95')) 666 | db.get('id', '892508163deb4da0b44e8a00802dc75a') 667 | 668 | db.close() 669 | 670 | def test_offset_in_functions(self, tmpdir, inserts): 671 | db = self._db(os.path.join(str(tmpdir), 'db')) 672 | db.set_indexes([UniqueHashIndex(db.path, 'id'), 673 | CustomHashIndex(db.path, 'custom')]) 674 | db.create() 675 | offset = inserts // 10 or 1 676 | real_inserts = inserts if inserts < 1000 else 1000 677 | for x in range(real_inserts): 678 | db.insert(dict(test=x)) 679 | assert real_inserts - offset == db.count(db.all, 'id', offset=offset) 680 | assert real_inserts - offset == db.count(db.all, 'custom', 681 | offset=offset) 682 | assert 1 == db.count(db.get_many, 'custom', 1, limit=1, offset=offset) 683 | 684 | db.close() 685 | -------------------------------------------------------------------------------- /tests/index_files/03md5_index.py: -------------------------------------------------------------------------------- 1 | # md5_index 2 | # Md5Index 3 | 4 | # inserted automatically 5 | import os 6 | import marshal 7 | 8 | import struct 9 | import shutil 10 | 11 | from hashlib import md5 12 | 13 | # custom db code start 14 | 15 | 16 | # custom index code start 17 | # source of classes in index.classes_code 18 | # index code start 19 | class Md5Index(HashIndex): 20 | 21 | def __init__(self, *args, **kwargs): 22 | kwargs['entry_line_format'] = '<32s32sIIcI' 23 | kwargs['hash_lim'] = 4 * 1024 24 | super(Md5Index, self).__init__(*args, **kwargs) 25 | 26 | def make_key_value(self, data): 27 | return md5(data['name'].encode()).hexdigest(), {} 28 | 29 | def make_key(self, key): 30 | return md5(key).hexdigest() 31 | -------------------------------------------------------------------------------- /tests/index_files/04withA_index.py: -------------------------------------------------------------------------------- 1 | # withA_index 2 | # WithAIndex 3 | 4 | # inserted automatically 5 | import os 6 | import marshal 7 | 8 | import struct 9 | import shutil 10 | 11 | from hashlib import md5 12 | 13 | # custom db code start 14 | 15 | 16 | # custom index code start 17 | # source of classes in index.classes_code 18 | # index code start 19 | class WithAIndex(HashIndex): 20 | 21 | def __init__(self, *args, **kwargs): 22 | kwargs['entry_line_format'] = '<32s32sIIcI' 23 | kwargs['hash_lim'] = 4 * 1024 24 | super(WithAIndex, self).__init__(*args, **kwargs) 25 | 26 | def make_key_value(self, data): 27 | a_val = data.get("a") 28 | if a_val: 29 | if isinstance(a_val, int): 30 | a_val = str(a_val) 31 | # unnecessary code? 32 | # if not isinstance(a_val, bytes): 33 | # a_val = bytes(a_val, 'utf-8') 34 | return md5(a_val.encode()).hexdigest(), {} 35 | return None 36 | 37 | def make_key(self, key): 38 | if not isinstance(key, bytes): 39 | key = bytes(key, 'utf-8') 40 | return md5(key).hexdigest() 41 | -------------------------------------------------------------------------------- /tests/index_files/05custom_hash_index.py: -------------------------------------------------------------------------------- 1 | # custom_hash_index 2 | # CustomHashIndex 3 | 4 | # inserted automatically 5 | import os 6 | import marshal 7 | 8 | import struct 9 | import shutil 10 | 11 | from hashlib import md5 12 | 13 | # custom db code start 14 | 15 | 16 | # custom index code start 17 | from CodernityDB3 import rr_cache 18 | # source of classes in index.classes_code 19 | # index code start 20 | 21 | 22 | class CustomHashIndex(HashIndex): 23 | 24 | def __init__(self, *args, **kwargs): 25 | kwargs['entry_line_format'] = '32sIIIcI' 26 | kwargs['hash_lim'] = 1 27 | super(CustomHashIndex, self).__init__(*args, **kwargs) 28 | 29 | def make_key_value(self, data): 30 | d = data.get('test') 31 | if d is None: 32 | return None 33 | if d > str(5): 34 | k = 1 35 | else: 36 | k = 0 37 | return k, dict(test=d) 38 | 39 | def make_key(self, key): 40 | return key 41 | -------------------------------------------------------------------------------- /tests/misc/words.txt: -------------------------------------------------------------------------------- 1 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque mollis lorem vel urna bibendum ultricies. Quisque vitae neque ac ipsum aliquet dictum. Quisque egestas magna sed urna pellentesque imperdiet. Nunc faucibus varius viverra. Pellentesque sagittis pulvinar faucibus. Nulla placerat pharetra sollicitudin. Quisque dolor mi, convallis et adipiscing id, tempor et tortor. Vestibulum et mi vitae sem malesuada tincidunt. Nullam malesuada nisi non est dignissim sed egestas dui ultrices. Aliquam ut velit dui, at semper dolor. Nam ultricies neque eu sem egestas porta convallis quam gravida. Nam vestibulum, lorem et vulputate lacinia, ipsum dui tempor orci, consequat porttitor enim quam id velit. Sed dapibus fermentum elementum. Curabitur at augue vitae libero ornare lacinia ut ut velit. Nam euismod pharetra egestas. Integer nec purus vitae nisi posuere vulputate. 2 | 3 | Vestibulum quis urna ligula. Duis fringilla nunc vitae velit aliquam ac varius nunc pulvinar. Praesent velit massa, lobortis in tempor vitae, mattis sit amet nunc. Integer eleifend fringilla viverra. Donec in velit ut sem imperdiet tempus non non metus. Nulla sodales est in purus ornare feugiat. Donec volutpat quam at est feugiat sed euismod dui viverra. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Integer sed metus id mi lobortis. 4 | 5 | Codernity 6 | -------------------------------------------------------------------------------- /tests/patch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from CodernityDB3 import patch 19 | 20 | 21 | class TestPatches: 22 | 23 | def test_lfu(self): 24 | 25 | from CodernityDB3.lfu import lfu_cache 26 | 27 | assert lfu_cache.__name__ == 'lfu_cache' 28 | del lfu_cache 29 | 30 | from threading import RLock 31 | patch.patch_lfu(RLock) 32 | from CodernityDB3.lfu import lfu_cache 33 | 34 | assert lfu_cache.__name__ != 'lfu_cache' 35 | -------------------------------------------------------------------------------- /tests/shard_tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | import pytest 20 | from CodernityDB3.database import Database 21 | from CodernityDB3.sharded_hash import ShardedUniqueHashIndex 22 | from CodernityDB3.index import IndexPreconditionsException 23 | 24 | 25 | class ShardedUniqueHashIndex5(ShardedUniqueHashIndex): 26 | 27 | custom_header = 'from CodernityDB3.sharded_hash import ShardedUniqueHashIndex' 28 | 29 | def __init__(self, *args, **kwargs): 30 | kwargs['sh_nums'] = 5 31 | super(ShardedUniqueHashIndex5, self).__init__(*args, **kwargs) 32 | 33 | 34 | class ShardedUniqueHashIndex10(ShardedUniqueHashIndex): 35 | 36 | custom_header = 'from CodernityDB3.sharded_hash import ShardedUniqueHashIndex' 37 | 38 | def __init__(self, *args, **kwargs): 39 | kwargs['sh_nums'] = 10 40 | super(ShardedUniqueHashIndex10, self).__init__(*args, **kwargs) 41 | 42 | 43 | class ShardedUniqueHashIndex50(ShardedUniqueHashIndex): 44 | 45 | custom_header = 'from CodernityDB3.sharded_hash import ShardedUniqueHashIndex' 46 | 47 | def __init__(self, *args, **kwargs): 48 | kwargs['sh_nums'] = 50 49 | super(ShardedUniqueHashIndex50, self).__init__(*args, **kwargs) 50 | 51 | 52 | class ShardTests: 53 | 54 | def test_create(self, tmpdir): 55 | db = Database(str(tmpdir) + '/db') 56 | db.create(with_id_index=False) 57 | db.add_index(ShardedUniqueHashIndex(db.path, 'id', sh_nums=3)) 58 | 59 | @pytest.mark.parametrize(('sh_nums', ), [(x,) for x in (5, 10, 50)]) 60 | def test_num_shards(self, tmpdir, sh_nums): 61 | db = Database(str(tmpdir) + '/db') 62 | db.create(with_id_index=False) 63 | n = globals()['ShardedUniqueHashIndex%d' % sh_nums] 64 | db.add_index(n(db.path, 'id')) 65 | assert db.id_ind.sh_nums == sh_nums 66 | 67 | @pytest.mark.parametrize(('sh_nums', ), [(x,) for x in (5, 10, 50)]) 68 | def test_insert_get(self, tmpdir, sh_nums): 69 | db = Database(str(tmpdir) + '/db') 70 | db.create(with_id_index=False) 71 | n = globals()['ShardedUniqueHashIndex%d' % sh_nums] 72 | db.add_index(n(db.path, 'id')) 73 | l = [] 74 | for x in range(100): 75 | l.append(db.insert(dict(x=x))['_id']) 76 | 77 | for curr in l: 78 | assert db.get('id', curr)['_id'] == curr 79 | 80 | @pytest.mark.parametrize(('sh_nums', ), [(x,) for x in (5, 10, 50)]) 81 | def test_all(self, tmpdir, sh_nums): 82 | db = Database(str(tmpdir) + '/db') 83 | db.create(with_id_index=False) 84 | n = globals()['ShardedUniqueHashIndex%d' % sh_nums] 85 | db.add_index(n(db.path, 'id')) 86 | l = [] 87 | for x in range(100): 88 | d = db.insert(dict(x=x))['_id'] 89 | l.append(d) 90 | 91 | for curr in db.all('id'): 92 | d = curr['_id'] 93 | l.remove(d) 94 | 95 | assert l == [] 96 | 97 | def test_to_many_shards(self, tmpdir): 98 | db = Database(str(tmpdir) + '/db') 99 | db.create(with_id_index=False) 100 | # it's ok to use sharded directly there 101 | with pytest.raises(IndexPreconditionsException): 102 | db.add_index(ShardedUniqueHashIndex(db.path, 'id', sh_nums=300)) 103 | with pytest.raises(IndexPreconditionsException): 104 | db.add_index(ShardedUniqueHashIndex(db.path, 'id', sh_nums=256)) 105 | 106 | def test_compact_shards(self, tmpdir): 107 | db = Database(str(tmpdir) + '/db') 108 | db.create(with_id_index=False) 109 | db.add_index(ShardedUniqueHashIndex5(db.path, 'id')) 110 | 111 | for x in range(100): 112 | db.insert({'x': x}) 113 | 114 | db.compact() 115 | assert db.count(db.all, 'id') == 100 116 | -------------------------------------------------------------------------------- /tests/shared.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from CodernityDB3.database import Database, RecordDeleted, RecordNotFound 19 | from CodernityDB3.database import DatabaseException, RevConflict, DatabasePathException, DatabaseConflict, PreconditionsException, IndexConflict 20 | 21 | from CodernityDB3.hash_index import HashIndex, UniqueHashIndex, MultiHashIndex 22 | from CodernityDB3.index import IndexException, TryReindexException, IndexNotFoundException, IndexPreconditionsException 23 | 24 | from CodernityDB3.tree_index import TreeBasedIndex, MultiTreeBasedIndex 25 | 26 | from CodernityDB3.debug_stuff import database_step_by_step 27 | 28 | from CodernityDB3 import rr_cache 29 | 30 | import six 31 | import pytest 32 | import os 33 | import random 34 | from hashlib import md5 35 | 36 | try: 37 | from collections import Counter 38 | except ImportError: 39 | class Counter(dict): 40 | 'Mapping where default values are zero' 41 | def __missing__(self, key): 42 | return 0 43 | 44 | 45 | class CustomHashIndex(HashIndex): 46 | 47 | def __init__(self, *args, **kwargs): 48 | # kwargs['entry_line_format'] = '<32sIIIcI' 49 | kwargs['key_format'] = 'I' 50 | kwargs['hash_lim'] = 1 51 | super(CustomHashIndex, self).__init__(*args, **kwargs) 52 | 53 | def make_key_value(self, data): 54 | d = data.get('test') 55 | if d is None: 56 | return None 57 | if d > 5: 58 | k = 1 59 | else: 60 | k = 0 61 | return k, dict(test=d) 62 | 63 | def make_key(self, key): 64 | return key 65 | 66 | 67 | class Md5Index(HashIndex): 68 | 69 | def __init__(self, *args, **kwargs): 70 | # kwargs['entry_line_format'] = '<32s32sIIcI' 71 | kwargs['key_format'] = '16s' 72 | kwargs['hash_lim'] = 4 * 1024 73 | super(Md5Index, self).__init__(*args, **kwargs) 74 | 75 | def make_key_value(self, data): 76 | return md5(data['name']).digest(), None 77 | 78 | def make_key(self, key): 79 | return md5(key).digest() 80 | 81 | 82 | class WithAIndex2(HashIndex): 83 | 84 | def __init__(self, *args, **kwargs): 85 | # kwargs['entry_line_format'] = '<32s32sIIcI' 86 | kwargs['key_format'] = '16s' 87 | kwargs['hash_lim'] = 4 * 1024 88 | super(WithAIndex, self).__init__(*args, **kwargs) 89 | 90 | def make_key_value(self, data): 91 | a_val = data.get("a") 92 | if a_val is not None: 93 | if not isinstance(a_val, bytes): 94 | a_val = bytes(a_val, 'utf-8') 95 | return md5(a_val).digest(), None 96 | return None 97 | 98 | def make_key(self, key): 99 | if not isinstance(key, bytes): 100 | key = bytes(key, 'utf-8') 101 | return md5(key).digest() 102 | 103 | 104 | class WithAIndex(HashIndex): 105 | 106 | def __init__(self, *args, **kwargs): 107 | # kwargs['entry_line_format'] = '<32s32sIIcI' 108 | kwargs['key_format'] = '16s' 109 | kwargs['hash_lim'] = 4 * 1024 110 | super(WithAIndex, self).__init__(*args, **kwargs) 111 | 112 | def make_key_value(self, data): 113 | a_val = data.get("a") 114 | if a_val is not None: 115 | if isinstance(a_val, int): 116 | a_val = str(a_val) 117 | if not isinstance(a_val, bytes): 118 | a_val = bytes(a_val, 'utf-8') 119 | return md5(a_val).digest(), None 120 | return None 121 | 122 | def make_key(self, key): 123 | if isinstance(key, int): 124 | key = str(key) 125 | if not isinstance(key, bytes): 126 | key = bytes(key, 'utf-8') 127 | return md5(key).digest() 128 | 129 | 130 | class Simple_TreeIndex(TreeBasedIndex): 131 | 132 | def __init__(self, *args, **kwargs): 133 | kwargs['node_capacity'] = 100 134 | kwargs['key_format'] = 'I' 135 | super(Simple_TreeIndex, self).__init__(*args, **kwargs) 136 | 137 | def make_key_value(self, data): 138 | t_val = data.get('t') 139 | if t_val is not None: 140 | return t_val, {} 141 | return None 142 | 143 | def make_key(self, key): 144 | return key 145 | 146 | 147 | class WithRun_Index(HashIndex): 148 | 149 | def __init__(self, *args, **kwargs): 150 | # kwargs['entry_line_format'] = '<32sIIIcI' 151 | kwargs['key_format'] = 'I' 152 | kwargs['hash_lim'] = 4 * 1024 153 | super(WithRun_Index, self).__init__(*args, **kwargs) 154 | 155 | def run_sum(self, db_obj, key): 156 | gen = db_obj.get_many(index_name=self.name, key=key, 157 | limit=-1, with_storage=True) 158 | vals = [] 159 | while True: 160 | try: 161 | d = next(gen) 162 | except StopIteration: 163 | break 164 | else: 165 | vals.append(d.get('x', 0)) 166 | return sum(vals) 167 | 168 | def make_key_value(self, data): 169 | a_val = data.get("a") 170 | if a_val is not None: 171 | out = {'x': data.get('x')} 172 | return a_val, out 173 | return None 174 | 175 | def make_key(self, key): 176 | return key 177 | 178 | 179 | class WithRunEdit_Index(HashIndex): 180 | 181 | def __init__(self, *args, **kwargs): 182 | # kwargs['entry_line_format'] = '<32sIIIcI' 183 | kwargs['key_format'] = 'I' 184 | kwargs['hash_lim'] = 4 * 1024 185 | super(WithRunEdit_Index, self).__init__(*args, **kwargs) 186 | 187 | def run_sum(self, db_obj, key): 188 | gen = db_obj.get_many(index_name=self.name, key=key, 189 | limit=-1, with_storage=True) 190 | vals = [] 191 | while True: 192 | try: 193 | d = next(gen) 194 | except StopIteration: 195 | break 196 | else: 197 | vals.append(d.get('x', 0)) 198 | return sum(vals) 199 | 200 | def make_key_value(self, data): 201 | a_val = data.get("a") 202 | if a_val is not None: 203 | out = {'x': data.get('x') * 2} 204 | return a_val, out 205 | return None 206 | 207 | def make_key(self, key): 208 | return key 209 | 210 | 211 | class TreeMultiTest(MultiTreeBasedIndex): 212 | 213 | custom_header = """from CodernityDB3.tree_index import MultiTreeBasedIndex 214 | """ 215 | 216 | def __init__(self, *args, **kwargs): 217 | kwargs['key_format'] = '16s' 218 | super(TreeMultiTest, self).__init__(*args, **kwargs) 219 | self.__l = kwargs.get('w_len', 2) 220 | 221 | def make_key_value(self, data): 222 | name = data['w'] 223 | l = self.__l 224 | max_l = len(name) 225 | out = set() 226 | for x in range(l - 1, max_l): 227 | m = (name, ) 228 | for y in range(0, x): 229 | m += (name[y + 1:],) 230 | out.update(set(''.join(x).rjust( 231 | 16, '_').lower() for x in zip(*m))) # ignore import error 232 | return out, dict(name=name) 233 | 234 | def make_key(self, key): 235 | return key.rjust(16, b'_').lower() 236 | 237 | 238 | class DB_Tests: 239 | 240 | def setup_method(self, method): 241 | self.counter = Counter() 242 | 243 | def test_update_conflict(self, tmpdir): 244 | db = self._db(os.path.join(str(tmpdir), 'db')) 245 | db.create() 246 | doc = dict(a=1) 247 | db.insert(doc) 248 | doc2 = doc.copy() 249 | doc2['_rev'] = '00000000' 250 | with pytest.raises(RevConflict): 251 | db.update(doc2) 252 | 253 | def test_wrong_id(self, tmpdir): 254 | db = self._db(os.path.join(str(tmpdir), 'db')) 255 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 256 | db.create() 257 | 258 | with pytest.raises(IndexPreconditionsException): 259 | db.insert(dict(_id='1', a=1)) 260 | 261 | with pytest.raises(IndexPreconditionsException): 262 | db.insert(dict(_id=1, a=1)) 263 | 264 | def test_open_close(self, tmpdir): 265 | db = self._db(os.path.join(str(tmpdir), 'db')) 266 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 267 | db.create() 268 | l = [] 269 | for i in range(5): 270 | c = dict(i=i) 271 | db.insert(c) 272 | l.append(c) 273 | db.close() 274 | db.open() 275 | db.close() 276 | db2 = self._db(os.path.join(str(tmpdir), 'db')) 277 | # db2.set_indexes([UniqueHashIndex(db.path, 'id')]) 278 | db2.open() 279 | for j in range(5): 280 | assert l[j] == db2.get('id', l[j]['_id']) 281 | db2.close() 282 | 283 | def test_destroy(self, tmpdir): 284 | db = self._db(os.path.join(str(tmpdir), 'db')) 285 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 286 | db.create() 287 | for i in range(5): 288 | db.insert(dict(i=i)) 289 | db.destroy() 290 | db = self._db(os.path.join(str(tmpdir), 'db')) 291 | with pytest.raises(DatabasePathException): 292 | db.open() 293 | 294 | def test_exists(self, tmpdir): 295 | p = os.path.join(str(tmpdir), 'db') 296 | db = self._db(p) 297 | assert db.exists() is False 298 | db.create() 299 | assert db.exists() is True 300 | db.destroy() 301 | assert db.exists() is False 302 | 303 | def test_double_create(self, tmpdir): 304 | p = os.path.join(str(tmpdir), 'db') 305 | db = self._db(p) 306 | db.create() 307 | db2 = self._db(p) 308 | with pytest.raises((DatabaseConflict, IndexConflict)): 309 | db2.create() 310 | db2.open() 311 | db.destroy() 312 | db2 = self._db(p) 313 | db.create() 314 | 315 | ''' 316 | def test_real_life_example_random(self, tmpdir, operations): 317 | 318 | db = self._db(os.path.join(str(tmpdir), 'db')) 319 | db.set_indexes([UniqueHashIndex(db.path, 'id'), 320 | WithAIndex(db.path, 'with_a'), 321 | CustomHashIndex(db.path, 'custom'), 322 | Simple_TreeIndex(db.path, 'tree')]) 323 | db.create() 324 | database_step_by_step(db) 325 | 326 | inserted = {} 327 | updated = {} 328 | 329 | def _insert(): 330 | doc = dict(i=random.randint(0, 1000)) 331 | if random.randint(0, 1): 332 | doc['a'] = random.randint(1, 15) 333 | if random.randint(0, 1): 334 | doc['test'] = random.randint(1, 9) 335 | if doc['test'] > 5: 336 | self.counter['r'] += 1 337 | else: 338 | self.counter['l'] += 1 339 | if random.randint(0, 1): 340 | doc['t'] = random.randint(0, 500) 341 | db.insert(doc) 342 | inserted[doc['_id']] = doc 343 | return True 344 | 345 | def _update(): 346 | vals = list(inserted.values()) 347 | if not vals: 348 | return False 349 | doc = random.choice(vals) 350 | a = random.randint(0, 1000) 351 | doc['upd'] = a 352 | was = doc.get('test') 353 | if was is not None: 354 | if was > 5: 355 | self.counter['r'] -= 1 356 | else: 357 | self.counter['l'] -= 1 358 | doc['test'] = random.randint(1, 9) 359 | if doc['test'] > 5: 360 | self.counter['r'] += 1 361 | else: 362 | self.counter['l'] += 1 363 | db.update(doc) 364 | assert db.get('id', doc['_id'])['upd'] == a 365 | updated[doc['_id']] = doc 366 | return True 367 | 368 | def _delete(): 369 | vals = list(inserted.values()) 370 | if not vals: 371 | return False 372 | doc = random.choice(vals) 373 | was = doc.get('test') 374 | if was is not None: 375 | if was > 5: 376 | self.counter['r'] -= 1 377 | else: 378 | self.counter['l'] -= 1 379 | db.delete(doc) 380 | del inserted[doc['_id']] 381 | try: 382 | del updated[doc['_id']] 383 | except: 384 | pass 385 | return True 386 | 387 | def _get(): 388 | vals = list(inserted.values()) 389 | if not vals: 390 | return False 391 | doc = random.choice(vals) 392 | got = db.get('id', doc['_id']) 393 | assert got == doc 394 | return True 395 | 396 | def _compact(): 397 | db.compact() 398 | 399 | def _reindex(): 400 | db.reindex() 401 | 402 | def count_and_check(): 403 | assert len(inserted) == db.count(db.all, 'id') 404 | l_c = db.count(db.get_many, 'custom', key=0, limit=operations) 405 | r_c = db.count(db.get_many, 'custom', key=1, limit=operations) 406 | same = set(map(lambda x: x['_id'], db.get_many('custom', key=0, limit=operations))).intersection(set(map(lambda x: x['_id'], db.get_many('custom', key=1, limit=operations)))) 407 | assert same == set() 408 | assert self.counter['l'] == l_c 409 | assert self.counter['r'] == r_c 410 | assert self.counter['l'] + self.counter[ 411 | 'r'] == db.count(db.all, 'custom') 412 | 413 | fcts = ( 414 | _insert,) * 20 + (_get,) * 10 + (_update,) * 10 + (_delete,) * 5 415 | for i in range(operations): 416 | f = random.choice(fcts) 417 | f() 418 | 419 | # db.reindex() 420 | count_and_check() 421 | 422 | db.reindex() 423 | 424 | count_and_check() 425 | 426 | fcts = ( 427 | _insert,) * 20 + (_get,) * 10 + (_update,) * 10 + (_delete,) * 5 428 | for i in range(operations): 429 | f = random.choice(fcts) 430 | f() 431 | 432 | count_and_check() 433 | 434 | db.close()''' 435 | 436 | def test_add_new_index_to_existing_db(self, tmpdir): 437 | db = self._db(os.path.join(str(tmpdir), 'db')) 438 | 439 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 440 | db.create() 441 | 442 | new_index = UniqueHashIndex(db.path, 'unique_hash_index') 443 | db.add_index(new_index) 444 | 445 | new_index = Md5Index(db.path, 'md5_index') 446 | db.add_index(new_index) 447 | 448 | new_index = WithAIndex(db.path, 'with_a_index') 449 | db.add_index(new_index) 450 | 451 | assert len(db.indexes) == 4 452 | 453 | def test_add_new_index_to_existing_db_2(self, tmpdir): 454 | db = self._db(os.path.join(str(tmpdir), 'db')) 455 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 456 | db.create() 457 | 458 | new_index = HashIndex(db.path, 'hash_index') 459 | db.add_index(new_index) 460 | 461 | new_index = Md5Index(db.path, 'md5_index') 462 | db.add_index(new_index) 463 | 464 | new_index = WithAIndex(db.path, 'withA_index') 465 | db.add_index(new_index) 466 | 467 | for y in range(100): 468 | db.insert(dict(y=y)) 469 | 470 | for index in ('id', 'hash_index', 'md5_index', 'withA_index'): 471 | elements = db.all('hash_index') 472 | assert len(list(elements)) == 100 473 | 474 | def test_add_duplicate_index_throws_exception(self, tmpdir): 475 | db = self._db(os.path.join(str(tmpdir), 'db')) 476 | 477 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 478 | db.create() 479 | 480 | new_index = UniqueHashIndex(db.path, 'another_index') 481 | db.add_index(new_index) 482 | 483 | new_index = UniqueHashIndex(db.path, 'another_index') 484 | with pytest.raises(IndexException): 485 | db.add_index(new_index) 486 | 487 | def test_add_new_index_from_string(self, tmpdir): 488 | 489 | db = self._db(os.path.join(str(tmpdir), 'db')) 490 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 491 | db.create() 492 | file_names = ('tests/index_files/03md5_index.py', 493 | 'tests/index_files/04withA_index.py', 494 | 'tests/index_files/05custom_hash_index.py') 495 | 496 | indexes_names = [db.add_index(open(f).read()) for f in file_names] 497 | 498 | assert len(db.indexes) == len(file_names) + 1 # 'id' + from files 499 | 500 | for y in range(100): 501 | db.insert(dict(a='blah', test='blah', name=str(y), y=y)) 502 | 503 | for index_name in indexes_names: 504 | assert db.count(db.all, index_name) == 100 505 | 506 | def test_adding_index_creates_dot_py_file(self, tmpdir): 507 | db = self._db(os.path.join(str(tmpdir), 'db')) 508 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 509 | db.create() 510 | 511 | path_indexes = os.path.join(db.path, '_indexes') 512 | 513 | before = set(os.listdir(path_indexes)) 514 | 515 | new_index = Md5Index(db.path, 'md5_index') 516 | db.add_index(new_index) 517 | 518 | after = set(os.listdir(path_indexes)) 519 | 520 | added_file = tuple(after - before)[0] 521 | 522 | assert len(after) == len(before) + 1 523 | assert new_index.name + '.py' in added_file 524 | 525 | def test_removing_index_from_db(self, tmpdir): 526 | db = self._db(os.path.join(str(tmpdir), 'db')) 527 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 528 | db.create() 529 | path_indexes = os.path.join(db.path, '_indexes') 530 | 531 | def file_exists_in_indexes_dir(name): 532 | return [f for f in os.listdir(path_indexes) if name in f] 533 | 534 | indexes = [klass(db.path, klass.__name__) for klass in ( 535 | HashIndex, WithAIndex, Md5Index)] 536 | 537 | # with name 538 | for index in indexes: 539 | index_name = index.name 540 | db.add_index(index) 541 | assert db.get_index_details(index_name) # wants name 542 | assert file_exists_in_indexes_dir(index_name) 543 | 544 | db.destroy_index(index_name) 545 | 546 | assert not file_exists_in_indexes_dir(index_name) 547 | # print(file_exists_in_indexes_dir(index_name)) 548 | with pytest.raises(IndexNotFoundException): 549 | db.get_index_details(index_name) 550 | # with instance 551 | indexes = [klass(db.path, klass.__name__) for klass in ( 552 | HashIndex, WithAIndex, Md5Index)] 553 | for index in indexes: 554 | index_name = index.name 555 | db.add_index(index) 556 | assert db.get_index_details(index_name) # wants name 557 | assert file_exists_in_indexes_dir(index_name) 558 | index_to_destory = db.indexes_names[index_name] 559 | db.destroy_index(index_to_destory) 560 | 561 | assert not file_exists_in_indexes_dir(index_name) 562 | # print(file_exists_in_indexes_dir(index_name)) 563 | with pytest.raises(IndexNotFoundException): 564 | db.get_index_details(index_name) 565 | 566 | # with other instance 567 | indexes = [klass(db.path, klass.__name__) for klass in ( 568 | HashIndex, WithAIndex, Md5Index)] 569 | for index in indexes: 570 | index_name = index.name 571 | db.add_index(index) 572 | assert db.get_index_details(index_name) # wants name 573 | assert file_exists_in_indexes_dir(index_name) 574 | 575 | klass = index.__class__ 576 | 577 | with pytest.raises(DatabaseException): 578 | db.destroy_index(klass(db.path, klass.__name__)) 579 | 580 | assert file_exists_in_indexes_dir(index_name) 581 | # print(file_exists_in_indexes_dir(index_name)) 582 | assert db.get_index_details(index_name) 583 | 584 | def test_removing_index_from_db_2(self, tmpdir): 585 | '''indexes are added from strings''' 586 | 587 | db = self._db(os.path.join(str(tmpdir), 'db')) 588 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 589 | db.create() 590 | path_indexes = os.path.join(db.path, '_indexes') 591 | 592 | def file_exists_in_indexes_dir(name): 593 | return [f for f in os.listdir(path_indexes) if name in f] 594 | 595 | file_names = ('tests/index_files/03md5_index.py', 596 | 'tests/index_files/04withA_index.py', 597 | 'tests/index_files/05custom_hash_index.py') 598 | 599 | indexes_names = [db.add_index(open(f).read()) for f in file_names] 600 | 601 | for index_name in indexes_names: 602 | assert db.get_index_details(index_name) 603 | assert file_exists_in_indexes_dir(index_name) 604 | 605 | db.destroy_index(index_name) 606 | 607 | assert not file_exists_in_indexes_dir(index_name) 608 | with pytest.raises(IndexException): 609 | db.get_index_details(index_name) 610 | 611 | def test_update_index(self, tmpdir): 612 | db = self._db(os.path.join(str(tmpdir), 'db')) 613 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 614 | db.create() 615 | 616 | hash_index = HashIndex(db.path, 'my_index') 617 | db.add_index(hash_index) 618 | assert db.get_index_details(hash_index.name) 619 | 620 | db.destroy_index(hash_index.name) 621 | with pytest.raises(IndexNotFoundException): 622 | db.get_index_details(hash_index.name) 623 | 624 | new_index = Md5Index(db.path, 'my_index') 625 | db.add_index(new_index) 626 | assert db.get_index_details(new_index.name) 627 | 628 | def test_create_index_duplicate_from_object(self, tmpdir): 629 | db = self._db(os.path.join(str(tmpdir), 'db')) 630 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 631 | db.create() 632 | 633 | hash_index = HashIndex(db.path, 'my_index') 634 | db.add_index(hash_index) 635 | 636 | l = len(db.indexes) 637 | hash_index = HashIndex(db.path, 'my_index') 638 | with pytest.raises(IndexException): 639 | db.add_index(hash_index) 640 | 641 | assert l == len(db.indexes) 642 | 643 | def test_create_index_duplicate_from_string(self, tmpdir): 644 | db = self._db(os.path.join(str(tmpdir), 'db')) 645 | db.set_indexes([UniqueHashIndex(db.path, 'id')]) 646 | db.create() 647 | path_indexes = os.path.join(db.path, '_indexes') 648 | 649 | file_name = 'tests/index_files/04withA_index.py' 650 | 651 | db.add_index(open(file_name).read()) 652 | l = len(db.indexes) 653 | with pytest.raises(IndexException): 654 | db.add_index(open(file_name).read()) 655 | 656 | assert l == len(db.indexes) 657 | 658 | def test_create_with_path(self, tmpdir): 659 | p = os.path.join(str(tmpdir), 'db') 660 | 661 | db = self._db(None) 662 | db.create(p, with_id_index=False) 663 | 664 | ind = UniqueHashIndex(p, 'id') 665 | db.add_index(ind) 666 | 667 | for x in range(10): 668 | db.insert(dict(x=x)) 669 | 670 | assert db.count(db.all, 'id') == 10 671 | 672 | def test_auto_add_id_index(self, tmpdir): 673 | p = os.path.join(str(tmpdir), 'db') 674 | 675 | db = self._db(None) 676 | db.initialize(p) 677 | db.create() 678 | 679 | ind = UniqueHashIndex(p, 'id') 680 | with pytest.raises(IndexException): 681 | db.add_index(ind) 682 | 683 | for x in range(10): 684 | db.insert(dict(x=x)) 685 | 686 | assert db.count(db.all, 'id') == 10 687 | 688 | def test_auto_add_id_index_without_initialize(self, tmpdir): 689 | p = os.path.join(str(tmpdir), 'db') 690 | 691 | db = self._db(None) 692 | db.create(p) 693 | 694 | ind = UniqueHashIndex(p, 'id') 695 | with pytest.raises(IndexException): 696 | db.add_index(ind) 697 | 698 | for x in range(10): 699 | db.insert(dict(x=x)) 700 | 701 | assert db.count(db.all, 'id') == 10 702 | 703 | def test_open_with_path(self, tmpdir): 704 | p = os.path.join(str(tmpdir), 'db') 705 | 706 | db = self._db(p) 707 | ind = UniqueHashIndex(p, 'id') 708 | db.set_indexes([ind]) 709 | db.create(with_id_index=False) 710 | 711 | for x in range(10): 712 | db.insert(dict(x=x)) 713 | 714 | db.close() 715 | 716 | db = self._db(p) 717 | db.open(p) 718 | 719 | assert db.count(db.all, 'id') == 10 720 | 721 | def test_create_path_delayed1(self, tmpdir): 722 | p = os.path.join(str(tmpdir), 'db') 723 | 724 | db = self._db(None) 725 | db.initialize(p) 726 | 727 | ind = UniqueHashIndex(p, 'id') 728 | db.add_index(ind, create=False) 729 | db.create(with_id_index=False) 730 | 731 | for x in range(10): 732 | db.insert(dict(x=x)) 733 | 734 | db.close() 735 | 736 | db = self._db(p) 737 | db.open(p) 738 | 739 | assert db.count(db.all, 'id') == 10 740 | 741 | def test_create_path_delayed2(self, tmpdir): 742 | p = os.path.join(str(tmpdir), 'db') 743 | 744 | db = self._db(None) 745 | db.initialize(p) 746 | db.create(with_id_index=False) 747 | 748 | ind = UniqueHashIndex(p, 'id') 749 | db.add_index(ind) 750 | 751 | for x in range(10): 752 | db.insert(dict(x=x)) 753 | 754 | db.close() 755 | 756 | db = self._db(p) 757 | db.open(p) 758 | 759 | assert db.count(db.all, 'id') == 10 760 | 761 | def test_compact_index_with_different_types(self, tmpdir): 762 | db = self._db(os.path.join(str(tmpdir), 'db')) 763 | ind_id = UniqueHashIndex(db.path, 'id') 764 | db.set_indexes([ind_id]) 765 | db.create() 766 | l = [] 767 | for i in range(10): 768 | c = dict(i=i) 769 | c.update(db.insert(c)) 770 | l.append(c) 771 | 772 | # with name 773 | for i in range(10): 774 | curr = l[i] 775 | c = db.get("id", curr['_id']) 776 | c['update'] = True 777 | c.update(db.update(c)) 778 | 779 | db.compact_index(ind_id.name) 780 | for j in range(10): 781 | curr = l[j] 782 | c = db.get('id', curr['_id']) 783 | assert c['_id'] == curr['_id'] 784 | assert c['i'] == j 785 | 786 | # with instance 787 | for i in range(10): 788 | curr = l[i] 789 | c = db.get("id", curr['_id']) 790 | c['update'] = True 791 | c.update(db.update(c)) 792 | 793 | ind_id = db.indexes_names[ind_id.name] 794 | db.compact_index(ind_id) 795 | for j in range(10): 796 | curr = l[j] 797 | c = db.get('id', curr['_id']) 798 | assert c['_id'] == curr['_id'] 799 | assert c['i'] == j 800 | 801 | # with different instance 802 | for i in range(10): 803 | curr = l[i] 804 | c = db.get("id", curr['_id']) 805 | c['update'] = True 806 | c.update(db.update(c)) 807 | 808 | with pytest.raises(DatabaseException): 809 | db.compact_index(UniqueHashIndex(db.path, 'id')) 810 | 811 | for j in range(10): 812 | curr = l[j] 813 | c = db.get('id', curr['_id']) 814 | assert c['_id'] == curr['_id'] 815 | assert c['i'] == j 816 | 817 | db.close() 818 | 819 | def test_reindex_index_with_different_types(self, tmpdir): 820 | db = self._db(os.path.join(str(tmpdir), 'db')) 821 | ind_id = UniqueHashIndex(db.path, 'id') 822 | ind_a = WithAIndex(db.path, 'with_a') 823 | ind_c = CustomHashIndex(db.path, 'custom') 824 | 825 | db.set_indexes([ind_id, 826 | ind_a, 827 | ind_c]) 828 | ind_id = db.indexes_names[ind_id.name] 829 | ind_a = db.indexes_names[ind_a.name] 830 | 831 | db.create() 832 | 833 | l = [] 834 | for i in range(100): 835 | c = dict(i=i) 836 | db.insert(c) 837 | l.append(c) 838 | 839 | for i in range(10): 840 | nr = random.randint(0, len(l) - 1) 841 | curr = l[nr] 842 | db.delete(curr) 843 | del(l[nr]) 844 | 845 | # index id 846 | with pytest.raises(DatabaseException): 847 | db.reindex_index(ind_id) 848 | 849 | for j in range(len(l)): 850 | curr = l[j] 851 | c = db.get('id', curr['_id']) 852 | assert c['_id'] == curr['_id'] 853 | 854 | # with instance 855 | for i in range(10): 856 | nr = random.randint(0, len(l) - 1) 857 | curr = l[nr] 858 | db.delete(curr) 859 | del(l[nr]) 860 | 861 | db.reindex_index(ind_a) 862 | for j in range(len(l)): 863 | curr = l[j] 864 | c = db.get('id', curr['_id']) 865 | assert c['_id'] == curr['_id'] 866 | 867 | # with name 868 | for i in range(10): 869 | nr = random.randint(0, len(l) - 1) 870 | curr = l[nr] 871 | db.delete(curr) 872 | del(l[nr]) 873 | 874 | db.reindex_index(ind_c.name) 875 | for j in range(len(l)): 876 | curr = l[j] 877 | c = db.get('id', curr['_id']) 878 | assert c['_id'] == curr['_id'] 879 | 880 | for i in range(10): 881 | nr = random.randint(0, len(l) - 1) 882 | curr = l[nr] 883 | db.delete(curr) 884 | del(l[nr]) 885 | # with different instance 886 | with pytest.raises(DatabaseException): 887 | db.reindex_index(WithAIndex(db.path, 'with_a')) 888 | 889 | for j in range(len(l)): 890 | curr = l[j] 891 | c = db.get('id', curr['_id']) 892 | assert c['_id'] == curr['_id'] 893 | 894 | db.close() 895 | 896 | def test_add_new_index_update_before_reindex_new_value(self, tmpdir): 897 | db = self._db(os.path.join(str(tmpdir), 'db')) 898 | db.create() 899 | for x in range(20): 900 | db.insert(dict(t=x, test=x)) 901 | el = db.insert(dict(t=1, test=1)) 902 | el['new_data'] = 'new' 903 | el['test'] = 2 904 | db.add_index(CustomHashIndex(db.path, 'custom')) 905 | db.add_index(Simple_TreeIndex(db.path, 'tree')) 906 | with pytest.raises(TryReindexException): 907 | db.update(el) 908 | 909 | def test_add_new_index_update_before_reindex_old_value(self, tmpdir): 910 | db = self._db(os.path.join(str(tmpdir), 'db')) 911 | db.create() 912 | for x in range(20): 913 | db.insert(dict(t=x, test=x)) 914 | el = db.insert(dict(t=1, test=1)) 915 | el['new_data'] = 'new' 916 | db.add_index(CustomHashIndex(db.path, 'custom')) 917 | db.add_index(Simple_TreeIndex(db.path, 'tree')) 918 | with pytest.raises(TryReindexException): 919 | db.update(el) 920 | 921 | def test_add_new_index_delete_before_reindex(self, tmpdir): 922 | db = self._db(os.path.join(str(tmpdir), 'db')) 923 | db.create() 924 | for x in range(20): 925 | db.insert(dict(t=x, a=x)) 926 | el = db.insert(dict(t=1, a=1)) 927 | # el['new_data']='new' 928 | db.add_index(WithAIndex(db.path, 'with_a')) 929 | db.add_index(Simple_TreeIndex(db.path, 'tree')) 930 | db.delete(el) 931 | 932 | def test_runnable_functions(self, tmpdir): 933 | db = self._db(os.path.join(str(tmpdir), 'db')) 934 | db.create() 935 | db.add_index(WithRun_Index(db.path, 'run')) 936 | for x in range(20): 937 | db.insert(dict(a=x % 2, x=x)) 938 | assert db.run('run', 'sum', 0) == 90 939 | assert db.run('run', 'sum', 1) == 100 940 | assert db.run('run', 'sum', 2) == 0 941 | # not existsing 942 | with pytest.raises(IndexException): 943 | db.run('run', 'not_existing', 10) 944 | # existing, but should't run because of run_ prefix 945 | with pytest.raises(IndexException): 946 | db.run('run', 'destroy', 10) 947 | 948 | def test_get_error(self, tmpdir): 949 | db = self._db(os.path.join(str(tmpdir), 'db')) 950 | db.create() 951 | _id = md5('test'.encode()).hexdigest() 952 | with pytest.raises(RecordNotFound): 953 | db.get('id', _id) 954 | db.insert(dict(_id=_id, test='test')) 955 | db.get('id', _id) 956 | db.close() 957 | 958 | def test_edit_index(self, tmpdir): 959 | db = self._db(os.path.join(str(tmpdir), 'db')) 960 | db.create() 961 | db.add_index(WithRun_Index(db.path, 'run')) 962 | for x in range(20): 963 | db.insert(dict(a=x % 2, x=x)) 964 | assert db.run('run', 'sum', 0) == 90 965 | assert db.run('run', 'sum', 1) == 100 966 | assert db.run('run', 'sum', 2) == 0 967 | db.edit_index(WithRunEdit_Index(db.path, 'run')) 968 | assert db.run('run', 'sum', 0) == 90 969 | assert db.run('run', 'sum', 1) == 100 970 | assert db.run('run', 'sum', 2) == 0 971 | db.insert(dict(a=1, x=1)) 972 | assert db.run('run', 'sum', 1) == 102 973 | db.edit_index(WithRunEdit_Index(db.path, 'run'), reindex=True) 974 | assert db.run('run', 'sum', 0) == 90 * 2 975 | assert db.run('run', 'sum', 1) == 100 * 2 + 2 976 | assert db.run('run', 'sum', 2) == 0 977 | 978 | def test_add_index_bef(self, tmpdir): 979 | db = self._db(os.path.join(str(tmpdir), 'db')) 980 | with pytest.raises(PreconditionsException): 981 | db.add_index(CustomHashIndex(db.path, 'custom')) 982 | db.add_index(UniqueHashIndex(db.path, 'id')) 983 | db.add_index(CustomHashIndex(db.path, 'custom')) 984 | 985 | def test_multi_index(self, tmpdir): 986 | with open('tests/misc/words.txt', 'r') as f: 987 | data = f.read().split() 988 | words = map( 989 | lambda x: x.strip().replace('.', "").replace(',', ""), data) 990 | db = self._db(os.path.join(str(tmpdir), 'db')) 991 | db.create() 992 | db.add_index(TreeMultiTest(db.path, 'words')) 993 | for word in words: 994 | db.insert(dict(w=word)) 995 | assert db.count(db.all, 'words') == 3245 996 | assert db.get('words', 'Coder')['name'] == 'Codernity' 997 | assert db.get('words', 'dern')['name'] == 'Codernity' 998 | assert db.get('words', 'Codernity')['name'] == 'Codernity' 999 | 1000 | u = db.get('words', 'Codernit', with_doc=True) 1001 | doc = u['doc'] 1002 | doc['up'] = True 1003 | db.update(doc) 1004 | assert db.get('words', "dern")['name'] == "Codernity" 1005 | 1006 | db.delete(doc) 1007 | with pytest.raises(RecordNotFound): 1008 | db.get('words', "Codern") 1009 | 1010 | def test_add_indented_index(self, tmpdir): 1011 | class IndentedMd5Index(HashIndex): 1012 | 1013 | def __init__(self, *args, **kwargs): 1014 | # kwargs['entry_line_format'] = '<32s32sIIcI' 1015 | kwargs['key_format'] = '16s' 1016 | kwargs['hash_lim'] = 4 * 1024 1017 | super(IndentedMd5Index, self).__init__(*args, **kwargs) 1018 | 1019 | def make_key_value(self, data): 1020 | return md5(data['name'].encode()).digest(), None 1021 | 1022 | def make_key(self, key): 1023 | return md5(key).digest() 1024 | 1025 | db = self._db(os.path.join(str(tmpdir), 'db')) 1026 | db.create() 1027 | db.add_index(IndentedMd5Index(db.path, 'ind')) 1028 | 1029 | db.insert(dict(name='a')) 1030 | assert db.get('ind', 'a', with_doc=True)['doc']['name'] == 'a' 1031 | 1032 | def test_patch_flush_fsync(self, tmpdir): 1033 | from CodernityDB3.patch import patch_flush_fsync 1034 | db = self._db(os.path.join(str(tmpdir), 'db')) 1035 | db.create() 1036 | patch_flush_fsync(db) # patch it 1037 | for x in range(100): 1038 | db.insert(dict(x=x)) 1039 | db.close() 1040 | 1041 | def test_revert_index(self, tmpdir): 1042 | db = self._db(os.path.join(str(tmpdir), 'db')) 1043 | db.create() 1044 | 1045 | ok = """name: test_revert 1046 | type: HashIndex 1047 | key_format: I 1048 | make_key_value: 1049 | x, None 1050 | """ 1051 | 1052 | ok2 = """name: test_revert 1053 | type: HashIndex 1054 | key_format: I 1055 | make_key_value: 1056 | x * 10, None 1057 | """ 1058 | db.add_index(ok) 1059 | 1060 | for x in range(10): 1061 | db.insert(dict(x=x)) 1062 | 1063 | a = sum(map(lambda x: x['key'], db.all('test_revert'))) 1064 | assert a == 45 1065 | 1066 | db.edit_index(ok2, reindex=True) 1067 | 1068 | a = sum(map(lambda x: x['key'], db.all('test_revert'))) 1069 | assert a == 450 1070 | 1071 | db.revert_index('test_revert', reindex=True) 1072 | 1073 | a = sum(map(lambda x: x['key'], db.all('test_revert'))) 1074 | assert a == 45 1075 | 1076 | with pytest.raises(DatabaseException): 1077 | db.revert_index('test_revert', reindex=True) # second restore 1078 | -------------------------------------------------------------------------------- /tests/test_db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from tests.shared import DB_Tests 19 | from CodernityDB3.database import Database 20 | from tests.hash_tests import HashIndexTests 21 | from tests.tree_tests import TreeIndexTests 22 | from tests.shard_tests import ShardTests 23 | 24 | 25 | class Test_Database(DB_Tests): 26 | 27 | _db = Database 28 | 29 | 30 | class Test_HashIndex(HashIndexTests): 31 | 32 | _db = Database 33 | 34 | 35 | class Test_TreeIndex(TreeIndexTests): 36 | 37 | _db = Database 38 | 39 | 40 | class Test_ShardIndex(ShardTests): 41 | 42 | _db = Database 43 | -------------------------------------------------------------------------------- /tests/test_db_super_thread_safe.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from CodernityDB3.database_super_thread_safe import SuperThreadSafeDatabase 20 | 21 | from tests.shared import DB_Tests 22 | from tests.hash_tests import HashIndexTests 23 | from tests.tree_tests import TreeIndexTests 24 | from tests.test_db_thread_safe import Test_Threads 25 | 26 | 27 | class Test_Database(DB_Tests): 28 | 29 | _db = SuperThreadSafeDatabase 30 | 31 | 32 | class Test_HashIndex(HashIndexTests): 33 | 34 | _db = SuperThreadSafeDatabase 35 | 36 | 37 | class Test_TreeIndex(TreeIndexTests): 38 | 39 | _db = SuperThreadSafeDatabase 40 | 41 | 42 | class Test_Threads(Test_Threads): 43 | 44 | _db = SuperThreadSafeDatabase 45 | -------------------------------------------------------------------------------- /tests/test_db_thread_safe.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from CodernityDB3.database_thread_safe import ThreadSafeDatabase 20 | from tests.shared import DB_Tests, WithAIndex 21 | from tests.hash_tests import HashIndexTests 22 | from tests.tree_tests import TreeIndexTests 23 | 24 | from threading import Thread 25 | import os 26 | import time 27 | import random 28 | import pytest 29 | 30 | 31 | class Test_Database(DB_Tests): 32 | 33 | _db = ThreadSafeDatabase 34 | 35 | 36 | class Test_HashIndex(HashIndexTests): 37 | 38 | _db = ThreadSafeDatabase 39 | 40 | 41 | class Test_TreeIndex(TreeIndexTests): 42 | 43 | _db = ThreadSafeDatabase 44 | 45 | 46 | class Test_Threads(object): 47 | 48 | _db = ThreadSafeDatabase 49 | 50 | def test_one(self, tmpdir): 51 | db = self._db(os.path.join(str(tmpdir), 'db')) 52 | db.create() 53 | db.add_index(WithAIndex(db.path, 'with_a')) 54 | ths = [] 55 | for x in range(100): 56 | ths.append(Thread(target=db.insert, args=(dict(a=x),))) 57 | for th in ths: 58 | th.start() 59 | for th in ths: 60 | th.join() 61 | assert db.count(db.all, 'with_a') == 100 62 | l = list(range(100)) 63 | for curr in db.all('with_a', with_doc=True): 64 | # print(curr) 65 | a = curr['doc']['a'] 66 | l.remove(a) 67 | assert l == [] 68 | 69 | @pytest.mark.parametrize(('threads_num', ), [(x, ) for x in (3, 10, 20, 50, 100, 250)]) 70 | def test_conc_update(self, tmpdir, threads_num): 71 | db = self._db(os.path.join(str(tmpdir), 'db')) 72 | db.create() 73 | db.add_index(WithAIndex(db.path, 'with_a')) 74 | db.insert(dict(a=1)) 75 | 76 | def updater(): 77 | i = 0 78 | time.sleep(random.random() / 100) 79 | while True: 80 | rec = list(db.all('id', limit=1)) 81 | doc = rec[0].copy() 82 | doc['a'] += 1 83 | try: 84 | db.update(doc) 85 | # pass 86 | except: 87 | i += 1 88 | if i > 100: 89 | return False 90 | time.sleep(random.random() / 100) 91 | else: 92 | return True 93 | ths = [] 94 | for x in range(threads_num): # python threads... beware!!! 95 | ths.append(Thread(target=updater)) 96 | for th in ths: 97 | th.start() 98 | for th in ths: 99 | th.join() 100 | 101 | assert db.count(db.all, 'with_a', with_doc=True) == 1 102 | assert db.count(db.all, 'id') == 1 103 | -------------------------------------------------------------------------------- /tests/test_indexcreator_db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2011-2013 Codernity (http://codernity.com) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from CodernityDB3.indexcreator import ( 20 | IndexCreatorFunctionException, 21 | IndexCreatorValueException, 22 | Parser) 23 | from CodernityDB3.database import Database, RecordNotFound 24 | from CodernityDB3.hash_index import HashIndex 25 | from CodernityDB3.tree_index import TreeBasedIndex 26 | from CodernityDB3.tree_index import MultiTreeBasedIndex 27 | from CodernityDB3.hash_index import MultiHashIndex 28 | from hashlib import md5 29 | from py.test import raises 30 | import os 31 | import uuid 32 | 33 | # class db_set(): 34 | # def __init__(self,t): 35 | # self.t = t 36 | # def __enter__(self): 37 | # self.db = Database(os.path.join(str(self.t), 'db')) 38 | # self.db.create() 39 | # return self.db 40 | # def __exit__(self,type,value,traceback): 41 | # self.db.destroy() 42 | 43 | 44 | def pytest_funcarg__db(request): 45 | db = Database(os.path.join(str(request.getfuncargvalue('tmpdir')), 'db')) 46 | db.create() 47 | return db 48 | 49 | 50 | def compare_for_multi_index(db, name, s, key_name, data, keys): 51 | db.add_index(s) 52 | 53 | for i in data: 54 | db.insert({key_name: i}) 55 | 56 | # for i in db.all(name): 57 | # print(i) 58 | 59 | for i, k in keys: 60 | if k is None: 61 | with raises(RecordNotFound): 62 | db.get(name, i, with_doc=True) 63 | else: 64 | assert db.get(name, i, with_doc=True)['doc'][key_name] == k 65 | 66 | 67 | class TestIndexCreatorWithDatabase: 68 | def test_output_check(self, db, tmpdir): 69 | s = """ 70 | type = HashIndex 71 | name = s 72 | key_format = 32s 73 | hash_lim = 1 74 | make_key_value: 75 | 0,None 76 | make_key: 77 | 0 78 | """ 79 | db.add_index(s) 80 | assert s == db.get_index_code('s', code_switch='S') 81 | 82 | s1 = """ 83 | type = TreeBasedIndex 84 | name = s1 85 | key_format = 32s 86 | make_key_value: 87 | 0,None 88 | make_key: 89 | 0 90 | """ 91 | db.add_index(s1) 92 | assert s1 == db.get_index_code('s1', code_switch='S') 93 | 94 | 95 | class TestMultiIndexCreatorWithInternalImports: 96 | def test_infix(self, db): 97 | s = """ 98 | name = s 99 | type = MultiTreeBasedIndex 100 | key_format: 3s 101 | make_key_value: 102 | infix(a,2,3,3),null 103 | make_key: 104 | fix_r(key,3) 105 | """ 106 | compare_for_multi_index( 107 | db, 's', s, 'a', ['abcd'], 108 | [("a", None), 109 | ("ab", "abcd"), 110 | ("abc", "abcd"), 111 | ("b", None), 112 | ("abcd", "abcd"), # fix_r will trim it to 3 letters! 113 | ("bcd", "abcd"), 114 | ("abdc", None)]) 115 | 116 | s2 = """ 117 | name = s2 118 | type = MultiTreeBasedIndex 119 | key_format: 5s 120 | make_key_value: 121 | infix(a,0,20,5),None 122 | make_key: 123 | fix_r(key,5) 124 | """ 125 | compare_for_multi_index( 126 | db, 's2', s2, 'a', ["abcd"], 127 | [("a", "abcd"), 128 | ("ab", "abcd"), 129 | ("abc", "abcd"), 130 | ("b", "abcd"), 131 | ("abcd", "abcd"), 132 | ("bcd", "abcd"), 133 | ("abdc", None)]) 134 | 135 | def test_more_than_one_func(self, db): 136 | s = """ 137 | name = s 138 | type = MultiTreeBasedIndex 139 | key_format: 3s 140 | make_key_value: 141 | len(a)>3:infix(a,2,3,3),null 142 | prefix(a,2,3,3),none 143 | make_key: 144 | fix_r(key,3) 145 | """ 146 | compare_for_multi_index( 147 | db, 's', s, 'a', ['abcd'], 148 | [("a", None), 149 | ("ab", "abcd"), 150 | ("abc", "abcd"), 151 | ("b", None), 152 | ("abcd", "abcd"), # fix_r will trim it to 3 letters! 153 | ("bcd", "abcd"), 154 | ("abdc", None)]) 155 | -------------------------------------------------------------------------------- /tox_cov.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist=py34 3 | 4 | [testenv:pypy] 5 | basepython=/usr/local/bin/pypy 6 | 7 | [testenv:pypy-dev] 8 | basepython=/usr/local/bin/pypy-dev 9 | 10 | [testenv] 11 | deps= 12 | pytest 13 | pytest-cov 14 | pytest-xdist 15 | commands= 16 | py.test --junitxml=junit-norm-{envname}.xml --cov-config .coveragerc_norm --cov-report=html --cov-report=xml --cov CodernityDB tests/test_db.py 17 | py.test --junitxml=junit-thread-safe-{envname}.xml --cov-config .coveragerc_thread_safe --cov-report=html --cov-report=xml --cov CodernityDB tests/test_db_thread_safe.py 18 | py.test --junitxml=junit-super-thread-safe-{envname}.xml --cov-config .coveragerc_super_thread_safe --cov-report=html --cov-report=xml --cov CodernityDB tests/test_db_super_thread_safe.py 19 | py.test --junitxml=junit-indexcreator-{envname}.xml --cov-config .coveragerc_indexcreator --cov-report=html --cov-report=xml --cov CodernityDB tests/test_indexcreator_db.py tests/test_indexcreator_exec.py 20 | -------------------------------------------------------------------------------- /tox_no_cov.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist=py34 3 | 4 | [testenv:pypy] 5 | basepython=/usr/local/bin/pypy 6 | 7 | [testenv:pypy-dev] 8 | basepython=/usr/local/bin/pypy-dev 9 | 10 | [testenv] 11 | deps= 12 | pytest 13 | pytest-cov 14 | pytest-xdist 15 | commands= 16 | py.test --junitxml=junit-norm-{envname}.xml tests/test_db.py 17 | py.test --junitxml=junit-thread-safe-{envname}.xml tests/test_db_thread_safe.py 18 | py.test --junitxml=junit-super-thread-safe-{envname}.xml tests/test_db_super_thread_safe.py 19 | py.test --junitxml=junit-indexcreator-{envname}.xml tests/test_indexcreator_db.py tests/test_indexcreator_exec.py 20 | --------------------------------------------------------------------------------