├── .github └── workflows │ └── tests.yaml ├── .gitignore ├── .readthedocs.yaml ├── MANIFEST.in ├── README.md ├── bench ├── bench_binary_vs_http.py ├── bench_reads_kt.py ├── bench_reads_tt.py ├── bench_writes_kt.py ├── bench_writes_tt.py ├── greentest.py └── scripts │ └── ttbench.lua ├── docs ├── Makefile ├── api.rst ├── conf.py ├── index.rst ├── installation.rst ├── logo.png └── usage.rst ├── kt ├── __init__.py ├── _binary.pyx ├── client.py ├── constants.py ├── embedded.py ├── exceptions.py ├── http.py ├── models.py ├── queue.py └── scripts │ ├── kt.lua │ └── tt.lua ├── setup.py ├── tests.py ├── ttcommands.csv ├── ttmisc.csv └── tuning.md /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: [push] 3 | jobs: 4 | tests: 5 | name: ${{ matrix.python-version }} 6 | runs-on: ubuntu-16.04 7 | strategy: 8 | fail-fast: false 9 | matrix: 10 | python-version: [3.8, "3.11"] 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-python@v2 14 | with: 15 | python-version: ${{ matrix.python-version }} 16 | - name: package deps 17 | run: sudo apt-get install kyototycoon libkyototycoon-dev tokyotyrant libtokyotyrant-dev 18 | - name: pip deps 19 | run: | 20 | pip install cython msgpack-python 21 | python setup.py build_ext -i 22 | - name: runtests 23 | run: python tests.py 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | kt/_binary.c 2 | kt/_binary*.so 3 | kt/_protocol.c 4 | kt/_protocol*.so 5 | MANIFEST 6 | build/ 7 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | build: 3 | os: ubuntu-22.04 4 | tools: 5 | python: "3.11" 6 | sphinx: 7 | configuration: docs/conf.py 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE 3 | include kt/_binary.c 4 | include kt/_binary.pyx 5 | include kt/scripts/*.lua 6 | include tests.py 7 | recursive-include docs * 8 | 9 | global-exclude *.pyc 10 | global-exclude *.o 11 | global-exclude *.so 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](http://media.charlesleifer.com/blog/photos/logo.png) 2 | 3 | Fast bindings to kyototycoon and tokyotyrant. 4 | 5 | * Binary APIs implemented as C extension. 6 | * Thread-safe and greenlet-safe. 7 | * Simple APIs. 8 | * Full-featured implementation of protocol. 9 | 10 | View the [documentation](http://kt-lib.readthedocs.io/en/latest/) for more 11 | info. 12 | 13 | #### installing 14 | 15 | ```console 16 | 17 | $ pip install kt 18 | ``` 19 | 20 | #### usage 21 | 22 | ```pycon 23 | 24 | >>> from kt import KyotoTycoon 25 | >>> client = KyotoTycoon() 26 | >>> client.set('k1', 'v1') 27 | 1 28 | >>> client.get('k1') 29 | 'v1' 30 | >>> client.remove('k1') 31 | 1 32 | 33 | >>> client.set_bulk({'k1': 'v1', 'k2': 'v2', 'k3': 'v3'}) 34 | 3 35 | >>> client.get_bulk(['k1', 'xx, 'k3']) 36 | {'k1': 'v1', 'k3': 'v3'} 37 | >>> client.remove_bulk(['k1', 'xx', 'k3']) 38 | 2 39 | ``` 40 | -------------------------------------------------------------------------------- /bench/bench_binary_vs_http.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Benchmark script to measure time taken to read, write and delete using the 5 | binary protocol and HTTP protocol. 6 | """ 7 | import os, sys 8 | sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) 9 | 10 | import contextlib 11 | import time 12 | 13 | from kt import * 14 | 15 | 16 | server = EmbeddedServer(quiet=True) 17 | server.run() 18 | db = server.client 19 | 20 | 21 | def do_set_bulk(nrows, chunksize, klen, vlen): 22 | kfmt = '%%0%sd' % klen 23 | vfmt = '%%0%sd' % vlen 24 | for i in range(0, nrows, chunksize): 25 | accum = {kfmt % j: vfmt % j for j in range(i, i + chunksize)} 26 | db.set_bulk(accum) 27 | 28 | def do_set_bulk_http(nrows, chunksize, klen, vlen): 29 | kfmt = '%%0%sd' % klen 30 | vfmt = '%%0%sd' % vlen 31 | for i in range(0, nrows, chunksize): 32 | accum = {kfmt % j: vfmt % j for j in range(i, i + chunksize)} 33 | db._http.set_bulk(accum) 34 | 35 | def do_get_bulk(nrows, chunksize, klen, vlen): 36 | kfmt = '%%0%sd' % klen 37 | for i in range(0, nrows, chunksize): 38 | accum = [kfmt % j for j in range(i, i + chunksize)] 39 | db.get_bulk(accum) 40 | 41 | def do_get_bulk_http(nrows, chunksize, klen, vlen): 42 | kfmt = '%%0%sd' % klen 43 | for i in range(0, nrows, chunksize): 44 | accum = [kfmt % j for j in range(i, i + chunksize)] 45 | db._http.get_bulk(accum) 46 | 47 | def do_remove_bulk(nrows, chunksize, klen, vlen): 48 | kfmt = '%%0%sd' % klen 49 | for i in range(0, nrows, chunksize): 50 | accum = [kfmt % j for j in range(i, i + chunksize)] 51 | db.remove_bulk(accum) 52 | 53 | def do_remove_bulk_http(nrows, chunksize, klen, vlen): 54 | kfmt = '%%0%sd' % klen 55 | for i in range(0, nrows, chunksize): 56 | accum = [kfmt % j for j in range(i, i + chunksize)] 57 | db._http.remove_bulk(accum) 58 | 59 | @contextlib.contextmanager 60 | def timed(msg, *params): 61 | pstr = ', '.join(map(str, params)) 62 | s = time.time() 63 | yield 64 | print('%0.3fs - %s(%s)' % (time.time() - s, msg, pstr)) 65 | 66 | 67 | SETTINGS = ( 68 | # (nrows, chunksiz, ksiz, vsiz). 69 | (200000, 10000, 48, 512), # ~100MB of data, 20 batches. 70 | (25000, 1250, 256, 1024 * 4), # ~100MB of data, 20 batches. 71 | (1700, 100, 256, 1024 * 64), # ~100MB of data, 17 batches. 72 | ) 73 | for nrows, chunksiz, ksiz, vsiz in SETTINGS: 74 | with timed('set_bulk', nrows, chunksiz, ksiz, vsiz): 75 | do_set_bulk(nrows, chunksiz, ksiz, vsiz) 76 | with timed('get_bulk', nrows, chunksiz, ksiz, vsiz): 77 | do_get_bulk(nrows, chunksiz, ksiz, vsiz) 78 | with timed('remove_bulk', nrows, chunksiz, ksiz, vsiz): 79 | do_remove_bulk(nrows, chunksiz, ksiz, vsiz) 80 | 81 | db.clear() 82 | with timed('set_bulk_http', nrows, chunksiz, ksiz, vsiz): 83 | do_set_bulk_http(nrows, chunksiz, ksiz, vsiz) 84 | with timed('get_bulk_http', nrows, chunksiz, ksiz, vsiz): 85 | do_get_bulk_http(nrows, chunksiz, ksiz, vsiz) 86 | with timed('remove_bulk_http', nrows, chunksiz, ksiz, vsiz): 87 | do_remove_bulk_http(nrows, chunksiz, ksiz, vsiz) 88 | db.clear() 89 | print('\n') 90 | 91 | try: 92 | server.stop() 93 | except OSError: 94 | pass 95 | 96 | -------------------------------------------------------------------------------- /bench/bench_reads_kt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Benchmark script to measure time taken to read values using a variety of 5 | different methods. 6 | """ 7 | import os, sys 8 | sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) 9 | 10 | import contextlib 11 | import time 12 | 13 | from kt import * 14 | 15 | 16 | # In-memory btree. 17 | server = EmbeddedServer(database='%', quiet=True) 18 | server.run() 19 | db = server.client 20 | 21 | 22 | def do_get(nrows, kprefix, klen): 23 | kfmt = '%s%%0%sd' % (kprefix, klen) 24 | for i in range(nrows): 25 | db.get(kfmt % i) 26 | 27 | def do_get_bulk(nrows, chunksize, kprefix, klen): 28 | kfmt = '%s%%0%sd' % (kprefix, klen) 29 | for i in range(0, nrows, chunksize): 30 | keys = [kfmt % j for j in range(i, i + chunksize)] 31 | db.get_bulk(keys) 32 | 33 | def do_match_prefix(nrows, chunksize, kprefix, klen): 34 | kfmt = '%s%%0%sd' % (kprefix, klen) 35 | for i in range(0, nrows, chunksize): 36 | prefix = (kfmt % i)[:-(len(str(chunksize)) - 1)] 37 | db.match_prefix(prefix, chunksize) 38 | 39 | def do_match_regex(nrows, chunksize, kprefix, klen): 40 | kfmt = '%s%%0%sd' % (kprefix, klen) 41 | for i in range(0, nrows, chunksize): 42 | regex = (kfmt % i)[:-(len(str(chunksize)) - 1)] 43 | db.match_regex(regex + '*', chunksize) 44 | 45 | def do_keys_nonlazy(): 46 | for _ in db.keys_nonlazy(): 47 | pass 48 | 49 | def do_keys(): 50 | for _ in db.keys(): 51 | pass 52 | 53 | def do_items(): 54 | for _ in db.items(): 55 | pass 56 | 57 | 58 | @contextlib.contextmanager 59 | def timed(msg, *params): 60 | pstr = ', '.join(map(str, params)) 61 | s = time.time() 62 | yield 63 | print('%0.3fs - %s(%s)' % (time.time() - s, msg, pstr)) 64 | 65 | 66 | SETTINGS = ( 67 | # (nrows, chunksiz, kprefix, ksiz, vsiz). 68 | (100000, 10000, 'a', 48, 512), 69 | (25000, 1250, 'b', 256, 1024 * 4), 70 | (1700, 100, 'c', 256, 1024 * 64), 71 | ) 72 | 73 | # Setup database. 74 | 75 | for nrows, chunksiz, kprefix, ksiz, vsiz in SETTINGS: 76 | for i in range(0, nrows, chunksiz): 77 | kfmt = '%s%%0%sd' % (kprefix, ksiz) 78 | vfmt = '%%0%sd' % (vsiz) 79 | accum = {kfmt % j: vfmt % j for j in range(i, i + chunksiz)} 80 | db.set_bulk(accum) 81 | 82 | mbsize = db.size / (1024. * 1024.) 83 | print('database initialized, size: %.fMB, %s records' % (mbsize, len(db))) 84 | 85 | with timed('get', nrows, kprefix, ksiz): 86 | do_get(nrows, kprefix, ksiz) 87 | 88 | with timed('get_bulk', nrows, chunksiz, kprefix, ksiz): 89 | do_get_bulk(nrows, chunksiz, kprefix, ksiz) 90 | 91 | with timed('match_prefix', nrows, chunksiz, kprefix, ksiz): 92 | do_match_prefix(nrows, chunksiz, kprefix, ksiz) 93 | 94 | with timed('match_regex', nrows, chunksiz, kprefix, ksiz): 95 | do_match_regex(nrows, chunksiz, kprefix, ksiz) 96 | 97 | with timed('keys (nonlazy)'): 98 | do_keys_nonlazy() 99 | 100 | #with timed('keys'): 101 | # do_keys() 102 | 103 | #with timed('items'): 104 | # do_items() 105 | 106 | print('\n') 107 | db.clear() 108 | 109 | try: 110 | server.stop() 111 | except OSError: 112 | pass 113 | -------------------------------------------------------------------------------- /bench/bench_reads_tt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Benchmark script to measure time taken to read values using a variety of 5 | different methods. 6 | """ 7 | import os, sys 8 | sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) 9 | 10 | import contextlib 11 | import time 12 | 13 | from kt import * 14 | 15 | 16 | # In-memory btree. 17 | server = EmbeddedTokyoTyrantServer(database='+', quiet=True) 18 | server.run() 19 | db = server.client 20 | 21 | 22 | def do_get(nrows, kprefix, klen): 23 | kfmt = '%s%%0%sd' % (kprefix, klen) 24 | for i in range(nrows): 25 | db.get(kfmt % i) 26 | 27 | def do_get_bulk(nrows, chunksize, kprefix, klen): 28 | kfmt = '%s%%0%sd' % (kprefix, klen) 29 | for i in range(0, nrows, chunksize): 30 | keys = [kfmt % j for j in range(i, i + chunksize)] 31 | db.get_bulk(keys) 32 | 33 | def do_get_range(nrows, chunksize, kprefix, klen): 34 | kfmt = '%s%%0%sd' % (kprefix, klen) 35 | for i in range(0, nrows, chunksize): 36 | start = kfmt % i 37 | stop = kfmt % (i + chunksize) 38 | db.get_range(start, stop, max_keys=chunksize) 39 | 40 | def do_match_prefix(nrows, chunksize, kprefix, klen): 41 | kfmt = '%s%%0%sd' % (kprefix, klen) 42 | for i in range(0, nrows, chunksize): 43 | prefix = (kfmt % i)[:-(len(str(chunksize)) - 1)] 44 | db.match_prefix(prefix, chunksize) 45 | 46 | def do_match_regex(nrows, chunksize, kprefix, klen): 47 | kfmt = '%s%%0%sd' % (kprefix, klen) 48 | for i in range(0, nrows, chunksize): 49 | regex = (kfmt % i)[:-(len(str(chunksize)) - 1)] 50 | db.match_regex(regex + '*', chunksize) 51 | 52 | def do_iter_from(kprefix, klen): 53 | kfmt = '%s%%0%sd' % (kprefix, klen) 54 | for _ in db.iter_from(kfmt % 0): 55 | pass 56 | 57 | def do_keys(): 58 | for _ in db.keys(): 59 | pass 60 | 61 | def do_items(): 62 | for _ in db.items(): 63 | pass 64 | 65 | 66 | @contextlib.contextmanager 67 | def timed(msg, *params): 68 | pstr = ', '.join(map(str, params)) 69 | s = time.time() 70 | yield 71 | print('%0.3fs - %s(%s)' % (time.time() - s, msg, pstr)) 72 | 73 | 74 | SETTINGS = ( 75 | # (nrows, chunksiz, kprefix, ksiz, vsiz). 76 | (100000, 10000, 'a', 48, 512), 77 | (25000, 1250, 'b', 256, 1024 * 4), 78 | (1700, 100, 'c', 256, 1024 * 64), 79 | ) 80 | 81 | # Setup database. 82 | 83 | for nrows, chunksiz, kprefix, ksiz, vsiz in SETTINGS: 84 | for i in range(0, nrows, chunksiz): 85 | kfmt = '%s%%0%sd' % (kprefix, ksiz) 86 | vfmt = '%%0%sd' % (vsiz) 87 | accum = {kfmt % j: vfmt % j for j in range(i, i + chunksiz)} 88 | db.set_bulk(accum) 89 | 90 | mbsize = db.size / (1024. * 1024.) 91 | print('database initialized, size: %.fMB, %s records' % (mbsize, len(db))) 92 | 93 | with timed('get', nrows, kprefix, ksiz): 94 | do_get(nrows, kprefix, ksiz) 95 | 96 | with timed('get_bulk', nrows, chunksiz, kprefix, ksiz): 97 | do_get_bulk(nrows, chunksiz, kprefix, ksiz) 98 | 99 | with timed('get_range', nrows, chunksiz, kprefix, ksiz): 100 | do_get_range(nrows, chunksiz, kprefix, ksiz) 101 | 102 | with timed('match_prefix', nrows, chunksiz, kprefix, ksiz): 103 | do_match_prefix(nrows, chunksiz, kprefix, ksiz) 104 | 105 | with timed('match_regex', nrows, chunksiz, kprefix, ksiz): 106 | do_match_regex(nrows, chunksiz, kprefix, ksiz) 107 | 108 | with timed('iter_from', kprefix, ksiz): 109 | do_iter_from(kprefix, ksiz) 110 | 111 | with timed('keys'): 112 | do_keys() 113 | 114 | with timed('items'): 115 | do_items() 116 | 117 | print('\n') 118 | db.clear() 119 | 120 | try: 121 | server.stop() 122 | except OSError: 123 | pass 124 | -------------------------------------------------------------------------------- /bench/bench_writes_kt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Benchmark script to measure time taken to set values using a variety of 5 | different methods (set, set_bulk, set via http, set_bulk via http). 6 | """ 7 | import os, sys 8 | sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) 9 | 10 | import contextlib 11 | import time 12 | 13 | from kt import * 14 | 15 | 16 | server = EmbeddedServer(quiet=True) 17 | server.run() 18 | db = server.client 19 | 20 | 21 | def do_set(nrows, klen, vlen): 22 | kfmt = '%%0%sd' % klen 23 | vfmt = '%%0%sd' % vlen 24 | for i in range(nrows): 25 | db.set(kfmt % i, vfmt % i) 26 | 27 | def do_set_bulk(nrows, chunksize, klen, vlen): 28 | kfmt = '%%0%sd' % klen 29 | vfmt = '%%0%sd' % vlen 30 | for i in range(0, nrows, chunksize): 31 | accum = {kfmt % j: vfmt % j for j in range(i, i + chunksize)} 32 | db.set_bulk(accum) 33 | 34 | def do_set_http(nrows, klen, vlen): 35 | kfmt = '%%0%sd' % klen 36 | vfmt = '%%0%sd' % vlen 37 | for i in range(nrows): 38 | db._http.set(kfmt % i, vfmt % i) 39 | 40 | def do_set_bulk_http(nrows, chunksize, klen, vlen): 41 | kfmt = '%%0%sd' % klen 42 | vfmt = '%%0%sd' % vlen 43 | for i in range(0, nrows, chunksize): 44 | accum = {kfmt % j: vfmt % j for j in range(i, i + chunksize)} 45 | db._http.set_bulk(accum) 46 | 47 | @contextlib.contextmanager 48 | def timed(msg, *params): 49 | pstr = ', '.join(map(str, params)) 50 | s = time.time() 51 | yield 52 | print('%0.3fs - %s(%s)' % (time.time() - s, msg, pstr)) 53 | 54 | 55 | SETTINGS = ( 56 | # (nrows, chunksiz, ksiz, vsiz). 57 | (200000, 10000, 48, 512), # ~100MB of data, 20 batches. 58 | (25000, 1250, 256, 1024 * 4), # ~100MB of data, 20 batches. 59 | (1700, 100, 256, 1024 * 64), # ~100MB of data, 17 batches. 60 | ) 61 | for nrows, chunksiz, ksiz, vsiz in SETTINGS: 62 | with timed('set', nrows, ksiz, vsiz): 63 | do_set(nrows, ksiz, vsiz) 64 | db.clear() 65 | 66 | # Lots of small requests is incredibly slow, so avoid pointless benchmark. 67 | if nrows < 25000: 68 | with timed('set_http', nrows, ksiz, vsiz): 69 | do_set_http(nrows, ksiz, vsiz) 70 | db.clear() 71 | 72 | with timed('set_bulk', nrows, chunksiz, ksiz, vsiz): 73 | do_set_bulk(nrows, chunksiz, ksiz, vsiz) 74 | db.clear() 75 | 76 | with timed('set_bulk_http', nrows, chunksiz, ksiz, vsiz): 77 | do_set_bulk_http(nrows, chunksiz, ksiz, vsiz) 78 | db.clear() 79 | print('\n') 80 | 81 | try: 82 | server.stop() 83 | except OSError: 84 | pass 85 | -------------------------------------------------------------------------------- /bench/bench_writes_tt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Benchmark script to measure time taken to set values using a variety of 5 | different methods (set, set_bulk, setnr, setnr_bulk). 6 | """ 7 | import os, sys 8 | sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) 9 | 10 | import contextlib 11 | import time 12 | 13 | from kt import * 14 | 15 | 16 | server = EmbeddedTokyoTyrantServer(quiet=True) 17 | server.run() 18 | db = server.client 19 | 20 | 21 | def do_set(nrows, klen, vlen): 22 | kfmt = '%%0%sd' % klen 23 | vfmt = '%%0%sd' % vlen 24 | for i in range(nrows): 25 | db.set(kfmt % i, vfmt % i) 26 | 27 | def do_set_bulk(nrows, chunksize, klen, vlen): 28 | kfmt = '%%0%sd' % klen 29 | vfmt = '%%0%sd' % vlen 30 | for i in range(0, nrows, chunksize): 31 | accum = {kfmt % j: vfmt % j for j in range(i, i + chunksize)} 32 | db.set_bulk(accum) 33 | 34 | def do_setnr(nrows, klen, vlen): 35 | kfmt = '%%0%sd' % klen 36 | vfmt = '%%0%sd' % vlen 37 | for i in range(nrows): 38 | db.set(kfmt % i, vfmt % i, no_reply=True) 39 | 40 | def do_setnr_bulk(nrows, chunksize, klen, vlen): 41 | kfmt = '%%0%sd' % klen 42 | vfmt = '%%0%sd' % vlen 43 | for i in range(0, nrows, chunksize): 44 | accum = {kfmt % j: vfmt % j for j in range(i, i + chunksize)} 45 | db.set_bulk(accum, no_reply=True) 46 | 47 | @contextlib.contextmanager 48 | def timed(msg, *params): 49 | pstr = ', '.join(map(str, params)) 50 | s = time.time() 51 | yield 52 | print('%0.3fs - %s(%s)' % (time.time() - s, msg, pstr)) 53 | 54 | 55 | SETTINGS = ( 56 | # (nrows, chunksiz, ksiz, vsiz). 57 | (200000, 10000, 48, 512), # ~100MB of data, 20 batches. 58 | (25000, 1250, 256, 1024 * 4), # ~100MB of data, 20 batches. 59 | (1700, 100, 256, 1024 * 64), # ~100MB of data, 17 batches. 60 | ) 61 | for nrows, chunksiz, ksiz, vsiz in SETTINGS: 62 | with timed('set', nrows, ksiz, vsiz): 63 | do_set(nrows, ksiz, vsiz) 64 | db.clear() 65 | 66 | with timed('setnr', nrows, ksiz, vsiz): 67 | do_setnr(nrows, ksiz, vsiz) 68 | db.clear() 69 | 70 | with timed('set_bulk', nrows, chunksiz, ksiz, vsiz): 71 | do_set_bulk(nrows, chunksiz, ksiz, vsiz) 72 | db.clear() 73 | 74 | with timed('setnr_bulk', nrows, chunksiz, ksiz, vsiz): 75 | do_setnr_bulk(nrows, chunksiz, ksiz, vsiz) 76 | db.clear() 77 | print('\n') 78 | 79 | try: 80 | server.stop() 81 | except OSError: 82 | pass 83 | -------------------------------------------------------------------------------- /bench/greentest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Benchmark script to ensure that *kt* plays nice with gevent. We spawn a number 5 | of green threads, each of which calls a Lua script that sleeps -- effectively 6 | blocking the socket. If gevent is working, then we should see the threads all 7 | finishing at about the same time. 8 | """ 9 | 10 | from gevent import monkey; monkey.patch_all() 11 | import gevent 12 | import os 13 | import sys 14 | import time 15 | 16 | from kt import * 17 | 18 | 19 | nsec = 1 20 | nthreads = 16 21 | print('\x1b[1;33m%s green threads, sleeping for %s seconds' % (nthreads, nsec)) 22 | print('\x1b[0m') 23 | 24 | curdir = os.path.dirname(__file__) 25 | script = os.path.join(curdir, 'scripts/ttbench.lua') 26 | 27 | 28 | # TokyoTyrant runs lua scripts in a dedicated thread, so we have only as much 29 | # concurrency as worker threads. 30 | server = EmbeddedTokyoTyrantServer(server_args=['-ext', script, 31 | '-thnum', str(nthreads)], 32 | connection_pool=True) 33 | server.run() 34 | 35 | tt = server._create_client() 36 | 37 | def call_slow_script(nsec): 38 | tt.script('sleep', key=str(nsec)) 39 | tt.status() 40 | 41 | threads = [] 42 | start = time.time() 43 | for i in range(nthreads): 44 | threads.append(gevent.spawn(call_slow_script, nsec)) 45 | 46 | for t in threads: 47 | t.join() 48 | 49 | tt._protocol.close_all() 50 | 51 | total = time.time() - start 52 | if total >= (nsec * nthreads): 53 | print('\x1b[1;31mFAIL! ') 54 | else: 55 | print('\x1b[1;32mOK! ') 56 | print('TOTAL TIME: %0.3fs\x1b[0m\n' % total) 57 | 58 | # Now run a whole shitload of connections. 59 | nconns = nthreads * 16 60 | 61 | def check_status_sleep(nsec): 62 | tt.status() 63 | tt.close() 64 | time.sleep(nsec) 65 | tt.status() 66 | tt.close() 67 | 68 | print('\x1b[1;33m%s green threads checking status' % (nconns)) 69 | print('\x1b[0m') 70 | 71 | threads = [] 72 | start = time.time() 73 | for i in range(nconns): 74 | threads.append(gevent.spawn(check_status_sleep, nsec)) 75 | 76 | for t in threads: 77 | t.join() 78 | 79 | total = time.time() - start 80 | if total >= (nsec * nthreads): 81 | print('\x1b[1;31mFAIL! ') 82 | else: 83 | print('\x1b[1;32mOK! ') 84 | print('TOTAL TIME: %0.3fs\x1b[0m' % total) 85 | 86 | server.stop() 87 | -------------------------------------------------------------------------------- /bench/scripts/ttbench.lua: -------------------------------------------------------------------------------- 1 | function sleep(key, value) 2 | _sleep(tonumber(key)) 3 | return "true" 4 | end 5 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = kt 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | API 4 | === 5 | 6 | Serializers 7 | ----------- 8 | 9 | .. py:data:: KT_BINARY 10 | 11 | Default value serialization. Serializes values as UTF-8 byte-strings and 12 | deserializes to unicode. 13 | 14 | .. py:data:: KT_JSON 15 | 16 | Serialize values as JSON (encoded as UTF-8). 17 | 18 | .. py:data:: KT_MSGPACK 19 | 20 | Uses ``msgpack`` to serialize and deserialize values. 21 | 22 | .. py:data:: KT_NONE 23 | 24 | No serialization or deserialization. Values must be byte-strings. 25 | 26 | .. py:data:: KT_PICKLE 27 | 28 | Serialize and deserialize using Python's pickle module. 29 | 30 | .. py:data:: TT_TABLE 31 | 32 | Special serializer for use with TokyoTyrant's remote table database. Values 33 | are represented as dictionaries. 34 | 35 | 36 | Kyoto Tycoon client 37 | ------------------- 38 | 39 | .. py:class:: KyotoTycoon(host='127.0.0.1', port=1978, serializer=KT_BINARY, decode_keys=True, timeout=None, connection_pool=False, default_db=0) 40 | 41 | :param str host: server host. 42 | :param int port: server port. 43 | :param serializer: serialization method to use for storing/retrieving values. 44 | Accepts ``KT_BINARY``, ``KT_JSON``, ``KT_MSGPACK``, ``KT_NONE`` or ``KT_PICKLE``. 45 | :param bool decode_keys: allow unicode keys, encoded as UTF-8. 46 | :param int timeout: socket timeout (optional). 47 | :param bool connection_pool: use a connection pool to manage sockets. 48 | :param int default_db: default database to operate on. 49 | 50 | Client for interacting with Kyoto Tycoon database. 51 | 52 | .. py:method:: close(allow_reuse=True) 53 | 54 | :param bool allow_reuse: when the connection pool is enabled, this flag 55 | indicates whether the connection can be reused. For unpooled 56 | clients this flag has no effect. 57 | 58 | Close the connection to the server. 59 | 60 | .. py:method:: close_all() 61 | 62 | When using the connection pool, this method can close *all* client 63 | connections. 64 | 65 | .. py:method:: get_bulk(keys, db=None, decode_values=True) 66 | 67 | :param list keys: keys to retrieve 68 | :param int db: database index 69 | :param bool decode_values: decode values using the configured 70 | serialization scheme. 71 | :return: result dictionary 72 | 73 | Efficiently retrieve multiple key/value pairs from the database. If a 74 | key does not exist, it will not be present in the result dictionary. 75 | 76 | .. py:method:: get_bulk_details(keys, db=None, decode_values=True) 77 | 78 | :param list keys: keys to retrieve 79 | :param int db: database index 80 | :param bool decode_values: decode values using the configured 81 | serialization scheme. 82 | :return: List of tuples: ``(db index, key, value, expire time)`` 83 | 84 | Like :py:meth:`~KyotoTycoon.get_bulk`, but the return value is a list 85 | of tuples with additional information for each key. 86 | 87 | .. py:method:: get_bulk_raw(db_key_list, decode_values=True) 88 | 89 | :param db_key_list: a list of 2-tuples to retrieve: ``(db index, key)`` 90 | :param bool decode_values: decode values using the configured 91 | serialization scheme. 92 | :return: result dictionary 93 | 94 | Like :py:meth:`~KyotoTycoon.get_bulk`, except it supports fetching 95 | key/value pairs from multiple databases. The input is a list of 96 | 2-tuples consisting of ``(db, key)`` and the return value is a 97 | dictionary of ``key: value`` pairs. 98 | 99 | .. py:method:: get_bulk_raw_details(db_key_list, decode_values=True) 100 | 101 | :param db_key_list: a list of 2-tuples to retrieve: ``(db index, key)`` 102 | :param bool decode_values: decode values using the configured 103 | serialization scheme. 104 | :return: List of tuples: ``(db index, key, value, expire time)`` 105 | 106 | Like :py:meth:`~KyotoTycoon.get_bulk_raw`, but the return value is a 107 | list of tuples with additional information for each key. 108 | 109 | .. py:method:: get(key, db=None) 110 | 111 | :param str key: key to look-up 112 | :param int db: database index 113 | :return: deserialized value or ``None`` if key does not exist. 114 | 115 | Fetch and (optionally) deserialize the value for the given key. 116 | 117 | .. py:method:: get_bytes(key, db=None) 118 | 119 | :param str key: key to look-up 120 | :param int db: database index 121 | :return: raw bytestring value or ``None`` if key does not exist. 122 | 123 | Fetch the value for the given key. The resulting value will not 124 | be deserialized. 125 | 126 | .. py:method:: set_bulk(data, db=None, expire_time=None, no_reply=False, encode_values=True) 127 | 128 | :param dict data: mapping of key/value pairs to set. 129 | :param int db: database index 130 | :param int expire_time: expiration time in seconds 131 | :param bool no_reply: execute the operation without a server 132 | acknowledgment. 133 | :param bool encode_values: serialize the values using the configured 134 | serialization scheme (e.g., ``KT_MSGPACK``). 135 | :return: number of keys that were set, or ``None`` if ``no_reply``. 136 | 137 | Efficiently set multiple key/value pairs. If given, the provided ``db`` 138 | and ``expire_time`` values will be used for all key/value pairs being 139 | set. 140 | 141 | .. py:method:: set_bulk_raw(data, no_reply=False, encode_values=True) 142 | 143 | :param list data: a list of 4-tuples: ``(db, key, value, expire time)`` 144 | :param bool no_reply: execute the operation without a server 145 | acknowledgment. 146 | :param bool encode_values: serialize the values using the configured 147 | serialization scheme (e.g., ``KT_MSGPACK``). 148 | :return: number of keys that were set, or ``None`` if ``no_reply``. 149 | 150 | Efficiently set multiple key/value pairs. Unlike 151 | :py:meth:`~KyotoTycoon.set_bulk`, this method can be used to set 152 | key/value pairs in multiple databases in a single call, and each key 153 | can specify its own expire time. 154 | 155 | .. py:method:: set(key, value, db=None, expire_time=None, no_reply=False) 156 | 157 | :param str key: key to set 158 | :param value: value to store (will be serialized using serializer) 159 | :param int db: database index 160 | :param int expire_time: expiration time in seconds 161 | :param bool no_reply: execute the operation without a server 162 | acknowledgment. 163 | :return: number of rows set (1) 164 | 165 | Set a single key/value pair. 166 | 167 | .. py:method:: set_bytes(key, value, db=None, expire_time=None, no_reply=False) 168 | 169 | :param str key: key to set 170 | :param value: raw value to store 171 | :param int db: database index 172 | :param int expire_time: expiration time in seconds 173 | :param bool no_reply: execute the operation without a server 174 | acknowledgment. 175 | :return: number of rows set (1) 176 | 177 | Set a single key/value pair without encoding the value. 178 | 179 | .. py:method:: remove_bulk(keys, db=None, no_reply=False) 180 | 181 | :param list keys: list of keys to remove 182 | :param int db: database index 183 | :param bool no_reply: execute the operation without a server 184 | acknowledgment. 185 | :return: number of keys that were removed 186 | 187 | .. py:method:: remove_bulk_raw(db_key_list, no_reply=False) 188 | 189 | :param db_key_list: a list of 2-tuples to retrieve: ``(db index, key)`` 190 | :param bool no_reply: execute the operation without a server 191 | acknowledgment. 192 | :return: number of keys that were removed 193 | 194 | Like :py:meth:`~KyotoTycoon.remove_bulk`, but allows keys to be removed 195 | from multiple databases in a single call. 196 | 197 | .. py:method:: remove(key, db=None, no_reply=False) 198 | 199 | :param str key: key to remove 200 | :param int db: database index 201 | :param bool no_reply: execute the operation without a server 202 | acknowledgment. 203 | :return: number of rows removed 204 | 205 | .. py:method:: script(name, data=None, no_reply=False, encode_values=True, decode_values=True) 206 | 207 | :param str name: name of lua function to call 208 | :param dict data: mapping of key/value pairs to pass to lua function. 209 | :param bool no_reply: execute the operation without a server 210 | acknowledgment. 211 | :param bool encode_values: serialize values passed to lua function. 212 | :param bool decode_values: deserialize values returned by lua function. 213 | :return: dictionary of key/value pairs returned by function 214 | 215 | Execute a lua function. Kyoto Tycoon lua extensions accept arbitrary 216 | key/value pairs as input, and return a result dictionary. If 217 | ``encode_values`` is ``True``, the input values will be serialized. 218 | Likewise, if ``decode_values`` is ``True`` the values returned by the 219 | Lua function will be deserialized using the configured serializer. 220 | 221 | .. py:method:: clear(db=None) 222 | 223 | :param int db: database index 224 | :return: boolean indicating success 225 | 226 | Remove all keys from the database. 227 | 228 | .. py:method:: status(db=None) 229 | 230 | :param int db: database index 231 | :return: status fields and values 232 | :rtype: dict 233 | 234 | Obtain status information from the server about the selected database. 235 | 236 | .. py:method:: report() 237 | 238 | :return: status fields and values 239 | :rtype: dict 240 | 241 | Obtain report on overall status of server, including all databases. 242 | 243 | .. py:method:: ulog_list() 244 | 245 | :return: a list of 3-tuples describing the files in the update log. 246 | 247 | Returns a list of metadata about the state of the update log. For each 248 | file in the update log, a 3-tuple is returned. For example: 249 | 250 | .. code-block:: pycon 251 | 252 | >>> kt.ulog_list() 253 | [('/var/lib/database/ulog/kt/0000000037.ulog', 254 | '67150706', 255 | datetime.datetime(2019, 1, 4, 1, 28, 42, 43000)), 256 | ('/var/lib/database/ulog/kt/0000000038.ulog', 257 | '14577366', 258 | datetime.datetime(2019, 1, 4, 1, 41, 7, 245000))] 259 | 260 | .. py:method:: ulog_remove(max_dt) 261 | 262 | :param datetime max_dt: maximum datetime to preserve 263 | :return: boolean indicating success 264 | 265 | Removes all update-log files older than the given datetime. 266 | 267 | .. py:method:: synchronize(hard=False, command=None, db=None) 268 | 269 | :param bool hard: perform a "hard" synchronization 270 | :param str command: command to run after synchronization 271 | :param int db: database index 272 | :return: boolean indicating success 273 | 274 | Synchronize the database, optionally executing the given command upon 275 | success. This can be used to create hot backups, for example. 276 | 277 | .. py:method:: vacuum(step=0, db=None) 278 | 279 | :param int step: number of steps, default is 0 280 | :param int db: database index 281 | :return: boolean indicating success 282 | 283 | .. py:method:: add(key, value, db=None, expire_time=None, encode_value=True) 284 | 285 | :param str key: key to add 286 | :param value: value to store 287 | :param int db: database index 288 | :param int expire_time: expiration time in seconds 289 | :param bool encode_value: serialize the value using the configured 290 | serialization method. 291 | :return: boolean indicating if key could be added or not 292 | :rtype: bool 293 | 294 | Add a key/value pair to the database. This operation will only succeed 295 | if the key does not already exist in the database. 296 | 297 | .. py:method:: replace(key, value, db=None, expire_time=None, encode_value=True) 298 | 299 | :param str key: key to replace 300 | :param value: value to store 301 | :param int db: database index 302 | :param int expire_time: expiration time in seconds 303 | :param bool encode_value: serialize the value using the configured 304 | serialization method. 305 | :return: boolean indicating if key could be replaced or not 306 | :rtype: bool 307 | 308 | Replace a key/value pair to the database. This operation will only 309 | succeed if the key alreadys exist in the database. 310 | 311 | .. py:method:: append(key, value, db=None, expire_time=None, encode_value=True) 312 | 313 | :param str key: key to append value to 314 | :param value: data to append 315 | :param int db: database index 316 | :param int expire_time: expiration time in seconds 317 | :param bool encode_value: serialize the value using the configured 318 | serialization method. 319 | :return: boolean indicating if value was appended 320 | :rtype: bool 321 | 322 | Appends data to an existing key/value pair. If the key does not exist, 323 | this is equivalent to :py:meth:`~KyotoTycoon.set`. 324 | 325 | .. py:method:: exists(key, db=None) 326 | 327 | :param str key: key to test 328 | :param int db: database index 329 | :return: boolean indicating if key exists 330 | 331 | Return whether or not the given key exists in the database. 332 | 333 | .. py:method:: length(key, db=None) 334 | 335 | :param str key: key 336 | :param int db: database index 337 | :return: length of the value in bytes, or ``None`` if not found 338 | 339 | Return the length of the raw value stored at the given key. If the key 340 | does not exist, returns ``None``. 341 | 342 | .. py:method:: seize(key, db=None, decode_value=True) 343 | 344 | :param str key: key to remove 345 | :param int db: database index 346 | :param bool decode_value: deserialize the value using the configured 347 | serialization method. 348 | :return: value stored at given key or ``None`` if key does not exist. 349 | 350 | Get and remove the data stored in a given key in a single operation. 351 | 352 | .. py:method:: cas(key, old_val, new_val, db=None, expire_time=None, encode_value=True) 353 | 354 | :param str key: key to append value to 355 | :param old_val: original value to test 356 | :param new_val: new value to store 357 | :param int db: database index 358 | :param int expire_time: expiration time in seconds 359 | :param bool encode_value: serialize the old and new values using the 360 | configured serialization method. 361 | :return: boolean indicating if compare-and-swap succeeded. 362 | :rtype: bool 363 | 364 | Compare-and-swap the value stored at a given key. 365 | 366 | .. py:method:: incr(key, n=1, orig=None, db=None, expire_time=None) 367 | 368 | :param str key: key to increment 369 | :param int n: value to add 370 | :param int orig: default value if key does not exist 371 | :param int db: database index 372 | :param int expire_time: expiration time in seconds 373 | :return: new value at key 374 | :rtype: int 375 | 376 | Increment the value stored in the given key. 377 | 378 | .. py:method:: incr_double(key, n=1., orig=None, db=None, expire_time=None) 379 | 380 | :param str key: key to increment 381 | :param float n: value to add 382 | :param float orig: default value if key does not exist 383 | :param int db: database index 384 | :param int expire_time: expiration time in seconds 385 | :return: new value at key 386 | :rtype: float 387 | 388 | Increment the floating-point value stored in the given key. 389 | 390 | .. py:method:: __getitem__(key_or_keydb) 391 | 392 | Item-lookup based on either ``key`` or a 2-tuple consisting of 393 | ``(key, db)``. Follows same semantics as :py:meth:`~KyotoTycoon.get`. 394 | 395 | .. py:method:: __setitem__(key_or_keydb, value_or_valueexpire) 396 | 397 | Item-setting based on either ``key`` or a 2-tuple consisting of 398 | ``(key, db)``. Value consists of either a ``value`` or a 2-tuple 399 | consisting of ``(value, expire_time)``. Follows same semantics 400 | as :py:meth:`~KyotoTycoon.set`. 401 | 402 | .. py:method:: __delitem__(key_or_keydb) 403 | 404 | Item-deletion based on either ``key`` or a 2-tuple consisting of 405 | ``(key, db)``. Follows same semantics as :py:meth:`~KyotoTycoon.remove`. 406 | 407 | .. py:method:: __contains__(key_or_keydb) 408 | 409 | Check if key exists. Accepts either ``key`` or a 2-tuple consisting of 410 | ``(key, db)``. Follows same semantics as :py:meth:`~KyotoTycoon.exists`. 411 | 412 | .. py:method:: __len__() 413 | 414 | :return: total number of keys in the default database. 415 | :rtype: int 416 | 417 | .. py:method:: count(db=None) 418 | 419 | :param db: database index 420 | :type db: int or None 421 | :return: total number of keys in the database. 422 | :rtype: int 423 | 424 | Count total number of keys in the database. 425 | 426 | .. py:method:: update(__data=None, db=None, expire_time=None, no_reply=False, encode_values=True, **kwargs) 427 | 428 | Efficiently set multiple key/value pairs. If given, the provided ``db`` 429 | and ``expire_time`` values will be used for all key/value pairs being 430 | set. 431 | 432 | See :py:meth:`KyotoTycoon.set_bulk` for details. 433 | 434 | .. py:method:: pop(key, db=None, decode_value=True) 435 | 436 | Get and remove the data stored in a given key in a single operation. 437 | 438 | See :py:meth:`KyotoTycoon.seize`. 439 | 440 | .. py:method:: match_prefix(prefix, max_keys=None, db=None) 441 | 442 | :param str prefix: key prefix to match 443 | :param int max_keys: maximum number of results to return (optional) 444 | :param int db: database index 445 | :return: list of keys that matched the given prefix. 446 | :rtype: list 447 | 448 | .. py:method:: match_regex(regex, max_keys=None, db=None) 449 | 450 | :param str regex: regular-expression to match 451 | :param int max_keys: maximum number of results to return (optional) 452 | :param int db: database index 453 | :return: list of keys that matched the given regular expression. 454 | :rtype: list 455 | 456 | .. py:method:: match_similar(origin, distance=None, max_keys=None, db=None) 457 | 458 | :param str origin: source string for comparison 459 | :param int distance: maximum edit-distance for similarity (optional) 460 | :param int max_keys: maximum number of results to return (optional) 461 | :param int db: database index 462 | :return: list of keys that were within a certain edit-distance of origin 463 | :rtype: list 464 | 465 | .. py:method:: cursor(db=None, cursor_id=None) 466 | 467 | :param int db: database index 468 | :param int cursor_id: cursor id (will be automatically created if None) 469 | :return: :py:class:`Cursor` object 470 | 471 | .. py:method:: keys(db=None) 472 | 473 | :param int db: database index 474 | :return: all keys in database 475 | :rtype: generator 476 | 477 | .. warning:: 478 | The :py:meth:`~KyotoCabinet.keys` method uses a cursor and can be 479 | rather slow. 480 | 481 | .. py:method:: keys_nonlazy(db=None) 482 | 483 | :param int db: database index 484 | :return: all keys in database 485 | :rtype: list 486 | 487 | Non-lazy implementation of :py:meth:`~KyotoTycoon.keys`. 488 | Behind-the-scenes, calls :py:meth:`~KyotoTycoon.match_prefix` with an 489 | empty string as the prefix. 490 | 491 | .. py:method:: values(db=None) 492 | 493 | :param int db: database index 494 | :return: all values in database 495 | :rtype: generator 496 | 497 | .. py:method:: items(db=None) 498 | 499 | :param int db: database index 500 | :return: all key/value tuples in database 501 | :rtype: generator 502 | 503 | .. py:attribute:: size 504 | 505 | Property which exposes the size information returned by the 506 | :py:meth:`~KyotoTycoon.status` API, for the default database. 507 | 508 | .. py:attribute:: path 509 | 510 | Property which exposes the filename/path returned by the 511 | :py:meth:`~KyotoTycoon.status` API, for the default database. 512 | 513 | .. py:method:: set_database(db) 514 | 515 | :param int db: database index 516 | 517 | Specify the default database index for the client. 518 | 519 | Tokyo Tyrant client 520 | ------------------- 521 | 522 | .. py:class:: TokyoTyrant(host='127.0.0.1', port=1978, serializer=KT_BINARY, decode_keys=True, timeout=None, connection_pool=False) 523 | 524 | :param str host: server host. 525 | :param int port: server port. 526 | :param serializer: serialization method to use for storing/retrieving values. 527 | Accepts ``KT_BINARY``, ``KT_JSON``, ``KT_MSGPACK``, ``KT_NONE``, ``KT_PICKLE``, 528 | or ``TT_TABLE`` (for use with table databases). 529 | :param bool decode_keys: automatically decode keys, encoded as UTF-8. 530 | :param int timeout: socket timeout (optional). 531 | :param bool connection_pool: use a connection pool to manage sockets. 532 | 533 | Client for interacting with Tokyo Tyrant database. 534 | 535 | .. py:method:: close(allow_reuse=True) 536 | 537 | :param bool allow_reuse: when the connection pool is enabled, this flag 538 | indicates whether the connection can be reused. For unpooled 539 | clients this flag has no effect. 540 | 541 | Close the connection to the server. 542 | 543 | .. py:method:: close_all() 544 | 545 | When using the connection pool, this method can close *all* client 546 | connections. 547 | 548 | .. py:method:: get_bulk(keys, decode_values=True) 549 | 550 | :param list keys: list of keys to retrieve 551 | :param bool decode_values: decode values using the configured 552 | serialization scheme. 553 | :return: dictionary of all key/value pairs that were found 554 | 555 | Efficiently retrieve multiple key/value pairs from the database. If a 556 | key does not exist, it will not be present in the result dictionary. 557 | 558 | .. py:method:: get(key) 559 | 560 | :param str key: key to look-up 561 | :return: deserialized value or ``None`` if key does not exist. 562 | 563 | Fetch and (optionally) deserialize the value for the given key. 564 | 565 | .. py:method:: get_bytes(key) 566 | 567 | :param str key: key to look-up 568 | :return: raw bytestring value or ``None`` if key does not exist. 569 | 570 | Fetch the value for the given key. The resulting value will not 571 | be deserialized. 572 | 573 | .. py:method:: set_bulk(data, no_reply=False, encode_values=True) 574 | 575 | :param dict data: mapping of key/value pairs to set. 576 | :param bool no_reply: execute the operation without a server 577 | acknowledgment. 578 | :param bool encode_values: serialize the values using the configured 579 | serialization scheme (e.g., ``KT_MSGPACK``). 580 | :return: boolean indicating success, or ``None`` if ``no_reply``. 581 | 582 | Efficiently set multiple key/value pairs. 583 | 584 | .. py:method:: set(key, value) 585 | 586 | :param str key: key to set 587 | :param value: value to store (will be serialized using serializer) 588 | :return: boolean indicating success 589 | 590 | Set a single key/value pair. 591 | 592 | .. py:method:: set_bytes(key, value) 593 | 594 | :param str key: key to set 595 | :param value: raw value to store 596 | :return: boolean indicating success 597 | 598 | Set a single key/value pair without encoding the value. 599 | 600 | .. py:method:: remove_bulk(keys) 601 | 602 | :param list keys: list of keys to remove 603 | :return: boolean indicating success 604 | 605 | .. py:method:: remove(key) 606 | 607 | :param str key: key to remove 608 | :return: boolean indicating success 609 | 610 | .. py:method:: script(name, key=None, value=None, lock_records=False, lock_all=False, encode_value=True, decode_result=False, as_list=False, as_dict=False, as_int=False) 611 | 612 | :param str name: name of lua function to call 613 | :param str key: key to pass to lua function (optional) 614 | :param str value: value to pass to lua function (optional) 615 | :param bool lock_records: lock records modified during script execution 616 | :param bool lock_all: lock all records during script execution 617 | :param bool encode_value: serialize the value before sending to the script 618 | :param bool decode_value: deserialize the script return value 619 | :param bool as_list: deserialize newline-separated value into a list 620 | :param bool as_dict: deserialize list of tab-separated key/value pairs into dict 621 | :param bool as_int: return value as integer 622 | :return: byte-string or object returned by function (depending on decode_value) 623 | 624 | Execute a lua function, passing as arguments the given ``key`` and 625 | ``value`` (if provided). The return value is a bytestring, which can be 626 | deserialized by specifying ``decode_value=True``. The arguments 627 | ``as_list``, ``as_dict`` and ``as_int`` can be used to apply specific 628 | deserialization to the returned value. 629 | 630 | .. py:method:: clear() 631 | 632 | :return: boolean indicating success 633 | 634 | Remove all keys from the database. 635 | 636 | .. py:method:: status() 637 | 638 | :return: status fields and values 639 | :rtype: dict 640 | 641 | Obtain status information from the server. 642 | 643 | .. py:method:: synchronize() 644 | 645 | :return: boolean indicating success 646 | 647 | Synchronize data to disk. 648 | 649 | .. py:method:: optimize(options) 650 | 651 | :param str options: option format string to use when optimizing database. 652 | :return: boolean indicating success 653 | 654 | .. py:method:: add(key, value, encode_value=True) 655 | 656 | :param str key: key to add 657 | :param value: value to store 658 | :param bool encode_value: serialize the value using the configured 659 | serialization scheme. 660 | :return: boolean indicating if key could be added or not 661 | 662 | Add a key/value pair to the database. This operation will only succeed 663 | if the key does not already exist in the database. 664 | 665 | .. py:method:: append(key, value, encode_value=True) 666 | 667 | :param str key: key to append value to 668 | :param value: value to append 669 | :param bool encode_value: serialize the value using the configured 670 | serialization scheme. 671 | :return: boolean indicating if value was appended 672 | 673 | Appends data to an existing key/value pair. If the key does not exist, 674 | this is equivalent to the :py:meth:`~TokyoTyrant.set` method. 675 | 676 | .. py:method:: addshl(key, value, width, encode_value=True) 677 | 678 | :param str key: key to append value to 679 | :param value: data to append 680 | :param int width: number of bytes to shift 681 | :param bool encode_value: serialize the value using the configured 682 | serialization scheme. 683 | :return: boolean indicating success 684 | 685 | Concatenate a value at the end of the existing record and shift it to 686 | the left by *width* bytes. 687 | 688 | .. py:method:: exists(key) 689 | 690 | :param str key: key to test 691 | :return: boolean indicating if key exists 692 | 693 | Return whether or not the given key exists in the database. 694 | 695 | .. py:method:: length(key) 696 | 697 | :param str key: key 698 | :param int db: database index 699 | :return: length of the value in bytes, or ``None`` if not found 700 | 701 | Return the length of the raw value stored at the given key. If the key 702 | does not exist, returns ``None``. 703 | 704 | .. py:method:: seize(key, decode_value=True) 705 | 706 | :param str key: key to remove 707 | :param bool decode_value: deserialize the value using the configured 708 | serialization method. 709 | :return: value stored at given key or ``None`` if key does not exist. 710 | 711 | Get and remove the data stored in a given key in a single operation. 712 | 713 | .. py:method:: incr(key, n=1) 714 | 715 | :param str key: key to increment 716 | :param int n: value to add 717 | :return: incremented result value 718 | 719 | .. py:method:: incr_double(key, n=1.) 720 | 721 | :param str key: key to increment 722 | :param float n: value to add 723 | :return: incremented result value 724 | 725 | Increment the floating-point value stored in the given key. 726 | 727 | .. py:method:: count() 728 | 729 | :return: number of key/value pairs in the database 730 | :rtype: int 731 | 732 | Count the number of key/value pairs in the database 733 | 734 | .. py:method:: __getitem__(key) 735 | 736 | Get value at given ``key``. Identical to :py:meth:`~TokyoTyrant.get`. 737 | 738 | .. note:: 739 | If the database is a tree, a slice of keys can be used to retrieve 740 | an ordered range of values. 741 | 742 | .. py:method:: __setitem__(key, value) 743 | 744 | Set value at given ``key``. Identical to :py:meth:`~TokyoTyrant.set`. 745 | 746 | .. py:method:: __delitem__(key) 747 | 748 | Remove the given ``key``. Identical to :py:meth:`~TokyoTyrant.remove`. 749 | 750 | .. py:method:: __contains__(key) 751 | 752 | Check if given ``key`` exists. Identical to :py:meth:`~TokyoTyrant.exists`. 753 | 754 | .. py:method:: __len__() 755 | 756 | :return: total number of keys in the database. 757 | 758 | Identical to :py:meth:`~TokyoTyrant.count`. 759 | 760 | .. py:method:: update(__data=None, no_reply=False, encode_values=True, **kwargs) 761 | 762 | :param dict __data: mapping of key/value pairs to set. 763 | :param bool no_reply: execute the operation without a server 764 | acknowledgment. 765 | :param bool encode_values: serialize the values using the configured 766 | serialization scheme. 767 | :param kwargs: arbitrary key/value pairs to set. 768 | :return: boolean indicating success. 769 | 770 | Efficiently set multiple key/value pairs. Data can be provided as a 771 | dict or as an arbitrary number of keyword arguments. 772 | 773 | See also: :py:meth:`~TokyoTyrant.set_bulk`. 774 | 775 | .. py:method:: setdup(key, value, encode_value=True) 776 | 777 | :param str key: key to set 778 | :param value: value to store 779 | :param bool encode_value: serialize the value using the configured 780 | serialization scheme. 781 | :return: boolean indicating success 782 | 783 | Set key/value pair. If using a B-Tree and the key already exists, the 784 | new value will be added to the beginning. 785 | 786 | .. py:method:: setdupback(key, value) 787 | 788 | :param str key: key to set 789 | :param value: value to store 790 | :param bool encode_value: serialize the value using the configured 791 | serialization scheme. 792 | :return: boolean indicating success 793 | 794 | Set key/value pair. If using a B-Tree and the key already exists, the 795 | new value will be added to the end. 796 | 797 | .. py:method:: get_part(key, start=None, end=None, decode_value=True) 798 | 799 | :param str key: key to look-up 800 | :param int start: start offset 801 | :param int end: number of characters to retrieve (after start). 802 | :param bool decode_value: deserialize the value using the configured 803 | serialization scheme. 804 | :return: the substring portion of value requested or ``False`` if the 805 | value does not exist or the start index exceeded the value length. 806 | 807 | .. py:method:: misc(cmd, args=None, update_log=True) 808 | 809 | :param str cmd: Command to execute 810 | :param list args: Zero or more bytestring arguments to misc function. 811 | :param bool update_log: Add misc command to update log. 812 | 813 | Run a miscellaneous command using the "misc" API. Returns a list of 814 | zero or more bytestrings. 815 | 816 | .. py:attribute:: size 817 | 818 | Property which exposes the size of the database. 819 | 820 | .. py:attribute:: error 821 | 822 | Return a 2-tuple of error code and message for the last error reported 823 | by the server (if set). 824 | 825 | .. py:method:: copy(path) 826 | 827 | :param str path: destination for copy of database. 828 | :return: boolean indicating success 829 | 830 | Copy the database file to the given path. 831 | 832 | .. py:method:: restore(path, timestamp, options=0) 833 | 834 | :param str path: path to update log directory 835 | :param datetime timestamp: datetime from which to restore 836 | :param int options: optional flags 837 | :return: boolean indicating success 838 | 839 | Restore the database file from the update log. 840 | 841 | .. py:method:: set_master(host, port, timestamp, options=0) 842 | 843 | :param str host: host of master server 844 | :param int port: port of master server 845 | :param datetime timestamp: start timestamp 846 | :param int options: optional flags 847 | :return: boolean indicating success 848 | 849 | Set the replication master. 850 | 851 | .. py:method:: clear_cache() 852 | 853 | :return: boolean indicating success 854 | 855 | .. py:method:: defragment(nsteps=None) 856 | 857 | :param int nsteps: number of defragmentation steps 858 | :return: boolean indicating success 859 | 860 | Defragment the database. 861 | 862 | .. py:method:: get_range(start, stop=None, max_keys=0, decode_values=True) 863 | 864 | :param str start: start-key for range 865 | :param str stop: stop-key for range (optional) 866 | :param int max_keys: maximum keys to fetch 867 | :param bool decode_values: deserialize the values using the configured 868 | serialization scheme. 869 | :return: a dictionary mapping of key-value pairs falling within the 870 | given range. 871 | 872 | Fetch a range of key/value pairs and return them as a dictionary. 873 | 874 | .. note:: Only works with tree databases. 875 | 876 | .. py:method:: get_rangelist(start, stop=None, max_keys=0, decode_values=True) 877 | 878 | :param str start: start-key for range 879 | :param str stop: stop-key for range (optional) 880 | :param int max_keys: maximum keys to fetch 881 | :param bool decode_values: deserialize the values using the configured 882 | serialization scheme. 883 | :return: a list of ordered key-value pairs falling within the given range. 884 | 885 | Fetch a range of key/value pairs and return them as an ordered list of 886 | key/value tuples. 887 | 888 | .. note:: Only works with tree databases. 889 | 890 | .. py:method:: match_prefix(prefix, max_keys=1024) 891 | 892 | :param str prefix: key prefix to match 893 | :param int max_keys: maximum number of results to return 894 | :return: list of keys that matched the given prefix. 895 | 896 | .. py:method:: match_regex(regex, max_keys=None, decode_values=True) 897 | 898 | :param str regex: regular-expression to match 899 | :param int max_keys: maximum number of results to return 900 | :param bool decode_values: deserialize the values using the configured 901 | serialization scheme. 902 | :return: a dictionary mapping of key-value pairs which matched the regex. 903 | 904 | .. py:method:: match_regexlist(regex, max_keys=None, decode_values=True) 905 | 906 | :param str regex: regular-expression to match 907 | :param int max_keys: maximum number of results to return 908 | :param bool decode_values: deserialize the values using the configured 909 | serialization scheme. 910 | :return: a list of ordered key-value pairs which matched the regex. 911 | 912 | .. py:method:: iter_from(start_key) 913 | 914 | :param start_key: key to start iteration. 915 | :return: list of key/value tuples obtained by iterating from start-key. 916 | 917 | .. py:method:: keys() 918 | 919 | :return: list all keys in database 920 | :rtype: generator 921 | 922 | .. py:method:: keys_fast() 923 | 924 | :return: list of all keys in database 925 | :rtype: list 926 | 927 | Return a list of all keys in the database in a single operation. 928 | 929 | .. py:method:: items() 930 | 931 | :return: list all key/value tuples in database 932 | :rtype: generator 933 | 934 | .. py:method:: items_fast() 935 | 936 | :return: list of all key/value tuples in database in a single operation. 937 | :rtype: list 938 | 939 | .. py:method:: set_index(name, index_type, check_exists=False) 940 | 941 | :param str name: column name to index 942 | :param int index_type: see :ref:`index-types` for values 943 | :param bool check_exists: if true, an error will be raised if the index 944 | already exists. 945 | :return: boolean indicating success 946 | 947 | Create an index on the given column in a table database. 948 | 949 | .. py:method:: optimize_index(name) 950 | 951 | :param str name: column name index to optimize 952 | :return: boolean indicating success 953 | 954 | Optimize the index on a given column. 955 | 956 | .. py:method:: delete_index(name) 957 | 958 | :param str name: column name index to delete 959 | :return: boolean indicating success 960 | 961 | Delete the index on a given column. 962 | 963 | .. py:method:: search(expressions, cmd=None) 964 | 965 | :param list expressions: zero or more search expressions 966 | :param str cmd: extra command to apply to search results 967 | :return: varies depending on ``cmd``. 968 | 969 | Perform a search on a table database. Rather than call this method 970 | directly, it is recommended that you use the :py:class:`QueryBuilder` 971 | to construct and execute table queries. 972 | 973 | .. py:method:: genuid() 974 | 975 | :return: integer id 976 | 977 | Generate a unique ID. 978 | 979 | 980 | .. py:class:: QueryBuilder 981 | 982 | Construct and execute table queries. 983 | 984 | .. py:method:: filter(column, op, value) 985 | 986 | :param str column: column name to filter on 987 | :param int op: operation, see :ref:`filter-types` for available values 988 | :param value: value for filter expression 989 | 990 | Add a filter expression to the query. 991 | 992 | .. py:method:: order_by(column, ordering=None) 993 | 994 | :param str column: column name to order by 995 | :param int ordering: ordering method, defaults to lexical ordering. 996 | See :ref:`ordering-types` for available values. 997 | 998 | Specify ordering of query results. 999 | 1000 | .. py:method:: limit(limit=None) 1001 | 1002 | :param int limit: maximum number of results 1003 | 1004 | Limit the number of results returned by query. 1005 | 1006 | .. py:method:: offset(offset=None) 1007 | 1008 | :param int offset: number of results to skip over. 1009 | 1010 | Skip over results returned by query. 1011 | 1012 | .. py:method:: execute(client) 1013 | 1014 | :param TokyoTyrant client: database client 1015 | :return: list of keys matching query criteria 1016 | :rtype: list 1017 | 1018 | Execute the query and return a list of the keys of matching records. 1019 | 1020 | .. py:method:: delete(client) 1021 | 1022 | :param TokyoTyrant client: database client 1023 | :return: boolean indicating success 1024 | 1025 | Delete records that match the query criteria. 1026 | 1027 | .. py:method:: get(client) 1028 | 1029 | :param TokyoTyrant client: database client 1030 | :return: list of 2-tuples consisting of ``key, value``. 1031 | :rtype list: 1032 | 1033 | Execute query and return a list of keys and values for records matching 1034 | the query criteria. 1035 | 1036 | .. py:method:: count(client) 1037 | 1038 | :param TokyoTyrant client: database client 1039 | :return: number of query results 1040 | 1041 | Return count of matching records. 1042 | 1043 | 1044 | .. _index-types: 1045 | 1046 | Index types 1047 | ^^^^^^^^^^^ 1048 | 1049 | .. py:data:: INDEX_STR 1050 | 1051 | .. py:data:: INDEX_NUM 1052 | 1053 | .. py:data:: INDEX_TOKEN 1054 | 1055 | .. py:data:: INDEX_QGRAM 1056 | 1057 | .. _filter-types: 1058 | 1059 | Filter types 1060 | ^^^^^^^^^^^^ 1061 | 1062 | .. py:data:: OP_STR_EQ 1063 | 1064 | .. py:data:: OP_STR_CONTAINS 1065 | 1066 | .. py:data:: OP_STR_STARTSWITH 1067 | 1068 | .. py:data:: OP_STR_ENDSWITH 1069 | 1070 | .. py:data:: OP_STR_ALL 1071 | 1072 | .. py:data:: OP_STR_ANY 1073 | 1074 | .. py:data:: OP_STR_ANYEXACT 1075 | 1076 | .. py:data:: OP_STR_REGEX 1077 | 1078 | .. py:data:: OP_NUM_EQ 1079 | 1080 | .. py:data:: OP_NUM_GT 1081 | 1082 | .. py:data:: OP_NUM_GE 1083 | 1084 | .. py:data:: OP_NUM_LT 1085 | 1086 | .. py:data:: OP_NUM_LE 1087 | 1088 | .. py:data:: OP_NUM_BETWEEN 1089 | 1090 | .. py:data:: OP_NUM_ANYEXACT 1091 | 1092 | .. py:data:: OP_FTS_PHRASE 1093 | 1094 | .. py:data:: OP_FTS_ALL 1095 | 1096 | .. py:data:: OP_FTS_ANY 1097 | 1098 | .. py:data:: OP_FTS_EXPRESSION 1099 | 1100 | .. py:data:: OP_NEGATE 1101 | 1102 | Combine with other operand using bitwise-or to negate the filter. 1103 | 1104 | .. py:data:: OP_NOINDEX 1105 | 1106 | Combine with other operand using bitwise-or to prevent using an index. 1107 | 1108 | .. _ordering-types: 1109 | 1110 | Ordering types 1111 | ^^^^^^^^^^^^^^ 1112 | 1113 | .. py:data:: ORDER_STR_ASC 1114 | 1115 | .. py:data:: ORDER_STR_DESC 1116 | 1117 | .. py:data:: ORDER_NUM_ASC 1118 | 1119 | .. py:data:: ORDER_NUM_DESC 1120 | 1121 | Embedded Servers 1122 | ---------------- 1123 | 1124 | .. py:class:: EmbeddedServer(server='ktserver', host='127.0.0.1', port=None, database='*', server_args=None) 1125 | 1126 | :param str server: path to ktserver executable 1127 | :param str host: host to bind server on 1128 | :param int port: port to use (optional) 1129 | :param str database: database filename, default is in-memory hash table 1130 | :param list server_args: additional command-line arguments for server 1131 | 1132 | Create a manager for running an embedded (sub-process) Kyoto Tycoon server. 1133 | If the port is not specified, a random high port will be used. 1134 | 1135 | Example: 1136 | 1137 | .. code-block:: pycon 1138 | 1139 | >>> from kt import EmbeddedServer 1140 | >>> server = EmbeddedServer() 1141 | >>> server.run() 1142 | True 1143 | >>> client = server.client 1144 | >>> client.set('k1', 'v1') 1145 | 1 1146 | >>> client.get('k1') 1147 | 'v1' 1148 | >>> server.stop() 1149 | True 1150 | 1151 | .. py:method:: run() 1152 | 1153 | :return: boolean indicating if server successfully started 1154 | 1155 | Run ``ktserver`` in a sub-process. 1156 | 1157 | .. py:method:: stop() 1158 | 1159 | :return: boolean indicating if server was stopped 1160 | 1161 | Stop the running embedded server. 1162 | 1163 | .. py:attribute:: client 1164 | 1165 | :py:class:`KyotoTycoon` client bound to the embedded server. 1166 | 1167 | 1168 | .. py:class:: EmbeddedTokyoTyrantServer(server='ttserver', host='127.0.0.1', port=None, database='*', server_args=None) 1169 | 1170 | :param str server: path to ttserver executable 1171 | :param str host: host to bind server on 1172 | :param int port: port to use (optional) 1173 | :param str database: database filename, default is in-memory hash table 1174 | :param list server_args: additional command-line arguments for server 1175 | 1176 | Create a manager for running an embedded (sub-process) Tokyo Tyrant server. 1177 | If the port is not specified, a random high port will be used. 1178 | 1179 | Example: 1180 | 1181 | .. code-block:: pycon 1182 | 1183 | >>> from kt import EmbeddedTokyoTyrantServer 1184 | >>> server = EmbeddedTokyoTyrantServer() 1185 | >>> server.run() 1186 | True 1187 | >>> client = server.client 1188 | >>> client.set('k1', 'v1') 1189 | True 1190 | >>> client.get('k1') 1191 | 'v1' 1192 | >>> server.stop() 1193 | True 1194 | 1195 | .. py:method:: run() 1196 | 1197 | :return: boolean indicating if server successfully started 1198 | 1199 | Run ``ttserver`` in a sub-process. 1200 | 1201 | .. py:method:: stop() 1202 | 1203 | :return: boolean indicating if server was stopped 1204 | 1205 | Stop the running embedded server. 1206 | 1207 | .. py:attribute:: client 1208 | 1209 | :py:class:`TokyoTyrant` client bound to the embedded server. 1210 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/stable/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | # import os 16 | # import sys 17 | # sys.path.insert(0, os.path.abspath('.')) 18 | 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | project = u'kt' 23 | copyright = u'2018, charles leifer' 24 | author = u'charles leifer' 25 | 26 | import os 27 | import sys 28 | 29 | src_dir = os.path.realpath(os.path.dirname(os.path.dirname(__file__))) 30 | sys.path.insert(0, src_dir) 31 | 32 | try: 33 | from kt import __version__ 34 | except ImportError: 35 | import warnings 36 | warnings.warn('Unable to determine project version!') 37 | __version__ = '0.7.0' 38 | 39 | # The short X.Y version 40 | version = __version__ 41 | # The full version, including alpha/beta/rc tags 42 | release = __version__ 43 | 44 | 45 | # -- General configuration --------------------------------------------------- 46 | 47 | # If your documentation needs a minimal Sphinx version, state it here. 48 | # 49 | # needs_sphinx = '1.0' 50 | 51 | # Add any Sphinx extension module names here, as strings. They can be 52 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 53 | # ones. 54 | extensions = [ 55 | ] 56 | 57 | # Add any paths that contain templates here, relative to this directory. 58 | templates_path = ['_templates'] 59 | 60 | # The suffix(es) of source filenames. 61 | # You can specify multiple suffix as a list of string: 62 | # 63 | # source_suffix = ['.rst', '.md'] 64 | source_suffix = '.rst' 65 | 66 | # The master toctree document. 67 | master_doc = 'index' 68 | 69 | # The language for content autogenerated by Sphinx. Refer to documentation 70 | # for a list of supported languages. 71 | # 72 | # This is also used if you do content translation via gettext catalogs. 73 | # Usually you set "language" from the command line for these cases. 74 | language = None 75 | 76 | # List of patterns, relative to source directory, that match files and 77 | # directories to ignore when looking for source files. 78 | # This pattern also affects html_static_path and html_extra_path . 79 | exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store'] 80 | 81 | # The name of the Pygments (syntax highlighting) style to use. 82 | pygments_style = 'sphinx' 83 | 84 | 85 | # -- Options for HTML output ------------------------------------------------- 86 | 87 | # The theme to use for HTML and HTML Help pages. See the documentation for 88 | # a list of builtin themes. 89 | # 90 | html_theme = 'alabaster' 91 | 92 | # Theme options are theme-specific and customize the look and feel of a theme 93 | # further. For a list of options available for each theme, see the 94 | # documentation. 95 | # 96 | # html_theme_options = {} 97 | 98 | # Add any paths that contain custom static files (such as style sheets) here, 99 | # relative to this directory. They are copied after the builtin static files, 100 | # so a file named "default.css" will overwrite the builtin "default.css". 101 | html_static_path = ['_static'] 102 | 103 | # Custom sidebar templates, must be a dictionary that maps document names 104 | # to template names. 105 | # 106 | # The default sidebars (for documents that don't match any pattern) are 107 | # defined by theme itself. Builtin themes are using these templates by 108 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 109 | # 'searchbox.html']``. 110 | # 111 | # html_sidebars = {} 112 | 113 | 114 | # -- Options for HTMLHelp output --------------------------------------------- 115 | 116 | # Output file base name for HTML help builder. 117 | htmlhelp_basename = 'ktdoc' 118 | 119 | 120 | # -- Options for LaTeX output ------------------------------------------------ 121 | 122 | latex_elements = { 123 | # The paper size ('letterpaper' or 'a4paper'). 124 | # 125 | # 'papersize': 'letterpaper', 126 | 127 | # The font size ('10pt', '11pt' or '12pt'). 128 | # 129 | # 'pointsize': '10pt', 130 | 131 | # Additional stuff for the LaTeX preamble. 132 | # 133 | # 'preamble': '', 134 | 135 | # Latex figure (float) alignment 136 | # 137 | # 'figure_align': 'htbp', 138 | } 139 | 140 | # Grouping the document tree into LaTeX files. List of tuples 141 | # (source start file, target name, title, 142 | # author, documentclass [howto, manual, or own class]). 143 | latex_documents = [ 144 | (master_doc, 'kt.tex', u'kt Documentation', 145 | u'charles leifer', 'manual'), 146 | ] 147 | 148 | 149 | # -- Options for manual page output ------------------------------------------ 150 | 151 | # One entry per manual page. List of tuples 152 | # (source start file, name, description, authors, manual section). 153 | man_pages = [ 154 | (master_doc, 'kt', u'kt Documentation', 155 | [author], 1) 156 | ] 157 | 158 | 159 | # -- Options for Texinfo output ---------------------------------------------- 160 | 161 | # Grouping the document tree into Texinfo files. List of tuples 162 | # (source start file, target name, title, author, 163 | # dir menu entry, description, category) 164 | texinfo_documents = [ 165 | (master_doc, 'kt', u'kt Documentation', 166 | author, 'kt', 'One line description of project.', 167 | 'Miscellaneous'), 168 | ] 169 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. kt documentation master file, created by 2 | sphinx-quickstart on Thu Apr 12 18:36:01 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | kt 7 | == 8 | 9 | .. image:: logo.png 10 | 11 | *kt* is a fast client library for use with `Kyoto Tycoon `_ 12 | and `Tokyo Tyrant `_. *kt* is designed to be 13 | performant and simple-to-use. 14 | 15 | * Full-featured implementation of protocol. 16 | * Binary protocols implemented as C extension. 17 | * Thread-safe and greenlet-safe. 18 | * Socket pooling. 19 | * Simple APIs. 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | :caption: Contents: 24 | 25 | installation 26 | usage 27 | api 28 | 29 | 30 | Indices and tables 31 | ================== 32 | 33 | * :ref:`genindex` 34 | * :ref:`modindex` 35 | * :ref:`search` 36 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. _installation: 2 | 3 | Installation 4 | ============ 5 | 6 | *kt* can be installed using ``pip``: 7 | 8 | .. code-block:: bash 9 | 10 | $ pip install kt 11 | 12 | Dependencies 13 | ------------ 14 | 15 | * *cython* - for building the binary protocol C extension. 16 | 17 | These libraries are installed automatically if you install *kt* with pip. To 18 | install these dependencies manually, run: 19 | 20 | .. code-block:: bash 21 | 22 | $ pip install cython 23 | 24 | Installing with git 25 | ------------------- 26 | 27 | To install the latest version with git: 28 | 29 | .. code-block:: bash 30 | 31 | $ git clone https://github.com/coleifer/kt 32 | $ cd kt/ 33 | $ python setup.py install 34 | 35 | Installing Kyoto Tycoon or Tokyo Tyrant 36 | --------------------------------------- 37 | 38 | If you're using a debian-based linux distribution, you can install using 39 | ``apt-get``: 40 | 41 | .. code-block:: bash 42 | 43 | $ sudo apt-get install kyototycoon tokyotyrant 44 | 45 | Alternatively you can use the following Docker images: 46 | 47 | .. code-block:: bash 48 | 49 | $ docker run -it --rm -v kyoto:/var/lib/kyototycoon -p 1978:1978 coleifer/kyototycoon 50 | $ docker run -it --rm -v tokyo:/var/lib/tokyotyrant -p 9871:9871 coleifer/tokyohash 51 | 52 | To build from source and read about the various command-line options, see the 53 | project documentation: 54 | 55 | * `Kyoto Tycoon documentation `_ 56 | * `Tokyo Tyrant documentation `_ 57 | -------------------------------------------------------------------------------- /docs/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coleifer/kt/40b62f46862061f2207ad922413d4e4cd66996ab/docs/logo.png -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | .. _usage: 2 | 3 | Usage 4 | ===== 5 | 6 | This document describes how to use *kt* with Kyoto Tycoon and Tokyo Tyrant. 7 | 8 | Common Features 9 | --------------- 10 | 11 | This section describes features and APIs that are common to both the 12 | :py:class:`KyotoTycoon` client and the :py:class:`TokyoTyrant` client. For 13 | simplicity, we'll use the :py:class:`EmbeddedServer`, which sets up the 14 | database server in a subprocess and makes it easy to develop. 15 | 16 | .. code-block:: pycon 17 | 18 | >>> from kt import EmbeddedServer 19 | >>> server = EmbeddedServer() 20 | >>> server.run() # Starts "ktserver" in a subprocess. 21 | True 22 | >>> client = server.client # Get a client for use with our embedded server. 23 | 24 | As you would expect for a key/value database, the client implements 25 | :py:meth:`~KyotoTycoon.get`, :py:meth:`~KyotoTycoon.set` and 26 | :py:meth:`~KyotoTycoon.remove`: 27 | 28 | .. code-block:: pycon 29 | 30 | >>> client.set('k1', 'v1') 31 | 1 32 | >>> client.get('k1') 33 | 'v1' 34 | >>> client.remove('k1') 35 | 1 36 | 37 | It is not an error to try to get or delete a key that doesn't exist: 38 | 39 | .. code-block:: pycon 40 | 41 | >>> client.get('not-here') # Returns None 42 | >>> client.remove('not-here') 43 | 0 44 | 45 | To check whether a key exists we can use :py:meth:`~KyotoTycoon.exists`: 46 | 47 | .. code-block:: pycon 48 | 49 | >>> client.set('k1', 'v1') 50 | >>> client.exists('k1') 51 | True 52 | >>> client.exists('not-here') 53 | False 54 | 55 | In addition, there are also efficient methods for bulk operations: 56 | :py:meth:`~KyotoTycoon.get_bulk`, :py:meth:`~KyotoTycoon.set_bulk` and 57 | :py:meth:`~KyotoTycoon.remove_bulk`: 58 | 59 | .. code-block:: pycon 60 | 61 | >>> client.set_bulk({'k1': 'v1', 'k2': 'v2', 'k3': 'v3'}) 62 | 3 63 | >>> client.get_bulk(['k1', 'k2', 'k3', 'not-here']) 64 | {'k1': 'v1', 'k2': 'v2', 'k3': 'v3'} 65 | >>> client.remove_bulk(['k1', 'k2', 'k3', 'not-here']) 66 | 3 67 | 68 | The client libraries also support a dict-like interface: 69 | 70 | .. code-block:: pycon 71 | 72 | >>> client['k1'] = 'v1' 73 | >>> print(client['k1']) 74 | v1 75 | >>> del client['k1'] 76 | >>> client.update({'k1': 'v1', 'k2': 'v2', 'k3': 'v3'}) 77 | 3 78 | >>> client.pop('k1') 79 | 'v1' 80 | >>> client.pop('k1') # Returns None 81 | >>> 'k1' in client 82 | False 83 | >>> len(client) 84 | 2 85 | 86 | To remove all records, you can use the :py:meth:`~KyotoTycoon.clear` method: 87 | 88 | .. code-block:: pycon 89 | 90 | >>> client.clear() 91 | True 92 | 93 | Serialization 94 | ^^^^^^^^^^^^^ 95 | 96 | By default the client will assume that keys and values should be encoded as 97 | UTF-8 byte-strings and decoded to unicode upon retrieval. You can set the 98 | ``serializer`` parameter when creating your client to use a different value 99 | serialization. *kt* provides the following: 100 | 101 | * ``KT_BINARY`` - **default**, treat values as unicode and serialize as UTF-8. 102 | * ``KT_JSON`` - use JSON to serialize values. 103 | * ``KT_MSGPACK`` - use msgpack to serialize values. 104 | * ``KT_PICKLE`` - use pickle to serialize values. 105 | * ``KT_NONE`` - no serialization, values must be bytestrings. 106 | 107 | For example, to use the pickle serializer: 108 | 109 | .. code-block:: pycon 110 | 111 | >>> from kt import KT_PICKLE, KyotoTycoon 112 | >>> client = KyotoTycoon(serializer=KT_PICKLE) 113 | >>> client.set('k1', {'this': 'is', 'a': ['python object']}) 114 | 1 115 | >>> client.get('k1') 116 | {'this': 'is', 'a': ['python object']} 117 | 118 | Kyoto Tycoon 119 | ------------ 120 | 121 | The Kyoto Tycoon section continues from the previous section, and assumes that 122 | you are running an :py:class:`EmbeddedServer` and accessing it through it's 123 | :py:attr:`~EmbeddedServer.client` property. 124 | 125 | Database filenames 126 | ^^^^^^^^^^^^^^^^^^ 127 | 128 | Kyoto Tycoon determines the database type by looking at the filename of the 129 | database(s) specified when ``ktserver`` is executed. Additionally, for 130 | in-memory databases, you use special symbols instead of filenames. 131 | 132 | * ``hash_table.kch`` - on-disk hash table ("kch"). 133 | * ``btree.kct`` - on-disk b-tree ("kct"). 134 | * ``dirhash.kcd`` - directory hash ("kcd"). 135 | * ``dirtree.kcf`` - directory b-tree ("kcf"). 136 | * ``*`` - cache-hash, in-memory hash-table with LRU deletion. 137 | * ``%`` - cache-tree, in-memory b-tree (ordered cache). 138 | * ``:`` - stash db, in-memory database with lower memory usage. 139 | * ``-`` - prototype hash, simple in-memory hash using ``std::unordered_map``. 140 | * ``+`` - prototype tree, simple in-memory hash using ``std::map`` (ordered). 141 | 142 | Generally: 143 | 144 | * For unordered collections, use either the cache-hash (``*``) or the 145 | file-hash (``.kch``). 146 | * For ordered collections or indexes, use either the cache-tree (``%``) or the 147 | file b-tree (``.kct``). 148 | * I avoid the prototype hash and btree as the *entire data-structure* is locked 149 | during writes (as opposed to an individual record or page). 150 | 151 | For more information about the above database types, their algorithmic 152 | complexity, and the unit of locking, see `kyotocabinet db chart `_. 153 | 154 | Key Expiration 155 | ^^^^^^^^^^^^^^ 156 | 157 | Kyoto Tycoon servers feature a built-in expiration mechanism, allowing you to 158 | use it as a cache. Whenever setting a value or otherwise writing to the 159 | database, you can also specify an expiration time (in seconds): 160 | 161 | .. code-block:: pycon 162 | 163 | >>> client.set('k1', 'v1', expire_time=5) 164 | >>> client.get('k1') 165 | 'v1' 166 | >>> time.sleep(5) 167 | >>> client.get('k1') # Returns None 168 | 169 | Multiple Databases 170 | ^^^^^^^^^^^^^^^^^^ 171 | 172 | Kyoto Tycoon can also be used as the front-end for multiple databases. For 173 | example, to start ``ktserver`` with an in-memory hash-table and an in-memory 174 | b-tree, you would run: 175 | 176 | .. code-block:: bash 177 | 178 | $ ktserver \* \% 179 | 180 | By default, the :py:class:`KyotoTycoon` client assumes you are working with the 181 | first database (starting from zero, our hash-table would be ``0`` and the 182 | b-tree would be ``1``). 183 | 184 | The client can be initialized to use a different database by default: 185 | 186 | .. code-block:: pycon 187 | 188 | >>> client = KyotoTycoon(default_db=1) 189 | 190 | To change the default database at run-time, you can call the 191 | :py:meth:`~KyotoTycoon.set_database` method: 192 | 193 | .. code-block:: pycon 194 | 195 | >>> client = KyotoTycoon() 196 | >>> client.set_database(1) 197 | 198 | Lastly, to perform a one-off operation against a specific database, all methods 199 | accept a ``db`` parameter which you can use to specify the database: 200 | 201 | .. code-block:: pycon 202 | 203 | >>> client.set('k1', 'v1', db=1) 204 | >>> client.get('k1', db=0) # Returns None 205 | >>> client.get('k1', db=1) 206 | 'v1' 207 | 208 | Similarly, if a ``tuple`` is passed into the dictionary APIs, it is assumed 209 | that the key consists of ``(key, db)`` and the value of ``(value, expire)``: 210 | 211 | .. code-block:: pycon 212 | 213 | >>> client['k1', 1] = 'v1' # Set k1=v1 in db1. 214 | >>> client['k1', 1] 215 | 'v1' 216 | >>> client['k2'] = ('v2', 10) # Set k2=v2 in default db with 10s expiration. 217 | >>> client['k2', 0] = ('v2', 20) # Set k2=v2 in db0 with 20s expiration. 218 | >>> del client['k1', 1] # Delete 'k1' in db1. 219 | 220 | Lua Scripts 221 | ^^^^^^^^^^^ 222 | 223 | Kyoto Tycoon can be scripted using `lua `_. 224 | To run a Lua script from the client, you can use the 225 | :py:meth:`~KyotoTycoon.script` method. In Kyoto Tycoon, a script may receive 226 | arbitrary key/value-pairs as parameters, and may return arbitrary key/value 227 | pairs: 228 | 229 | .. code-block:: pycon 230 | 231 | >>> client.script('myfunction', {'key': 'some-key', 'data': 'etc'}) 232 | {'data': 'returned', 'by': 'user-script'} 233 | 234 | To simplify script execution, you can also use the :py:meth:`~KyotoTycoon.lua` 235 | helper, which provides a slightly more Pythonic API: 236 | 237 | .. code-block:: pycon 238 | 239 | >>> lua = client.lua 240 | >>> lua.myfunction(key='some-key', data='etc') 241 | {'data': 'returned', 'by': 'user-script'} 242 | >>> lua.another_function(key='another-key') 243 | {} 244 | 245 | Learn more about scripting Kyoto Tycoon by reading the `lua doc `_. 246 | 247 | Tokyo Tyrant 248 | ------------ 249 | 250 | To experiment with Tokyo Tyrant, an easy way to get started is to use the 251 | :py:class:`EmbeddedTokyoTyrantServer`, which sets up the database server in a 252 | subprocess and makes it easy to develop. 253 | 254 | .. code-block:: pycon 255 | 256 | >>> from kt import EmbeddedTokyoTyrantServer 257 | >>> server = EmbeddedTokyoTyrantServer() 258 | >>> server.run() 259 | True 260 | >>> client = server.client 261 | 262 | .. note:: 263 | Unlike Kyoto Tycoon, the Tokyo Tyrant server process can only embed a 264 | single database, and does not support expiration. 265 | 266 | Database filenames 267 | ^^^^^^^^^^^^^^^^^^ 268 | 269 | Tokyo Tyrant determines the database type by looking at the filename of the 270 | database(s) specified when ``ttserver`` is executed. Additionally, for 271 | in-memory databases, you use special symbols instead of filenames. 272 | 273 | * ``hash_table.tch`` - on-disk hash table ("tch"). 274 | * ``btree.tcb`` - on-disk b-tree ("tcb"). 275 | * ``table.tct`` - on-disk table database ("tct"). 276 | * ``*`` - in-memory hash-table. 277 | * ``+`` - in-memory tree (ordered). 278 | 279 | There is an additional database-types, but their usage is beyond the scope of 280 | this document: 281 | 282 | * ``table.tcf`` - fixed-length database ("tcf"). 283 | 284 | For more information about the above database types, their algorithmic 285 | complexity, and the unit of locking, see `ttserver documentation `_. 286 | 287 | Using the table database 288 | ^^^^^^^^^^^^^^^^^^^^^^^^ 289 | 290 | The table database is neat, as it you can store another layer of key/value 291 | pairs in the value field. These key/value pairs are serialized using ``0x0`` as 292 | the delimiter. :py:class:`TokyoTyrant` provides a special serializer, 293 | ``TT_TABLE``, which properly handles reading and writing data dictionaries to a 294 | table database. 295 | 296 | .. code-block:: pycon 297 | 298 | >>> tt = TokyoTyrant(serializer=TT_TABLE) 299 | >>> tt.set('k1', {'name': 'charlie', 'location': 'Topeka, KS'}) 300 | True 301 | >>> tt.get('k1') 302 | {'name': 'charlie', 'location': 'Topeka, KS'} 303 | 304 | Table databases support a special search API which can be used to filter 305 | records by attributes stored in the value field. In this way, they act like a 306 | rudimentary relational database table, that can be queried with a simple 307 | ``WHERE`` clause. 308 | 309 | Secondary indexes can be created to improve the efficiency of the search API. 310 | 311 | Simple example using the :py:class:`QueryBuilder` helper. First let's setup our 312 | table database with some data: 313 | 314 | .. code-block:: pycon 315 | 316 | >>> from kt import TokyoTyrant 317 | >>> tt = TokyoTyrant(serializer=TT_TABLE) 318 | >>> tt.set_bulk({ 319 | ... 'huey': {'name': 'huey', 'type': 'cat', 'eyes': 'blue', 'age': '7'}, 320 | ... 'mickey': {'name': 'mickey', 'type': 'dog', 'eyes': 'blue', 'age': '9'}, 321 | ... 'zaizee': {'name': 'zaizee', 'type': 'cat', 'eyes': 'blue', 'age': '5'}, 322 | ... 'charlie': {'name': 'charlie', 'type': 'human', 'eyes': 'brown', 'age': '35'}, 323 | ... 'leslie': {'name': 'leslie', 'type': 'human', 'eyes': 'blue', 'age': '34'}, 324 | ... 'connor': {'name': 'connor', 'type': 'human', 'eyes': 'brown', 'age': '3'}, 325 | ... }) 326 | ... 327 | True 328 | 329 | We can use the :py:class:`QueryBuilder` to construct query expressions: 330 | 331 | .. code-block:: pycon 332 | 333 | >>> from kt import QueryBuilder 334 | >>> from kt import constants 335 | >>> query = (QueryBuilder() 336 | ... .filter('type', constants.OP_STR_EQ, 'cat') 337 | ... .order_by('name', constants.ORDER_STR_DESC)) 338 | ... 339 | >>> query.execute(tt) # Returns the matching keys. 340 | ['zaizee', 'huey'] 341 | 342 | We can use multiple filter expressions: 343 | 344 | .. code-block:: pycon 345 | 346 | >>> query = (QueryBuilder() 347 | ... .filter('age', constants.OP_NUM_GE, '7') 348 | ... .filter('type', constants.OP_STR_ANY, 'human,cat') 349 | ... .order_by('age', constants.ORDER_NUM_DESC)) 350 | >>> query.execute(tt) 351 | ['charlie', 'leslie', 'huey'] 352 | 353 | To get both the key and the associated value, we use the 354 | :py:meth:`QueryBuilder.get` method: 355 | 356 | .. code-block:: pycon 357 | 358 | >>> query.get(tt) 359 | [('charlie', {'name': 'charlie', 'type': 'human', ...}), 360 | ('leslie', {'name': 'leslie', 'type': 'human', ...}), 361 | ('huey', {'name': 'huey', 'type': 'cat', ...{)] 362 | 363 | To avoid scanning the entire table database, we can create secondary indexes on 364 | the fields we wish to query: 365 | 366 | .. code-block:: pycon 367 | 368 | >>> tt.set_index('name', constants.INDEX_STR) # Create string index on name. 369 | True 370 | >>> tt.set_index('age', constants.INDEX_NUM) # Numeric index on age. 371 | True 372 | 373 | We can optimize and delete indexes: 374 | 375 | .. code-block:: pycon 376 | 377 | >>> tt.optimize_index('name') # Optimize the "name" index. 378 | True 379 | >>> tt.delete_index('age') # Delete the "age" index. 380 | True 381 | 382 | Lua Scripts 383 | ^^^^^^^^^^^ 384 | 385 | Tokyo Tyrant can be scripted using `lua `_. 386 | To run a Lua script from the client, you can use the 387 | :py:meth:`~TokyoTyrant.script` method. In Tokyo Tyrant, a script may receive 388 | a key and a value parameter, and will return a byte-string as a result: 389 | 390 | .. code-block:: pycon 391 | 392 | >>> client.script('incr', key='counter', value='1') 393 | '1' 394 | >>> client.script('incr', 'counter', '4') 395 | '5' 396 | 397 | To simplify script execution, you can also use the :py:meth:`~TokyoTyrant.lua` 398 | helper, which provides a slightly more Pythonic API: 399 | 400 | .. code-block:: pycon 401 | 402 | >>> lua = client.lua 403 | >>> lua.incr(key='counter', value='2') 404 | '7' 405 | >>> lua.incr('counter', '1') 406 | '8' 407 | 408 | Learn more about scripting Tokyo Tyrant by reading the `lua docs `_. 409 | -------------------------------------------------------------------------------- /kt/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.9.6' 2 | 3 | from .client import KT_BINARY 4 | from .client import KT_JSON 5 | from .client import KT_MSGPACK 6 | from .client import KT_NONE 7 | from .client import KT_PICKLE 8 | from .client import KyotoTycoon 9 | from .client import QueryBuilder 10 | from .client import TokyoTyrant 11 | from .client import TT_TABLE 12 | from .embedded import EmbeddedServer 13 | from .embedded import EmbeddedTokyoTyrantServer 14 | from .exceptions import ImproperlyConfigured 15 | from .exceptions import KyotoTycoonError 16 | from .exceptions import ProtocolError 17 | from .exceptions import ServerConnectionError 18 | from .exceptions import ServerError 19 | -------------------------------------------------------------------------------- /kt/client.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from functools import partial 3 | import json 4 | import re 5 | import socket 6 | import sys 7 | import time 8 | try: 9 | import cPickle as pickle 10 | except ImportError: 11 | import pickle 12 | 13 | try: 14 | import msgpack 15 | except ImportError: 16 | msgpack = None 17 | 18 | from ._binary import KTBinaryProtocol 19 | from ._binary import TTBinaryProtocol 20 | from ._binary import decode 21 | from ._binary import dict_to_table 22 | from ._binary import encode 23 | from ._binary import table_to_dict 24 | from .constants import IOP_DELETE 25 | from .constants import IOP_KEEP 26 | from .constants import IOP_OPTIMIZE 27 | from .constants import ORDER_STR_ASC 28 | from .exceptions import ImproperlyConfigured 29 | from .exceptions import KyotoTycoonError 30 | from .exceptions import ProtocolError 31 | from .exceptions import ServerConnectionError 32 | from .exceptions import ServerError 33 | from .http import HttpProtocol 34 | 35 | 36 | if sys.version_info[0] > 2: 37 | basestring = (bytes, str) 38 | 39 | 40 | KT_BINARY = 'binary' 41 | KT_JSON = 'json' 42 | KT_MSGPACK = 'msgpack' 43 | KT_NONE = 'none' 44 | KT_PICKLE = 'pickle' 45 | TT_TABLE = 'table' 46 | KT_SERIALIZERS = set((KT_BINARY, KT_JSON, KT_MSGPACK, KT_NONE, KT_PICKLE, 47 | TT_TABLE)) 48 | 49 | 50 | class BaseClient(object): 51 | def __init__(self, host='127.0.0.1', port=1978, serializer=KT_BINARY, 52 | decode_keys=True, timeout=None, connection_pool=False): 53 | self._host = host 54 | self._port = port 55 | self._serializer = serializer 56 | self._decode_keys = decode_keys 57 | self._timeout = timeout 58 | self._connection_pool = connection_pool 59 | 60 | if self._serializer == KT_MSGPACK and msgpack is None: 61 | raise ImproperlyConfigured('msgpack library not found') 62 | elif self._serializer == KT_BINARY: 63 | self._encode_value = encode 64 | self._decode_value = decode 65 | elif self._serializer == KT_JSON: 66 | self._encode_value = lambda v: (json 67 | .dumps(v, separators=(',', ':')) 68 | .encode('utf-8')) 69 | self._decode_value = lambda v: json.loads(v.decode('utf-8')) 70 | elif self._serializer == KT_MSGPACK: 71 | self._encode_value = lambda o: msgpack.packb(o, use_bin_type=True) 72 | self._decode_value = lambda b: msgpack.unpackb(b, raw=False) 73 | elif self._serializer == KT_NONE: 74 | self._encode_value = encode 75 | self._decode_value = lambda x: x 76 | elif self._serializer == KT_PICKLE: 77 | self._encode_value = partial(pickle.dumps, 78 | protocol=pickle.HIGHEST_PROTOCOL) 79 | self._decode_value = pickle.loads 80 | elif self._serializer == TT_TABLE: 81 | self._encode_value = dict_to_table 82 | self._decode_value = table_to_dict 83 | else: 84 | raise ImproperlyConfigured('unrecognized serializer "%s" - use one' 85 | ' of: %s' % (self._serializer, 86 | ','.join(KT_SERIALIZERS))) 87 | 88 | # Session and socket used for rpc and binary protocols, respectively. 89 | self._initialize_protocols() 90 | 91 | @property 92 | def lua(self): 93 | if not hasattr(self, '_script_runner'): 94 | self._script_runner = ScriptRunner(self) 95 | return self._script_runner 96 | 97 | def __enter__(self): 98 | return self 99 | 100 | def __exit__(self, exc_type, exc_val, exc_tb): 101 | self._protocol.close() 102 | 103 | def open(self): 104 | return self._protocol.connect() 105 | 106 | def close(self, allow_reuse=True): 107 | return self._protocol.close(allow_reuse) 108 | 109 | def close_all(self): 110 | return self._protocol.close_all() 111 | 112 | def close_idle(self, cutoff=60): 113 | return self._protocol.close_idle(cutoff) 114 | 115 | 116 | class ScriptRunner(object): 117 | def __init__(self, client): 118 | self.client = client 119 | 120 | def __getattr__(self, attr_name): 121 | def run_script(*args, **kwargs): 122 | return self.client._script(attr_name, *args, **kwargs) 123 | return run_script 124 | 125 | 126 | class KyotoTycoon(BaseClient): 127 | def __init__(self, *args, **kwargs): 128 | self._default_db = kwargs.pop('default_db', 0) 129 | super(KyotoTycoon, self).__init__(*args, **kwargs) 130 | 131 | def _initialize_protocols(self): 132 | # Protocol handlers. 133 | self._protocol = KTBinaryProtocol( 134 | host=self._host, 135 | port=self._port, 136 | decode_keys=self._decode_keys, 137 | encode_value=self._encode_value, 138 | decode_value=self._decode_value, 139 | timeout=self._timeout, 140 | connection_pool=self._connection_pool, 141 | default_db=self._default_db) 142 | self._http = HttpProtocol( 143 | host=self._host, 144 | port=self._port, 145 | decode_keys=self._decode_keys, 146 | encode_value=self._encode_value, 147 | decode_value=self._decode_value, 148 | default_db=self._default_db) 149 | 150 | def open(self): 151 | self._http.connect() 152 | return self._protocol.connect() 153 | 154 | def close(self, allow_reuse=True): 155 | self._protocol.close(allow_reuse) 156 | self._http.close() 157 | 158 | def get_bulk(self, keys, db=None, decode_values=True): 159 | return self._protocol.get_bulk(keys, db, decode_values) 160 | 161 | def get_bulk_details(self, keys, db=None, decode_values=True): 162 | return self._protocol.get_bulk_details(keys, db, decode_values) 163 | 164 | def get_bulk_raw(self, db_key_list, decode_values=True): 165 | return self._protocol.get_bulk_raw(db_key_list, decode_values) 166 | 167 | def get_bulk_raw_details(self, db_key_list, decode_values=True): 168 | return self._protocol.get_bulk_raw_details(db_key_list, decode_values) 169 | 170 | def get(self, key, db=None): 171 | return self._protocol.get(key, db, True) 172 | 173 | def get_bytes(self, key, db=None): 174 | return self._protocol.get(key, db, False) 175 | 176 | def set_bulk(self, data, db=None, expire_time=None, no_reply=False, 177 | encode_values=True): 178 | return self._protocol.set_bulk(data, db, expire_time, no_reply, 179 | encode_values) 180 | 181 | def set_bulk_raw(self, data, no_reply=False, encode_values=True): 182 | return self._protocol.set_bulk_raw(data, no_reply, encode_values) 183 | 184 | def set(self, key, value, db=None, expire_time=None, no_reply=False): 185 | return self._protocol.set(key, value, db, expire_time, no_reply, True) 186 | 187 | def set_bytes(self, key, value, db=None, expire_time=None, no_reply=False): 188 | return self._protocol.set(key, value, db, expire_time, no_reply, False) 189 | 190 | def remove_bulk(self, keys, db=None, no_reply=False): 191 | return self._protocol.remove_bulk(keys, db, no_reply) 192 | 193 | def remove_bulk_raw(self, db_key_list, no_reply=False): 194 | return self._protocol.remove_bulk_raw(db_key_list, no_reply) 195 | 196 | def remove(self, key, db=None, no_reply=False): 197 | return self._protocol.remove(key, db, no_reply) 198 | 199 | def _script(self, name, __data=None, no_reply=False, encode_values=True, 200 | decode_values=True, **kwargs): 201 | if __data is None: 202 | __data = kwargs 203 | elif kwargs: 204 | __data.update(kwargs) 205 | return self._protocol.script(name, __data, no_reply, encode_values, 206 | decode_values) 207 | 208 | def script(self, name, data=None, no_reply=False, encode_values=True, 209 | decode_values=True): 210 | return self._protocol.script(name, data, no_reply, encode_values, 211 | decode_values) 212 | 213 | def clear(self, db=None): 214 | return self._http.clear(db) 215 | 216 | def status(self, db=None): 217 | return self._http.status(db) 218 | 219 | def report(self): 220 | return self._http.report() 221 | 222 | def ulog_list(self): 223 | return self._http.ulog_list() 224 | 225 | def ulog_remove(self, max_dt): 226 | return self._http.ulog_remove(max_dt) 227 | 228 | def synchronize(self, hard=False, command=None, db=None): 229 | return self._http.synchronize(hard, command, db) 230 | 231 | def vacuum(self, step=0, db=None): 232 | return self._http.vacuum(step, db) 233 | 234 | def add(self, key, value, db=None, expire_time=None, encode_value=True): 235 | return self._http.add(key, value, db, expire_time, encode_value) 236 | 237 | def replace(self, key, value, db=None, expire_time=None, 238 | encode_value=True): 239 | return self._http.replace(key, value, db, expire_time, encode_value) 240 | 241 | def append(self, key, value, db=None, expire_time=None, encode_value=True): 242 | return self._http.append(key, value, db, expire_time, encode_value) 243 | 244 | def exists(self, key, db=None): 245 | return self._http.check(key, db) 246 | 247 | def length(self, key, db=None): 248 | return self._http.length(key, db) 249 | 250 | def seize(self, key, db=None, decode_value=True): 251 | return self._http.seize(key, db, decode_value) 252 | 253 | def cas(self, key, old_val, new_val, db=None, expire_time=None, 254 | encode_value=True): 255 | return self._http.cas(key, old_val, new_val, db, expire_time, 256 | encode_value) 257 | 258 | def incr(self, key, n=1, orig=None, db=None, expire_time=None): 259 | return self._http.increment(key, n, orig, db, expire_time) 260 | 261 | def incr_double(self, key, n=1., orig=None, db=None, expire_time=None): 262 | return self._http.increment_double(key, n, orig, db, expire_time) 263 | 264 | def _kdb_from_key(self, key): 265 | if isinstance(key, tuple): 266 | if len(key) != 2: 267 | raise ValueError('expected key-tuple of (key, db)') 268 | return key 269 | return (key, None) 270 | 271 | def __getitem__(self, key): 272 | return self.get(*self._kdb_from_key(key)) 273 | 274 | def __setitem__(self, key, value): 275 | key, db = self._kdb_from_key(key) 276 | if isinstance(value, tuple): 277 | if len(value) != 2: 278 | raise ValueError('expected value-tuple of (value, expires)') 279 | value, expire_time = value 280 | else: 281 | expire_time = None 282 | self._protocol.set(key, value, db, expire_time, no_reply=True) 283 | 284 | def __delitem__(self, key): 285 | self.remove(*self._kdb_from_key(key)) 286 | 287 | def update(self, __data=None, **kwargs): 288 | if __data is None: 289 | __data = kwargs 290 | elif kwargs: 291 | __data.update(kwargs) 292 | return self.set_bulk(__data) 293 | 294 | pop = seize 295 | 296 | def __contains__(self, key): 297 | return self.exists(*self._kdb_from_key(key)) 298 | 299 | def __len__(self): 300 | return int(self.status()['count']) 301 | 302 | def count(self, db=None): 303 | return int(self.status(db)['count']) 304 | 305 | def match_prefix(self, prefix, max_keys=None, db=None): 306 | return self._http.match_prefix(prefix, max_keys, db) 307 | 308 | def match_regex(self, regex, max_keys=None, db=None): 309 | return self._http.match_regex(regex, max_keys, db) 310 | 311 | def match_similar(self, origin, distance=None, max_keys=None, db=None): 312 | return self._http.match_similar(origin, distance, max_keys, db) 313 | 314 | def cursor(self, db=None, cursor_id=None): 315 | return self._http.cursor(cursor_id, db) 316 | 317 | def keys(self, db=None): 318 | cursor = self.cursor(db=db) 319 | if not cursor.jump(): return 320 | while True: 321 | key = cursor.key() 322 | if key is None: return 323 | yield key 324 | if not cursor.step(): return 325 | 326 | def keys_nonlazy(self, db=None): 327 | return self.match_prefix('', db=db) 328 | 329 | def values(self, db=None): 330 | cursor = self.cursor(db=db) 331 | if not cursor.jump(): return 332 | while True: 333 | value = cursor.value() 334 | if value is None: return 335 | yield value 336 | if not cursor.step(): return 337 | 338 | def items(self, db=None): 339 | cursor = self.cursor(db=db) 340 | if not cursor.jump(): return 341 | while True: 342 | kv = cursor.get() 343 | if kv is None: return 344 | yield kv 345 | if not cursor.step(): return 346 | 347 | def __iter__(self): 348 | return iter(self.keys()) 349 | 350 | @property 351 | def size(self): 352 | return int(self.status()['size']) 353 | 354 | @property 355 | def path(self): 356 | return decode(self.status()['path']) 357 | 358 | def set_database(self, db): 359 | self._default_database = db 360 | self._protocol.set_database(db) 361 | self._http.set_database(db) 362 | return self 363 | 364 | 365 | class TokyoTyrant(BaseClient): 366 | def _initialize_protocols(self): 367 | self._protocol = TTBinaryProtocol( 368 | host=self._host, 369 | port=self._port, 370 | decode_keys=self._decode_keys, 371 | encode_value=self._encode_value, 372 | decode_value=self._decode_value, 373 | timeout=self._timeout, 374 | connection_pool=self._connection_pool) 375 | 376 | def get_bulk(self, keys, decode_values=True): 377 | return self._protocol.mget(keys, decode_values) 378 | 379 | def get(self, key): 380 | return self._protocol.get(key, True) 381 | 382 | def get_bytes(self, key): 383 | return self._protocol.get(key, False) 384 | 385 | def set_bulk(self, data, no_reply=False, encode_values=True): 386 | if no_reply: 387 | self._protocol.putnr_bulk(data, encode_values) 388 | else: 389 | return self._protocol.misc_putlist(data, True, encode_values) 390 | 391 | def set(self, key, value, no_reply=False): 392 | if no_reply: 393 | self._protocol.putnr(key, value, True) 394 | else: 395 | return self._protocol.put(key, value, True) 396 | 397 | def set_bytes(self, key, value): 398 | if no_reply: 399 | self._protocol.putnr(key, value, False) 400 | else: 401 | return self._protocol.put(key, value, False) 402 | 403 | def remove_bulk(self, keys): 404 | return self._protocol.misc_outlist(keys) 405 | 406 | def remove(self, key): 407 | return self._protocol.out(key) 408 | 409 | def script(self, name, key=None, value=None, lock_records=False, 410 | lock_all=False, encode_value=True, decode_value=False, 411 | as_list=False, as_dict=False, as_int=False): 412 | res = self._protocol.ext(name, key, value, lock_records, lock_all, 413 | encode_value, decode_value) 414 | if as_list or as_dict: 415 | # In the event the return value is an empty string, then we just 416 | # return the empty container type. 417 | if not res: 418 | return {} if as_dict else [] 419 | 420 | # Split on newlines -- dicts are additionally split on tabs. 421 | delim = '\n' if decode_value else b'\n' 422 | res = res.rstrip(delim).split(delim) 423 | if as_dict: 424 | delim = '\t' if decode_value else b'\t' 425 | res = dict([r.split(delim) for r in res]) 426 | elif as_int: 427 | res = int(res) if res else None 428 | return res 429 | _script = script 430 | 431 | def clear(self): 432 | return self._protocol.vanish() 433 | 434 | def status(self): 435 | data = self._protocol.stat() 436 | status = {} 437 | for key_value in data.decode('utf-8').splitlines(): 438 | key, val = key_value.split('\t', 1) 439 | if val.replace('.', '').isdigit(): 440 | try: 441 | val = float(val) if val.find('.') >= 0 else int(val) 442 | except ValueError: 443 | pass 444 | status[key] = val 445 | return status 446 | 447 | def synchronize(self): 448 | return self._protocol.sync() 449 | 450 | def optimize(self, options): 451 | return self._protocol.optimize(options) 452 | 453 | def add(self, key, value, encode_value=True): 454 | return self._protocol.putkeep(key, value, encode_value) 455 | 456 | def append(self, key, value, encode_value=True): 457 | return self._protocol.putcat(key, value, encode_value) 458 | 459 | def addshl(self, key, value, width, encode_value=True): 460 | return self._protocol.putshl(key, value, width, encode_value) 461 | 462 | def exists(self, key): 463 | return self._protocol.vsiz(key) is not None 464 | 465 | def length(self, key): 466 | return self._protocol.vsiz(key) 467 | 468 | def seize(self, key, decode_value=True): 469 | return self._protocol.seize(key, decode_value) 470 | 471 | def incr(self, key, n=1): 472 | return self._protocol.addint(key, n) 473 | 474 | def incr_double(self, key, n=1.): 475 | return self._protocol.adddouble(key, n) 476 | 477 | def count(self): 478 | return self._protocol.rnum() 479 | 480 | def __getitem__(self, item): 481 | if isinstance(item, slice): 482 | return self.get_range(item.start, item.stop or None) 483 | else: 484 | return self.get(item) 485 | 486 | def __setitem__(self, key, value): 487 | self._protocol.putnr(key, value, True) 488 | 489 | __delitem__ = remove 490 | __contains__ = exists 491 | __len__ = count 492 | pop = seize 493 | 494 | def update(self, __data=None, no_reply=False, encode_values=True, **kw): 495 | if __data is None: 496 | __data = kw 497 | elif kw: 498 | __data.update(kw) 499 | return self.set_bulk(__data, no_reply, encode_values) 500 | 501 | def setdup(self, key, value, encode_value=True): 502 | return self._protocol.misc_putdup(key, value, True, encode_value) 503 | 504 | def setdupback(self, key, value, encode_value=True): 505 | return self._protocol.misc_putdupback(key, value, True, encode_value) 506 | 507 | def get_part(self, key, start=None, end=None, decode_value=True): 508 | return self._protocol.misc_getpart(key, start or 0, end, decode_value) 509 | 510 | def misc(self, cmd, args=None, update_log=True, decode_values=False): 511 | ok, data = self._protocol.misc(cmd, args, update_log, decode_values) 512 | if ok: 513 | return data 514 | 515 | @property 516 | def size(self): 517 | return self._protocol.size() 518 | 519 | @property 520 | def error(self): 521 | error_str = self._protocol.misc_error() 522 | if error_str is not None: 523 | code, msg = error_str.split(': ', 1) 524 | return int(code), msg 525 | 526 | def copy(self, path): 527 | return self._protocol.copy(path) 528 | 529 | def _datetime_to_timestamp(self, dt): 530 | timestamp = time.mktime(dt.timetuple()) 531 | timestamp *= 1000000 532 | return int(timestamp + dt.microsecond) 533 | 534 | def restore(self, path, timestamp, options=0): 535 | if isinstance(timestamp, datetime.datetime): 536 | timestamp = self._datetime_to_timestamp(timestamp) 537 | return self._protocol.restore(path, timestamp, options) 538 | 539 | def set_master(self, host, port, timestamp, options=0): 540 | if isinstance(timestamp, datetime.datetime): 541 | timestamp = self._datetime_to_timestamp(timestamp) 542 | return self._protocol.setmst(host, port, timestamp, options) 543 | 544 | def clear_cache(self): 545 | return self._protocol.misc_cacheclear() 546 | 547 | def defragment(self, nsteps=None): 548 | return self._protocol.misc_defragment(nsteps) 549 | 550 | def get_range(self, start, stop=None, max_keys=0, decode_values=True): 551 | return self._protocol.misc_range(start, stop, max_keys, decode_values) 552 | 553 | def get_rangelist(self, start, stop=None, max_keys=0, decode_values=True): 554 | return self._protocol.misc_rangelist(start, stop, max_keys, 555 | decode_values) 556 | 557 | def match_prefix(self, prefix, max_keys=None): 558 | return self._protocol.fwmkeys(prefix, max_keys) 559 | 560 | def match_regex(self, regex, max_keys=None, decode_values=True): 561 | return self._protocol.misc_regex(regex, max_keys, decode_values) 562 | 563 | def match_regexlist(self, regex, max_keys=None, decode_values=True): 564 | return self._protocol.misc_regexlist(regex, max_keys, decode_values) 565 | 566 | def iter_from(self, start_key): 567 | return self._protocol.items(start_key) 568 | 569 | def keys(self): 570 | return self._protocol.keys() 571 | 572 | def keys_fast(self): 573 | return self._protocol.fwmkeys('') 574 | 575 | def items(self, start_key=None): 576 | return self._protocol.items(start_key) 577 | 578 | def items_fast(self): 579 | return self._protocol.misc_rangelist('') 580 | 581 | def set_index(self, name, index_type, check_exists=False): 582 | if check_exists: 583 | index_type |= IOP_KEEP 584 | return self._protocol.misc_setindex(name, index_type) 585 | 586 | def optimize_index(self, name): 587 | return self._protocol.misc_setindex(name, IOP_OPTIMIZE) 588 | 589 | def delete_index(self, name): 590 | return self._protocol.misc_setindex(name, IOP_DELETE) 591 | 592 | def search(self, expressions, cmd=None): 593 | conditions = [_pack_misc_cmd(*expr) for expr in expressions] 594 | return self._protocol.misc_search(conditions, cmd) 595 | 596 | def genuid(self): 597 | return self._protocol.misc_genuid() 598 | 599 | def __iter__(self): 600 | return iter(self._protocol.keys()) 601 | 602 | 603 | def _pack_misc_cmd(*args): 604 | message = [encode(str(arg) if not isinstance(arg, basestring) else arg) 605 | for arg in args] 606 | return b'\x00'.join(message) 607 | 608 | 609 | def clone_query(method): 610 | def inner(self, *args, **kwargs): 611 | clone = self.clone() 612 | method(clone, *args, **kwargs) 613 | return clone 614 | return inner 615 | 616 | 617 | class QueryBuilder(object): 618 | def __init__(self): 619 | self._conditions = [] 620 | self._order_by = [] 621 | self._limit = None 622 | self._offset = None 623 | 624 | def clone(self): 625 | obj = QueryBuilder() 626 | obj._conditions = list(self._conditions) 627 | obj._order_by = list(self._order_by) 628 | obj._limit = self._limit 629 | obj._offset = self._offset 630 | return obj 631 | 632 | @clone_query 633 | def filter(self, column, op, value): 634 | self._conditions.append((column, op, value)) 635 | 636 | @clone_query 637 | def order_by(self, column, ordering=None): 638 | self._order_by.append((column, ordering or ORDER_STR_ASC)) 639 | 640 | @clone_query 641 | def limit(self, limit=None): 642 | self._limit = limit 643 | 644 | @clone_query 645 | def offset(self, offset=None): 646 | self._offset = offset 647 | 648 | def build_search(self): 649 | cmd = [('addcond', col, op, val) for col, op, val in self._conditions] 650 | for col, order in self._order_by: 651 | cmd.append(('setorder', col, order)) 652 | if self._limit is not None or self._offset is not None: 653 | cmd.append(('setlimit', self._limit or 1 << 31, self._offset or 0)) 654 | return cmd 655 | 656 | def execute(self, client): 657 | return client.search(self.build_search()) 658 | 659 | def delete(self, client): 660 | return client.search(self.build_search(), 'out') 661 | 662 | def get(self, client): 663 | results = client.search(self.build_search(), 'get') 664 | accum = [] 665 | for key, raw_data in results: 666 | accum.append((key, table_to_dict(raw_data))) 667 | return accum 668 | 669 | def count(self, client): 670 | return client.search(self.build_search(), 'count') 671 | -------------------------------------------------------------------------------- /kt/constants.py: -------------------------------------------------------------------------------- 1 | OP_STR_EQ = 0 2 | OP_STR_CONTAINS = 1 3 | OP_STR_STARTSWITH = 2 4 | OP_STR_ENDSWITH = 3 5 | OP_STR_ALL = 4 6 | OP_STR_ANY = 5 7 | OP_STR_ANYEXACT = 6 8 | OP_STR_REGEX = 7 9 | OP_NUM_EQ = 8 10 | OP_NUM_GT = 9 11 | OP_NUM_GE = 10 12 | OP_NUM_LT = 11 13 | OP_NUM_LE = 12 14 | OP_NUM_BETWEEN = 13 15 | OP_NUM_ANYEXACT = 14 16 | OP_FTS_PHRASE = 15 17 | OP_FTS_ALL = 16 18 | OP_FTS_ANY = 17 19 | OP_FTS_EXPRESSION = 18 20 | 21 | OP_NEGATE = 1 << 24 22 | OP_NOINDEX = 1 << 25 23 | 24 | ORDER_STR_ASC = 0 25 | ORDER_STR_DESC = 1 26 | ORDER_NUM_ASC = 2 27 | ORDER_NUM_DESC = 3 28 | 29 | INDEX_STR = 0 30 | INDEX_NUM = 1 31 | INDEX_TOKEN = 2 32 | INDEX_QGRAM = 3 33 | IOP_OPTIMIZE = 9998 34 | IOP_DELETE = 9999 35 | IOP_KEEP = 1 << 24 36 | -------------------------------------------------------------------------------- /kt/embedded.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | import logging 3 | import random 4 | import socket 5 | import subprocess 6 | import sys 7 | import threading 8 | import time 9 | 10 | from .client import KT_BINARY 11 | from .client import KyotoTycoon 12 | from .client import TokyoTyrant 13 | from .exceptions import KyotoTycoonError 14 | 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class EmbeddedServer(object): 20 | def __init__(self, server='ktserver', host='127.0.0.1', port=None, 21 | database='*', serializer=None, server_args=None, quiet=False, 22 | connection_pool=False): 23 | self._server = server 24 | self._host = host 25 | self._port = port 26 | self._serializer = serializer or KT_BINARY 27 | self._database = database 28 | self._server_args = server_args or [] 29 | self._quiet = quiet 30 | self._connection_pool = connection_pool 31 | 32 | # Signals for server startup and shutdown. 33 | self._server_started = threading.Event() 34 | self._server_terminated = threading.Event() 35 | self._server_terminated.set() # Start off in terminated state. 36 | 37 | # Placeholders for server process and client. 38 | self._server_p = None 39 | self._client = None 40 | 41 | def _create_client(self): 42 | return KyotoTycoon(self._host, self._port, self._serializer, 43 | connection_pool=self._connection_pool) 44 | 45 | @property 46 | def client(self): 47 | if self._server_terminated.is_set(): 48 | raise KyotoTycoonError('server not running') 49 | elif self._client is None: 50 | self._client = self._create_client() 51 | return self._client 52 | 53 | @property 54 | def pid(self): 55 | if not self._server_terminated.is_set(): 56 | return self._server_p.pid 57 | 58 | def _run_server(self, port): 59 | command = [ 60 | self._server, 61 | '-le', # Log errors. 62 | '-host', 63 | self._host, 64 | '-port', 65 | str(port)] + self._server_args + [self._database] 66 | 67 | while not self._server_terminated.is_set(): 68 | if self._quiet: 69 | out, err = subprocess.PIPE, subprocess.PIPE 70 | else: 71 | out, err = sys.__stdout__.fileno(), sys.__stderr__.fileno() 72 | self._server_p = subprocess.Popen(command, stderr=err, stdout=out) 73 | 74 | self._server_started.set() 75 | self._server_p.wait() 76 | self._client = None 77 | 78 | time.sleep(0.1) 79 | if not self._server_terminated.is_set(): 80 | logger.error('server process died, restarting...') 81 | 82 | logger.info('server shutdown') 83 | 84 | def _stop_server(self): 85 | self._server_terminated.set() 86 | self._server_p.terminate() 87 | self._server_p.wait() 88 | self._server_p = self._client = None 89 | 90 | def run(self): 91 | """ 92 | Run ktserver on a random high port and return a client connected to it. 93 | """ 94 | if not self._server_terminated.is_set(): 95 | logger.warning('server already running') 96 | return False 97 | 98 | if self._port is None: 99 | self._port = self._find_open_port() 100 | 101 | self._server_started.clear() 102 | self._server_terminated.clear() 103 | 104 | t = threading.Thread(target=self._run_server, args=(self._port,)) 105 | t.daemon = True 106 | t.start() 107 | 108 | self._server_started.wait() # Wait for server to start up. 109 | atexit.register(self._stop_server) 110 | 111 | attempts = 0 112 | while attempts < 20: 113 | attempts += 1 114 | try: 115 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 116 | s.connect((self._host, self._port)) 117 | s.close() 118 | return True 119 | except socket.error: 120 | time.sleep(0.1) 121 | except OSError: 122 | time.sleep(0.1) 123 | 124 | self._stop_server() 125 | raise KyotoTycoonError('Unable to connect to server on %s:%s' % 126 | (self._host, self._port)) 127 | 128 | def stop(self): 129 | if self._server_terminated.is_set(): 130 | logger.warning('server already stopped') 131 | return False 132 | 133 | if hasattr(atexit, 'unregister'): 134 | atexit.unregister(self._stop_server) 135 | else: 136 | funcs = [] 137 | for fn, arg, kw in atexit._exithandlers: 138 | if fn != self._stop_server: 139 | funcs.append((fn, arg, kw)) 140 | atexit._exithandlers = funcs 141 | self._stop_server() 142 | 143 | def _find_open_port(self): 144 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 145 | attempts = 0 146 | while attempts < 32: 147 | attempts += 1 148 | port = random.randint(16000, 32000) 149 | try: 150 | sock.bind(('127.0.0.1', port)) 151 | sock.listen(1) 152 | sock.close() 153 | time.sleep(0.1) 154 | return port 155 | except OSError: 156 | pass 157 | 158 | raise KyotoTycoonError('Could not find open port') 159 | 160 | 161 | class EmbeddedTokyoTyrantServer(EmbeddedServer): 162 | def __init__(self, server='ttserver', host='127.0.0.1', port=None, 163 | database='*', serializer=None, server_args=None, quiet=False, 164 | connection_pool=False): 165 | super(EmbeddedTokyoTyrantServer, self).__init__( 166 | server, host, port, database, serializer, server_args, quiet, 167 | connection_pool) 168 | 169 | def _create_client(self): 170 | return TokyoTyrant(self._host, self._port, self._serializer, 171 | connection_pool=self._connection_pool) 172 | -------------------------------------------------------------------------------- /kt/exceptions.py: -------------------------------------------------------------------------------- 1 | class KyotoTycoonError(Exception): pass 2 | class ImproperlyConfigured(KyotoTycoonError): pass 3 | class ProtocolError(KyotoTycoonError): pass 4 | class ScriptError(KyotoTycoonError): pass 5 | class ServerConnectionError(KyotoTycoonError): pass 6 | class ServerError(KyotoTycoonError): pass 7 | -------------------------------------------------------------------------------- /kt/http.py: -------------------------------------------------------------------------------- 1 | from base64 import b64decode 2 | from base64 import b64encode 3 | from functools import partial 4 | import datetime 5 | import sys 6 | try: 7 | from http.client import HTTPConnection 8 | from urllib.parse import quote_from_bytes 9 | from urllib.parse import unquote_to_bytes 10 | from urllib.parse import urlencode 11 | except ImportError: 12 | from httplib import HTTPConnection 13 | from urllib import quote as quote_from_bytes 14 | from urllib import unquote as unquote_to_bytes 15 | from urllib import urlencode 16 | 17 | from ._binary import decode 18 | from ._binary import encode 19 | from .exceptions import ProtocolError 20 | from .exceptions import ServerError 21 | 22 | 23 | IS_PY2 = sys.version_info[0] == 2 24 | 25 | if not IS_PY2: 26 | unicode = str 27 | 28 | 29 | quote_b = partial(quote_from_bytes, safe='') 30 | unquote_b = partial(unquote_to_bytes) 31 | 32 | 33 | def decode_from_content_type(content_type): 34 | if content_type.endswith('colenc=B'): 35 | return b64decode 36 | elif content_type.endswith('colenc=U'): 37 | return unquote_b 38 | 39 | 40 | class HttpProtocol(object): 41 | _content_type = 'text/tab-separated-values; colenc=B' 42 | cursor_id = 0 43 | 44 | def __init__(self, host='127.0.0.1', port=1978, decode_keys=True, 45 | encode_value=None, decode_value=None, timeout=None, 46 | default_db=None): 47 | self._host = host 48 | self._port = port 49 | self._timeout = timeout 50 | self._decode_keys = decode_keys 51 | self.encode_value = encode_value or encode 52 | self.decode_value = decode_value or decode 53 | self.default_db = default_db or 0 54 | self._prefix = '/rpc' 55 | self._conn = self._get_conn() 56 | self._headers = {'Content-Type': self._content_type} 57 | 58 | def set_database(self, db): 59 | self.default_db = db 60 | 61 | def _get_conn(self): 62 | return HTTPConnection(self._host, self._port, timeout=self._timeout) 63 | 64 | def connect(self): 65 | self.close() 66 | self._conn = self._get_conn() 67 | self._conn.connect() 68 | return True 69 | 70 | def close(self): 71 | self._conn.close() 72 | 73 | def __del__(self): 74 | self._conn.close() 75 | 76 | def _encode_keys_values(self, data): 77 | accum = [] 78 | for key, value in data.items(): 79 | bkey = encode(key) 80 | bvalue = encode(value) 81 | accum.append(b'%s\t%s' % (b64encode(bkey), b64encode(bvalue))) 82 | return b'\n'.join(accum) 83 | 84 | def _encode_keys(self, keys): 85 | accum = [] 86 | for key in keys: 87 | accum.append(b'%s\t' % b64encode(b'_' + encode(key))) 88 | return b'\n'.join(accum) 89 | 90 | def _decode_response(self, tsv, content_type, decode_keys=None): 91 | if decode_keys is None: 92 | decode_keys = self._decode_keys 93 | decoder = decode_from_content_type(content_type) 94 | accum = {} 95 | for line in tsv.split(b'\n'): 96 | try: 97 | key, value = line.split(b'\t', 1) 98 | except ValueError: 99 | continue 100 | 101 | if decoder is not None: 102 | key, value = decoder(key), decoder(value) 103 | 104 | if decode_keys: 105 | key = decode(key) 106 | accum[key] = value 107 | 108 | return accum 109 | 110 | def _post(self, path, body): 111 | self._conn.request('POST', self._prefix + path, body, self._headers) 112 | return self._conn.getresponse() 113 | 114 | def request(self, path, data, db=None, allowed_status=None, atomic=False, 115 | decode_keys=None): 116 | if isinstance(data, dict): 117 | body = self._encode_keys_values(data) 118 | elif isinstance(data, list): 119 | body = self._encode_keys(data) 120 | else: 121 | body = data 122 | 123 | prefix = {} 124 | if db is not False: 125 | prefix['DB'] = self.default_db if db is None else db 126 | if atomic: 127 | prefix['atomic'] = '' 128 | 129 | if prefix: 130 | db_data = self._encode_keys_values(prefix) 131 | if body: 132 | body = b'\n'.join((db_data, body)) 133 | else: 134 | body = db_data 135 | 136 | try: 137 | r = self._post(path, body) 138 | except Exception as exc: 139 | self.close() 140 | raise 141 | 142 | content = r.read() 143 | content_type = r.getheader('content-type') 144 | status = r.status 145 | 146 | if status != 200: 147 | if allowed_status is None or status not in allowed_status: 148 | raise ProtocolError('protocol error [%s]' % status) 149 | 150 | data = self._decode_response(content, content_type, decode_keys) 151 | return data, status 152 | 153 | def report(self): 154 | resp, status = self.request('/report', {}, None) 155 | return resp 156 | 157 | def script(self, name, data=None, encode_values=True, decode_values=True): 158 | accum = {} 159 | if data is not None: 160 | for key, value in data.items(): 161 | if encode_values: 162 | value = self.encode_value(value) 163 | accum['_%s' % key] = value 164 | 165 | resp, status = self.request('/play_script', accum, False, (450,)) 166 | if status == 450: 167 | return 168 | 169 | accum = {} 170 | for key, value in resp.items(): 171 | if decode_values: 172 | value = self.decode_value(value) 173 | accum[key[1:]] = value 174 | return accum 175 | 176 | def tune_replication(self, host=None, port=None, timestamp=None, 177 | interval=None): 178 | data = {} 179 | if host is not None: 180 | data['host'] = host 181 | if port is not None: 182 | data['port'] = str(port) 183 | if timestamp is not None: 184 | data['ts'] = str(timestamp) 185 | if interval is not None: 186 | data['iv'] = str(interval) 187 | resp, status = self.request('/tune_replication', data, None) 188 | return status == 200 189 | 190 | def status(self, db=None): 191 | resp, status = self.request('/status', {}, db, decode_keys=True) 192 | return resp 193 | 194 | def clear(self, db=None): 195 | resp, status = self.request('/clear', {}, db) 196 | return status == 200 197 | 198 | def synchronize(self, hard=False, command=None, db=None): 199 | data = {} 200 | if hard: 201 | data['hard'] = '' 202 | if command is not None: 203 | data['command'] = command 204 | _, status = self.request('/synchronize', data, db) 205 | return status == 200 206 | 207 | def _simple_write(self, cmd, key, value, db=None, expire_time=None, 208 | encode_value=True): 209 | if encode_value: 210 | value = self.encode_value(value) 211 | data = {'key': key, 'value': value} 212 | if expire_time is not None: 213 | data['xt'] = str(expire_time) 214 | resp, status = self.request('/%s' % cmd, data, db, (450,)) 215 | return status != 450 216 | 217 | def set(self, key, value, db=None, expire_time=None, encode_value=True): 218 | return self._simple_write('set', key, value, db, expire_time, 219 | encode_value) 220 | 221 | def add(self, key, value, db=None, expire_time=None, encode_value=True): 222 | return self._simple_write('add', key, value, db, expire_time, 223 | encode_value) 224 | 225 | def replace(self, key, value, db=None, expire_time=None, 226 | encode_value=True): 227 | return self._simple_write('replace', key, value, db, expire_time, 228 | encode_value) 229 | 230 | def append(self, key, value, db=None, expire_time=None, encode_value=True): 231 | return self._simple_write('append', key, value, db, expire_time, 232 | encode_value) 233 | 234 | def increment(self, key, n=1, orig=None, db=None, expire_time=None): 235 | data = {'key': key, 'num': str(n)} 236 | if orig is not None: 237 | data['orig'] = str(orig) 238 | if expire_time is not None: 239 | data['xt'] = str(expire_time) 240 | resp, status = self.request('/increment', data, db, decode_keys=False) 241 | return int(resp[b'num']) 242 | 243 | def increment_double(self, key, n=1, orig=None, db=None, expire_time=None): 244 | data = {'key': key, 'num': str(n)} 245 | if orig is not None: 246 | data['orig'] = str(orig) 247 | if expire_time is not None: 248 | data['xt'] = str(expire_time) 249 | resp, status = self.request('/increment_double', data, db, 250 | decode_keys=False) 251 | return float(resp[b'num']) 252 | 253 | def cas(self, key, old_val, new_val, db=None, expire_time=None, 254 | encode_value=True): 255 | if old_val is None and new_val is None: 256 | raise ValueError('old value and/or new value must be specified.') 257 | 258 | data = {'key': key} 259 | if old_val is not None: 260 | if encode_value: 261 | old_val = self.encode_value(old_val) 262 | data['oval'] = old_val 263 | if new_val is not None: 264 | if encode_value: 265 | new_val = self.encode_value(new_val) 266 | data['nval'] = new_val 267 | if expire_time is not None: 268 | data['xt'] = str(expire_time) 269 | 270 | resp, status = self.request('/cas', data, db, (450,)) 271 | return status != 450 272 | 273 | def remove(self, key, db=None): 274 | resp, status = self.request('/remove', {'key': key}, db, (450,)) 275 | return status != 450 276 | 277 | def get(self, key, db=None, decode_value=True): 278 | resp, status = self.request('/get', {'key': key}, db, (450,), 279 | decode_keys=False) 280 | if status == 450: 281 | return 282 | value = resp[b'value'] 283 | if decode_value: 284 | value = self.decode_value(value) 285 | return value 286 | 287 | def check(self, key, db=None): 288 | resp, status = self.request('/check', {'key': key}, db, (450,)) 289 | return status != 450 290 | 291 | def length(self, key, db=None): 292 | resp, status = self.request('/check', {'key': key}, db, (450,), 293 | decode_keys=False) 294 | if status == 200: 295 | return int(resp[b'vsiz']) 296 | 297 | def seize(self, key, db=None, decode_value=True): 298 | resp, status = self.request('/seize', {'key': key}, db, (450,), 299 | decode_keys=False) 300 | if status == 450: 301 | return 302 | value = resp[b'value'] 303 | if decode_value: 304 | value = self.decode_value(value) 305 | return value 306 | 307 | def set_bulk(self, data, db=None, expire_time=None, atomic=True, 308 | encode_values=True): 309 | accum = {} 310 | if expire_time is not None: 311 | accum['xt'] = str(expire_time) 312 | 313 | # Keys must be prefixed by "_". 314 | for key, value in data.items(): 315 | if encode_values: 316 | value = self.encode_value(value) 317 | accum['_%s' % key] = value 318 | 319 | resp, status = self.request('/set_bulk', accum, db, atomic=atomic, 320 | decode_keys=False) 321 | return int(resp[b'num']) 322 | 323 | def remove_bulk(self, keys, db=None, atomic=True): 324 | resp, status = self.request('/remove_bulk', keys, db, atomic=atomic, 325 | decode_keys=False) 326 | return int(resp[b'num']) 327 | 328 | def _do_bulk_command(self, cmd, params, db=None, decode_values=True, **kw): 329 | resp, status = self.request(cmd, params, db, **kw) 330 | 331 | n = resp.pop('num' if self._decode_keys else b'num') 332 | if n == b'0': 333 | return {} 334 | 335 | accum = {} 336 | for key, value in resp.items(): 337 | if decode_values: 338 | value = self.decode_value(value) 339 | accum[key[1:]] = value 340 | return accum 341 | 342 | def get_bulk(self, keys, db=None, atomic=True, decode_values=True): 343 | return self._do_bulk_command('/get_bulk', keys, db, atomic=atomic, 344 | decode_values=decode_values) 345 | 346 | def vacuum(self, step=0, db=None): 347 | # If step > 0, the whole region is scanned. 348 | data = {'step': str(step)} if step > 0 else {} 349 | resp, status = self.request('/vacuum', data, db) 350 | return status == 200 351 | 352 | def _do_bulk_sorted_command(self, cmd, params, db=None): 353 | results = self._do_bulk_command(cmd, params, db, decode_values=False) 354 | return sorted(results, key=lambda k: int(results[k])) 355 | 356 | def match_prefix(self, prefix, max_keys=None, db=None): 357 | data = {'prefix': prefix} 358 | if max_keys is not None: 359 | data['max'] = str(max_keys) 360 | return self._do_bulk_sorted_command('/match_prefix', data, db) 361 | 362 | def match_regex(self, regex, max_keys=None, db=None): 363 | data = {'regex': regex} 364 | if max_keys is not None: 365 | data['max'] = str(max_keys) 366 | return self._do_bulk_sorted_command('/match_regex', data, db) 367 | 368 | def match_similar(self, origin, distance=None, max_keys=None, db=None): 369 | data = {'origin': origin, 'utf': 'true'} 370 | if distance is not None: 371 | data['range'] = str(distance) 372 | if max_keys is not None: 373 | data['max'] = str(max_keys) 374 | return self._do_bulk_sorted_command('/match_similar', data, db) 375 | 376 | def _cursor_command(self, cmd, cursor_id, data, db=None): 377 | data['CUR'] = cursor_id 378 | resp, status = self.request('/%s' % cmd, data, db, (450, 501), 379 | decode_keys=False) 380 | if status == 501: 381 | raise NotImplementedError('%s is not supported' % cmd) 382 | return resp, status 383 | 384 | def cur_jump(self, cursor_id, key=None, db=None): 385 | data = {'key': key} if key else {} 386 | resp, s = self._cursor_command('cur_jump', cursor_id, data, db) 387 | return s == 200 388 | 389 | def cur_jump_back(self, cursor_id, key=None, db=None): 390 | data = {'key': key} if key else {} 391 | resp, s = self._cursor_command('cur_jump_back', cursor_id, data, db) 392 | return s == 200 393 | 394 | def cur_step(self, cursor_id): 395 | resp, status = self._cursor_command('cur_step', cursor_id, {}) 396 | return status == 200 397 | 398 | def cur_step_back(self, cursor_id): 399 | resp, status = self._cursor_command('cur_step_back', cursor_id, {}) 400 | return status == 200 401 | 402 | def cur_set_value(self, cursor_id, value, step=False, expire_time=None, 403 | encode_value=True): 404 | if encode_value: 405 | value = self.encode_value(value) 406 | data = {'value': value} 407 | if expire_time is not None: 408 | data['xt'] = str(expire_time) 409 | if step: 410 | data['step'] = '' 411 | resp, status = self._cursor_command('cur_set_value', cursor_id, data) 412 | return status == 200 413 | 414 | def cur_remove(self, cursor_id): 415 | resp, status = self._cursor_command('cur_remove', cursor_id, {}) 416 | return status == 200 417 | 418 | def cur_get_key(self, cursor_id, step=False): 419 | data = {'step': ''} if step else {} 420 | resp, status = self._cursor_command('cur_get_key', cursor_id, data) 421 | if status == 450: 422 | return 423 | key = resp[b'key'] 424 | if self._decode_keys: 425 | key = decode(key) 426 | return key 427 | 428 | def cur_get_value(self, cursor_id, step=False, decode_value=True): 429 | data = {'step': ''} if step else {} 430 | resp, status = self._cursor_command('cur_get_value', cursor_id, data) 431 | if status == 450: 432 | return 433 | value = resp[b'value'] 434 | if decode_value: 435 | value = self.decode_value(value) 436 | return value 437 | 438 | def cur_get(self, cursor_id, step=False, decode_value=True): 439 | data = {'step': ''} if step else {} 440 | resp, status = self._cursor_command('cur_get', cursor_id, data) 441 | if status == 450: 442 | return 443 | key = resp[b'key'] 444 | if self._decode_keys: 445 | key = decode(key) 446 | value = resp[b'value'] 447 | if decode_value: 448 | value = self.decode_value(value) 449 | return (key, value) 450 | 451 | def cur_seize(self, cursor_id, step=False, decode_value=True): 452 | resp, status = self._cursor_command('cur_seize', cursor_id, {}) 453 | if status == 450: 454 | return 455 | key = resp[b'key'] 456 | if self._decode_keys: 457 | key = decode(key) 458 | value = resp[b'value'] 459 | if decode_value: 460 | value = self.decode_value(value) 461 | return (key, value) 462 | 463 | def cur_delete(self, cursor_id): 464 | resp, status = self._cursor_command('cur_delete', cursor_id, {}) 465 | return status == 200 466 | 467 | def cursor(self, cursor_id=None, db=None): 468 | if cursor_id is None: 469 | HttpProtocol.cursor_id += 1 470 | cursor_id = HttpProtocol.cursor_id 471 | return Cursor(self, cursor_id, db) 472 | 473 | def ulog_list(self): 474 | resp, status = self.request('/ulog_list', {}, None, decode_keys=True) 475 | log_list = [] 476 | for filename, meta in resp.items(): 477 | size, ts_str = meta.decode('utf-8').split(':') 478 | ts = datetime.datetime.fromtimestamp(int(ts_str) / 1e9) 479 | log_list.append((filename, size, ts)) 480 | return log_list 481 | 482 | def ulog_remove(self, max_dt=None): 483 | max_dt = max_dt or datetime.datetime.now() 484 | data = {'ts': str(int(max_dt.timestamp() * 1e9))} 485 | resp, status = self.request('/ulog_remove', data, None) 486 | return status == 200 487 | 488 | def count(self, db=None): 489 | resp = self.status(db) 490 | return int(resp.get('count') or 0) 491 | 492 | def size(self, db=None): 493 | resp = self.status(db) 494 | return int(resp.get('size') or 0) 495 | 496 | 497 | class Cursor(object): 498 | def __init__(self, protocol, cursor_id, db=None): 499 | self.protocol = protocol 500 | self.cursor_id = cursor_id 501 | self.db = db 502 | self._valid = False 503 | 504 | def __iter__(self): 505 | if not self._valid: 506 | self.jump() 507 | return self 508 | 509 | def is_valid(self): 510 | return self._valid 511 | 512 | def jump(self, key=None): 513 | self._valid = self.protocol.cur_jump(self.cursor_id, key, self.db) 514 | return self._valid 515 | 516 | def jump_back(self, key=None): 517 | self._valid = self.protocol.cur_jump_back(self.cursor_id, key, self.db) 518 | return self._valid 519 | 520 | def step(self): 521 | self._valid = self.protocol.cur_step(self.cursor_id) 522 | return self._valid 523 | 524 | def step_back(self): 525 | self._valid = self.protocol.cur_step_back(self.cursor_id) 526 | return self._valid 527 | 528 | def key(self): 529 | if self._valid: 530 | return self.protocol.cur_get_key(self.cursor_id) 531 | 532 | def value(self): 533 | if self._valid: 534 | return self.protocol.cur_get_value(self.cursor_id) 535 | 536 | def get(self): 537 | if self._valid: 538 | return self.protocol.cur_get(self.cursor_id) 539 | 540 | def set_value(self, value): 541 | if self._valid: 542 | if not self.protocol.cur_set_value(self.cursor_id, value): 543 | self._valid = False 544 | return self._valid 545 | 546 | def remove(self): 547 | if self._valid: 548 | if not self.protocol.cur_remove(self.cursor_id): 549 | self._valid = False 550 | return self._valid 551 | 552 | def seize(self): 553 | if self._valid: 554 | kv = self.protocol.cur_seize(self.cursor_id) 555 | if kv is None: 556 | self._valid = False 557 | return kv 558 | 559 | def close(self): 560 | if self._valid and self.protocol.cur_delete(self.cursor_id): 561 | self._valid = False 562 | return True 563 | return False 564 | 565 | def __next__(self): 566 | if not self._valid: 567 | raise StopIteration 568 | kv = self.protocol.cur_get(self.cursor_id) 569 | if kv is None: 570 | self._valid = False 571 | raise StopIteration 572 | elif not self.step(): 573 | self._valid = False 574 | return kv 575 | next = __next__ 576 | -------------------------------------------------------------------------------- /kt/models.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | from copy import deepcopy 3 | import calendar 4 | import datetime 5 | import io 6 | import sys 7 | import time 8 | 9 | from kt import constants as C 10 | from kt.client import KT_NONE 11 | from kt.client import QueryBuilder 12 | from kt.client import TokyoTyrant 13 | from kt.client import decode 14 | from kt.client import encode 15 | 16 | 17 | if sys.version_info[0] == 2: 18 | string_type = (str, unicode) 19 | else: 20 | string_type = str 21 | 22 | 23 | class TableDatabase(TokyoTyrant): 24 | def __init__(self, host='127.0.0.1', port=1978, decode_keys=True): 25 | # Initialize with no value serialization / deserialization -- that will 26 | # be handled in the model layer. 27 | super(TableDatabase, self).__init__(host, port, KT_NONE, decode_keys) 28 | 29 | 30 | Expression = namedtuple('Expression', ('lhs', 'op', 'rhs')) 31 | Ordering = namedtuple('Ordering', ('field', 'value')) 32 | 33 | 34 | class Field(object): 35 | _index_type = C.INDEX_STR 36 | _order_asc = C.ORDER_STR_ASC 37 | _order_desc = C.ORDER_STR_DESC 38 | 39 | def __init__(self, index=False, default=None): 40 | self._index = index 41 | self._default = default 42 | self.name = None 43 | 44 | def deserialize(self, raw_value): 45 | return raw_value 46 | 47 | def serialize(self, value): 48 | return value 49 | 50 | def asc(self): 51 | return Ordering(self, self._order_asc) 52 | 53 | def desc(self): 54 | return Ordering(self, self._order_desc) 55 | 56 | def add_to_class(self, model, name): 57 | self.model = model 58 | self.name = name 59 | setattr(model, name, self) 60 | 61 | def __get__(self, instance, instance_type=None): 62 | if instance is not None: 63 | return instance.__data__.get(self.name) 64 | return self 65 | 66 | def __set__(self, instance, value): 67 | instance.__data__[self.name] = value 68 | 69 | 70 | def _e(op): 71 | def inner(self, rhs): 72 | return Expression(self, op, rhs) 73 | return inner 74 | def _me(op): 75 | def inner(self, *rhs): 76 | return Expression(self, op, rhs) 77 | return inner 78 | 79 | 80 | class BytesField(Field): 81 | __eq__ = _e(C.OP_STR_EQ) 82 | __ne__ = _e(C.OP_STR_EQ | C.OP_NEGATE) 83 | contains = _e(C.OP_STR_CONTAINS) 84 | startswith = _e(C.OP_STR_STARTSWITH) 85 | endswith = _e(C.OP_STR_ENDSWITH) 86 | contains_all = _me(C.OP_STR_ALL) 87 | contains_any = _me(C.OP_STR_ANY) 88 | contains_any_exact = _me(C.OP_STR_ANYEXACT) 89 | regex = _e(C.OP_STR_REGEX) 90 | 91 | 92 | class TextField(BytesField): 93 | deserialize = decode 94 | serialize = encode 95 | 96 | 97 | class DateTimeField(BytesField): 98 | _format = '%Y-%m-%dT%H:%M:%S.%f' 99 | 100 | def deserialize(self, raw_value): 101 | return datetime.datetime.strptime(decode(raw_value), self._format) 102 | 103 | def serialize(self, value): 104 | return encode(value.strftime(self._format)) 105 | 106 | 107 | class DateField(DateTimeField): 108 | _format = '%Y-%m-%d' 109 | 110 | def deserialize(self, raw_value): 111 | dt = datetime.datetime.strptime(decode(raw_value), self._format) 112 | return dt.date() 113 | 114 | 115 | class IntegerField(Field): 116 | _index_type = C.INDEX_NUM 117 | _order_asc = C.ORDER_NUM_ASC 118 | _order_desc = C.ORDER_NUM_DESC 119 | 120 | __eq__ = _e(C.OP_NUM_EQ) 121 | __ne__ = _e(C.OP_NUM_EQ | C.OP_NEGATE) 122 | __gt__ = _e(C.OP_NUM_GT) 123 | __ge__ = _e(C.OP_NUM_GE) 124 | __lt__ = _e(C.OP_NUM_LT) 125 | __le__ = _e(C.OP_NUM_LE) 126 | between = _me(C.OP_NUM_BETWEEN) # Includes both endpoints. 127 | matches_any = _me(C.OP_NUM_ANYEXACT) 128 | 129 | def deserialize(self, raw_value): 130 | return int(decode(raw_value)) 131 | 132 | def serialize(self, value): 133 | return encode(str(value)) 134 | 135 | 136 | class FloatField(IntegerField): 137 | def deserialize(self, raw_value): 138 | return float(decode(raw_value)) 139 | 140 | 141 | class TimestampField(IntegerField): 142 | def __init__(self, utc=True, *args, **kwargs): 143 | self._utc = utc 144 | super(TimestampField, self).__init__(*args, **kwargs) 145 | 146 | def deserialize(self, raw_value): 147 | num = int(decode(raw_value)) 148 | ts, microsecond = divmod(num, 1000000) 149 | if self._utc: 150 | dt = datetime.datetime.utcfromtimestamp(ts) 151 | else: 152 | dt = datetime.datetime.fromtimestamp(ts) 153 | return dt.replace(microsecond=microsecond) 154 | 155 | def serialize(self, value): 156 | if isinstance(value, datetime.datetime): 157 | pass # Note: date is a subclass of datetime. 158 | elif isinstance(value, datetime.date): 159 | value = datetime.datetime(value.year, value.month, value.day) 160 | elif isinstance(value, int): 161 | return encode(str(value * 1000000)) 162 | 163 | if self._utc: 164 | timestamp = calendar.timegm(value.utctimetuple()) 165 | else: 166 | timestamp = time.mktime(value.timetuple()) 167 | timestamp = (timestamp * 1000000) + value.microsecond 168 | return encode(str(timestamp)) 169 | 170 | 171 | class SearchField(Field): 172 | _index_type = C.INDEX_QGRAM 173 | __eq__ = _e(C.OP_STR_EQ) 174 | __ne__ = _e(C.OP_STR_EQ | C.OP_NEGATE) 175 | match = _e(C.OP_FTS_PHRASE) 176 | match_all = _me(C.OP_FTS_ALL) 177 | match_any = _me(C.OP_FTS_ANY) 178 | search = _e(C.OP_FTS_EXPRESSION) 179 | 180 | deserialize = decode 181 | serialize = encode 182 | 183 | 184 | class TokenField(TextField): 185 | _index_type = C.INDEX_TOKEN 186 | 187 | 188 | class BaseModel(type): 189 | def __new__(cls, name, bases, attrs): 190 | if not bases: 191 | return super(BaseModel, cls).__new__(cls, name, bases, attrs) 192 | 193 | for base in bases: 194 | for key, value in base.__dict__.items(): 195 | if key in attrs: continue 196 | if isinstance(value, Field): 197 | attrs[key] = deepcopy(value) 198 | 199 | model_class = super(BaseModel, cls).__new__(cls, name, bases, attrs) 200 | model_class.__data__ = None 201 | 202 | defaults = {} 203 | fields = {} 204 | indexes = [] 205 | for key, value in model_class.__dict__.items(): 206 | if isinstance(value, Field): 207 | value.add_to_class(model_class, key) 208 | if value._index: 209 | indexes.append(value) 210 | fields[key] = value 211 | if value._default is not None: 212 | defaults[key] = value._default 213 | 214 | model_class.__defaults__ = defaults 215 | model_class.__fields__ = fields 216 | model_class.__indexes__ = indexes 217 | return model_class 218 | 219 | def __getitem__(self, key): 220 | if isinstance(key, (list, tuple, set)): 221 | return self.get_list(key) 222 | else: 223 | return self.get(key) 224 | 225 | def __setitem__(self, key, value): 226 | if isinstance(value, dict): 227 | value = self(**value) 228 | 229 | if value.key and value.key != key: 230 | raise ValueError('Data contains key which does not match key used ' 231 | 'for setitem.') 232 | 233 | _, data = serialize_model(value) 234 | self.__database__.set(key, data) 235 | 236 | def __delitem__(self, key): 237 | if isinstance(key, (list, tuple, set)): 238 | self.__database__.remove_bulk(key) 239 | else: 240 | self.__database__.remove(key) 241 | 242 | 243 | def _with_metaclass(meta, base=object): 244 | return meta("NewBase", (base,), {'__database__': None}) 245 | 246 | 247 | def serialize_model(model): 248 | buf = io.BytesIO() 249 | for name, field in model.__fields__.items(): 250 | if name == 'key': continue 251 | value = getattr(model, name, None) 252 | if value is not None: 253 | buf.write(encode(name)) 254 | buf.write(b'\x00') 255 | buf.write(field.serialize(value)) 256 | buf.write(b'\x00') 257 | return model.key, buf.getvalue() 258 | 259 | 260 | def deserialize_into_model(model_class, key, raw_data): 261 | data = {'key': key} 262 | items = raw_data.split(b'\x00') 263 | i, l = 0, len(items) - 1 264 | while i < l: 265 | key = decode(items[i]) 266 | value = items[i + 1] 267 | field = model_class.__fields__.get(key) 268 | if field is not None: 269 | data[key] = field.deserialize(value) 270 | else: 271 | data[key] = decode(value) 272 | i += 2 273 | return model_class(**data) 274 | 275 | 276 | class KeyField(TextField): 277 | def __set__(self, instance, value): 278 | instance.__data__[self.name] = value 279 | instance.__key__ = value 280 | 281 | 282 | class Model(_with_metaclass(BaseModel)): 283 | __database__ = None 284 | 285 | # Key is used to indicate the key in which the model data is stored. 286 | key = KeyField() 287 | 288 | def __init__(self, **kwargs): 289 | self.__data__ = {} 290 | self.__key__ = None 291 | self._load_default_dict() 292 | for key, value in kwargs.items(): 293 | setattr(self, key, value) 294 | 295 | def _load_default_dict(self): 296 | for field_name, default in self.__defaults__.items(): 297 | if callable(default): 298 | default = default() 299 | setattr(self, field_name, default) 300 | 301 | def __getitem__(self, attr): 302 | return getattr(self, attr) 303 | 304 | def __setitem__(self, attr, value): 305 | setattr(self, attr, value) 306 | 307 | def __repr__(self): 308 | return '<%s: %s>' % (type(self).__name__, self.key) 309 | 310 | def save(self): 311 | if not self.key: 312 | raise ValueError('Cannot save model without specifying a key.') 313 | key, data = serialize_model(self) 314 | return self.__database__.set(key, data) 315 | 316 | def delete(self): 317 | if not self.key: 318 | raise ValueError('Cannot delete model without specifying a key.') 319 | return self.__database__.remove(self.key) 320 | 321 | @classmethod 322 | def create_indexes(cls, safe=True): 323 | for field in cls.__indexes__: 324 | cls.__database__.set_index(field.name, field._index_type, not safe) 325 | 326 | @classmethod 327 | def drop_indexes(cls): 328 | for field in cls.__indexes__: 329 | cls.__database__.delete_index(field.name) 330 | 331 | @classmethod 332 | def optimize_indexes(cls): 333 | for field in cls.__indexes__: 334 | cls.__database__.optimize_index(field.name) 335 | 336 | @classmethod 337 | def create(cls, key, **data): 338 | model = cls(key=key, **data) 339 | model.save() 340 | return model 341 | 342 | @classmethod 343 | def get(cls, key): 344 | data = cls.__database__.get(key) 345 | if data is None: 346 | raise KeyError(key) 347 | return deserialize_into_model(cls, key, data) 348 | 349 | @classmethod 350 | def get_list(cls, keys): 351 | data = cls.__database__.get_bulk(keys) 352 | return [deserialize_into_model(cls, key, data[key]) 353 | for key in keys if key in data] 354 | 355 | @classmethod 356 | def create_list(cls, models): 357 | accum = {} 358 | for model in models: 359 | key, data = serialize_model(model) 360 | accum[key] = data 361 | return cls.__database__.set_bulk(accum) 362 | 363 | @classmethod 364 | def delete_list(cls, keys): 365 | return cls.__database__.remove_bulk(keys) 366 | 367 | @classmethod 368 | def all(cls): 369 | return cls.query().get() 370 | 371 | @classmethod 372 | def query(cls): 373 | return ModelSearch(cls) 374 | 375 | @classmethod 376 | def count(cls): 377 | return cls.query().count() 378 | 379 | 380 | def clone_query(method): 381 | def inner(self, *args, **kwargs): 382 | clone = self.clone() 383 | method(clone, *args, **kwargs) 384 | return clone 385 | return inner 386 | 387 | 388 | class ModelSearch(object): 389 | def __init__(self, model): 390 | self._model = model 391 | self._conditions = [] 392 | self._order_by = [] 393 | self._limit = self._offset = None 394 | 395 | def clone(self): 396 | obj = ModelSearch(self._model) 397 | obj._conditions = list(self._conditions) 398 | obj._order_by = list(self._order_by) 399 | obj._limit = self._limit 400 | obj._offset = self._offset 401 | return obj 402 | 403 | @clone_query 404 | def filter(self, *expressions): 405 | for (field, op, value) in expressions: 406 | if isinstance(value, (list, set, tuple)): 407 | items = [(field.serialize(item) 408 | if not isinstance(item, string_type) 409 | else encode(item)) for item in value] 410 | value = b','.join(items) 411 | elif isinstance(value, string_type): 412 | value = encode(value) 413 | else: 414 | value = field.serialize(value) 415 | self._conditions.append((field.name, op, value)) 416 | 417 | @clone_query 418 | def order_by(self, *ordering): 419 | for item in ordering: 420 | if isinstance(item, Field): 421 | item = item.asc() 422 | self._order_by.append((item.field.name, item.value)) 423 | 424 | @clone_query 425 | def limit(self, limit=None): 426 | self._limit = limit 427 | 428 | @clone_query 429 | def offset(self, offset=None): 430 | self._offset = offset 431 | 432 | def _build_search(self): 433 | cmd = [('addcond', col, op, val) for col, op, val in self._conditions] 434 | for col, order in self._order_by: 435 | cmd.append(('setorder', col, order)) 436 | if self._limit is not None or self._offset is not None: 437 | cmd.append(('setlimit', self._limit or 1 << 31, self._offset or 0)) 438 | return cmd 439 | 440 | def execute(self): 441 | return self._model.__database__.search(self._build_search()) 442 | 443 | def delete(self): 444 | return self._model.__database__.search(self._build_search(), 'out') 445 | 446 | def get(self): 447 | accum = [] 448 | results = self._model.__database__.search(self._build_search(), 'get') 449 | for key, data in results: 450 | accum.append(deserialize_into_model(self._model, key, data)) 451 | return accum 452 | 453 | def count(self): 454 | return self._model.__database__.search(self._build_search(), 'count') 455 | 456 | def __iter__(self): 457 | return iter(self.execute()) 458 | -------------------------------------------------------------------------------- /kt/queue.py: -------------------------------------------------------------------------------- 1 | class Queue(object): 2 | """ 3 | Helper-class for working with the Kyoto Tycoon Lua queue functions. 4 | """ 5 | def __init__(self, client, key, db=None): 6 | self._client = client 7 | self._key = key 8 | self._db = client._default_db if db is None else db 9 | 10 | def _lua(self, fn, **kwargs): 11 | kwargs.update(queue=self._key, db=self._db) 12 | return self._client.script(fn, kwargs) 13 | 14 | def add(self, item): 15 | return int(self._lua('queue_add', data=item)['id']) 16 | 17 | def extend(self, items): 18 | args = {str(i): item for i, item in enumerate(items)} 19 | return int(self._lua('queue_madd', **args)['num']) 20 | 21 | def _item_list(self, fn, n=1): 22 | items = self._lua(fn, n=n) 23 | if n == 1: 24 | return items['0'] if items else None 25 | 26 | accum = [] 27 | if items: 28 | for key in sorted(items, key=int): 29 | accum.append(items[key]) 30 | return accum 31 | 32 | def pop(self, n=1): 33 | return self._item_list('queue_pop', n) 34 | def rpop(self, n=1): 35 | return self._item_list('queue_rpop', n) 36 | 37 | def peek(self, n=1): 38 | return self._item_list('queue_peek', n) 39 | def rpeek(self, n=1): 40 | return self._item_list('queue_rpeek', n) 41 | 42 | def count(self): 43 | return int(self._lua('queue_size')['num']) 44 | __len__ = count 45 | 46 | def remove(self, data, n=None): 47 | if n is None: 48 | n = -1 49 | return int(self._lua('queue_remove', data=data, n=n)['num']) 50 | def rremove(self, data, n=None): 51 | if n is None: 52 | n = -1 53 | return int(self._lua('queue_rremove', data=data, n=n)['num']) 54 | 55 | def clear(self): 56 | return int(self._lua('queue_clear')['num']) 57 | -------------------------------------------------------------------------------- /kt/scripts/kt.lua: -------------------------------------------------------------------------------- 1 | kt = __kyototycoon__ 2 | db = kt.db 3 | 4 | 5 | function _select_db(inmap) 6 | if inmap.db then 7 | db_idx = tonumber(inmap.db) + 1 8 | inmap.db = nil 9 | db = kt.dbs[db_idx] 10 | else 11 | db = kt.db 12 | end 13 | return db 14 | end 15 | 16 | -- helper function for hash functions. 17 | function hkv(inmap, outmap, fn) 18 | local key = inmap.table_key 19 | if not key then 20 | kt.log("system", "hash function missing required: 'table_key'") 21 | return kt.RVEINVALID 22 | end 23 | local db = _select_db(inmap) -- Allow db to be specified as argument. 24 | inmap.table_key = nil 25 | local value, xt = db:get(key) 26 | local value_tbl = {} 27 | if value then 28 | value_tbl = kt.mapload(value) 29 | end 30 | local new_value, ok = fn(key, value_tbl, inmap, outmap) 31 | if ok then 32 | if new_value and not db:set(key, kt.mapdump(new_value), xt) then 33 | return kt.RVEINTERNAL 34 | else 35 | return kt.RVSUCCESS 36 | end 37 | else 38 | return kt.RVELOGIC 39 | end 40 | end 41 | 42 | -- Redis-like HMSET functionality for setting multiple key/value pairs. 43 | -- accepts: { table_key, ... } 44 | -- returns: { num } 45 | function hmset(inmap, outmap) 46 | local fn = function(k, v, i, o) 47 | local num = 0 48 | for key, value in pairs(i) do 49 | v[key] = value 50 | num = num + 1 51 | end 52 | o.num = num 53 | return v, true 54 | end 55 | return hkv(inmap, outmap, fn) 56 | end 57 | 58 | 59 | -- Redis-like HMGET functionality for getting multiple key/value pairs. 60 | -- accepts: { table_key, k1, k2 ... } 61 | -- returns: { k1=v1, k2=v2, ... } 62 | function hmget(inmap, outmap) 63 | local fn = function(k, v, i, o) 64 | for key, value in pairs(i) do 65 | o[key] = v[key] 66 | end 67 | return nil, true 68 | end 69 | return hkv(inmap, outmap, fn) 70 | end 71 | 72 | 73 | -- Redis-like HMDEL functionality for deleting multiple key/value pairs. 74 | -- accepts: { table_key, k1, k2 ... } 75 | -- returns: { num } 76 | function hmdel(inmap, outmap) 77 | local fn = function(k, v, i, o) 78 | local num = 0 79 | for key, value in pairs(i) do 80 | if v[key] then 81 | num = num + 1 82 | v[key] = nil 83 | end 84 | end 85 | o.num = num 86 | return v, true 87 | end 88 | return hkv(inmap, outmap, fn) 89 | end 90 | 91 | 92 | -- Redis-like HGETALL functionality for getting entire contents of hash. 93 | -- accepts: { table_key } 94 | -- returns: { k1=v1, k2=v2, ... } 95 | function hgetall(inmap, outmap) 96 | local fn = function(k, v, i, o) 97 | for key, value in pairs(v) do 98 | o[key] = value 99 | end 100 | return nil, true 101 | end 102 | return hkv(inmap, outmap, fn) 103 | end 104 | 105 | 106 | -- Redis-like HSET functionality for setting a single key/value in a hash. 107 | -- accepts: { table_key, key, value } 108 | -- returns: { num } 109 | function hset(inmap, outmap) 110 | local fn = function(k, v, i, o) 111 | local key, value = i.key, i.value 112 | if not key or not value then 113 | return nil, false 114 | end 115 | v[key] = value 116 | o.num = 1 117 | return v, true 118 | end 119 | return hkv(inmap, outmap, fn) 120 | end 121 | 122 | 123 | -- Redis-like HSET functionality for setting a key/value if key != exist. 124 | -- accepts: { table_key, key, value } 125 | -- returns: { num } 126 | function hsetnx(inmap, outmap) 127 | local fn = function(k, v, i, o) 128 | local key, value = i.key, i.value 129 | if not key or not value then 130 | return nil, false 131 | end 132 | if v[key] ~= nil then 133 | o.num = 0 134 | return nil, true 135 | else 136 | v[key] = value 137 | o.num = 1 138 | return v, true 139 | end 140 | end 141 | return hkv(inmap, outmap, fn) 142 | end 143 | 144 | 145 | -- Redis-like HGET functionality for getting a single key/value in a hash. 146 | -- accepts: { table_key, key } 147 | -- returns: { value } 148 | function hget(inmap, outmap) 149 | local fn = function(k, v, i, o) 150 | local key = i.key 151 | if not key then 152 | return nil, false 153 | end 154 | o.value = v[key] 155 | return nil, true 156 | end 157 | return hkv(inmap, outmap, fn) 158 | end 159 | 160 | 161 | -- Redis-like HDEL functionality for deleting a single key/value in a hash. 162 | -- accepts: { table_key, key } 163 | -- returns: { num } 164 | function hdel(inmap, outmap) 165 | local fn = function(k, v, i, o) 166 | local key = i.key 167 | if not key then 168 | return nil, false 169 | end 170 | if v[key] then 171 | v[key] = nil 172 | o.num = 1 173 | else 174 | o.num = 0 175 | end 176 | return v, true 177 | end 178 | return hkv(inmap, outmap, fn) 179 | end 180 | 181 | 182 | -- Redis-like HLEN functionality for determining number of items in a hash. 183 | -- accepts: { table_key } 184 | -- returns: { num } 185 | function hlen(inmap, outmap) 186 | local fn = function(k, v, i, o) 187 | local count = 0 188 | for _ in pairs(v) do 189 | count = count + 1 190 | end 191 | o.num = count 192 | return nil, true 193 | end 194 | return hkv(inmap, outmap, fn) 195 | end 196 | 197 | 198 | -- Redis-like HCONTAINS functionality for determining if key exists in a hash. 199 | -- accepts: { table_key, key } 200 | -- returns: { num } 201 | function hcontains(inmap, outmap) 202 | local fn = function(k, v, i, o) 203 | local key = i.key 204 | if not key then 205 | return nil, false 206 | end 207 | if v[key] then 208 | o.num = 1 209 | else 210 | o.num = 0 211 | end 212 | return nil, true 213 | end 214 | return hkv(inmap, outmap, fn) 215 | end 216 | 217 | 218 | -- helper function for set functions. 219 | function skv(inmap, outmap, fn) 220 | local key = inmap.key 221 | if not key then 222 | kt.log("system", "set function missing required: 'key'") 223 | return kt.RVEINVALID 224 | end 225 | local db = _select_db(inmap) -- Allow db to be specified as argument. 226 | inmap.key = nil 227 | local value, xt = db:get(key) 228 | local value_tbl = {} 229 | if value then 230 | value_tbl = kt.mapload(value) 231 | end 232 | local new_value, ok = fn(key, value_tbl, inmap, outmap) 233 | if ok then 234 | if new_value and not db:set(key, kt.mapdump(new_value), xt) then 235 | return kt.RVEINTERNAL 236 | else 237 | return kt.RVSUCCESS 238 | end 239 | else 240 | return kt.RVELOGIC 241 | end 242 | end 243 | 244 | 245 | -- Redis-like SADD functionality for adding value/score to set. 246 | -- accepts: { key, value } where multiple values are delimited by '\x01' 247 | -- returns: { num } 248 | function sadd(inmap, outmap) 249 | local fn = function(k, v, i, o) 250 | local value = i.value 251 | if not value then 252 | return nil, false 253 | end 254 | local n = 0 255 | local values = kt.split(value, "\1") 256 | for i = 1, #values do 257 | if v[values[i]] == nil then 258 | v[values[i]] = "" 259 | n = n + 1 260 | end 261 | end 262 | outmap.num = n 263 | if n == 0 then 264 | return nil, true 265 | else 266 | return v, true 267 | end 268 | end 269 | return skv(inmap, outmap, fn) 270 | end 271 | 272 | 273 | -- Redis-like SCARD functionality for determining cardinality of a set. 274 | -- accepts: { key } 275 | -- returns: { num } 276 | function scard(inmap, outmap) 277 | local fn = function(k, v, i, o) 278 | local count = 0 279 | for _ in pairs(v) do 280 | count = count + 1 281 | end 282 | o.num = count 283 | return nil, true 284 | end 285 | return skv(inmap, outmap, fn) 286 | end 287 | 288 | 289 | -- Redis-like SISMEMBER functionality for determining if value in a set. 290 | -- accepts: { key, value } 291 | -- returns: { num } 292 | function sismember(inmap, outmap) 293 | local fn = function(k, v, i, o) 294 | local value = i.value 295 | if not value then 296 | return nil, false 297 | end 298 | 299 | o.num = 0 300 | if v[value] ~= nil then 301 | o.num = 1 302 | end 303 | return nil, true 304 | end 305 | return skv(inmap, outmap, fn) 306 | end 307 | 308 | 309 | -- Redis-like SMEMBERS functionality for getting all values in a set. 310 | -- accepts: { key } 311 | -- returns: { v1, v2, ... } 312 | function smembers(inmap, outmap) 313 | local fn = function(k, v, i, o) 314 | for key, value in pairs(v) do 315 | o[key] = '1' 316 | end 317 | return nil, true 318 | end 319 | return skv(inmap, outmap, fn) 320 | end 321 | 322 | 323 | -- Redis-like SPOP functionality for removing a member from a set. 324 | -- accepts: { key } 325 | -- returns: { num, value } 326 | function spop(inmap, outmap) 327 | local fn = function(k, v, i, o) 328 | o.num = 0 329 | for key, value in pairs(v) do 330 | o.num = 1 331 | o.value = key 332 | v[key] = nil 333 | return v, true 334 | end 335 | return nil, true 336 | end 337 | return skv(inmap, outmap, fn) 338 | end 339 | 340 | 341 | -- Redis-like SREM functionality for removing a value from a set. 342 | -- accepts: { key, value } 343 | -- returns: { num } 344 | function srem(inmap, outmap) 345 | local fn = function(k, v, i, o) 346 | local value = i.value 347 | if not value then 348 | return nil, false 349 | end 350 | 351 | local n = 0 352 | local values = kt.split(value, "\1") 353 | for i = 1, #values do 354 | if v[values[i]] ~= nil then 355 | n = n + 1 356 | v[values[i]] = nil 357 | end 358 | end 359 | o.num = n 360 | if n > 0 then 361 | return v, true 362 | else 363 | return nil, true 364 | end 365 | end 366 | return skv(inmap, outmap, fn) 367 | end 368 | 369 | 370 | -- helper function for set operations on 2 keys. 371 | function svv(inmap, outmap, fn) 372 | local key1, key2 = inmap.key1, inmap.key2 373 | if not key1 or not key2 then 374 | kt.log("system", "set function missing required: 'key1' or 'key2'") 375 | return kt.RVEINVALID 376 | end 377 | local db = _select_db(inmap) -- Allow db to be specified as argument. 378 | local value1, xt = db:get(key1) 379 | local value2, xt = db:get(key2) 380 | 381 | local value_tbl1 = {} 382 | local value_tbl2 = {} 383 | if value1 then value_tbl1 = kt.mapload(value1) end 384 | if value2 then value_tbl2 = kt.mapload(value2) end 385 | 386 | local ret = fn(value_tbl1, value_tbl2, inmap, outmap) 387 | if ret == kt.RVSUCCESS and inmap.dest then 388 | if not db:set(inmap.dest, kt.mapdump(outmap), xt) then 389 | return kt.RVEINTERNAL 390 | end 391 | end 392 | return kt.RVSUCCESS 393 | end 394 | 395 | 396 | -- Redis-like SINTER functionality for finding intersection of 2 sets. 397 | -- accepts: { key1, key2, (dest) } 398 | -- returns: { ... } 399 | function sinter(inmap, outmap) 400 | local fn = function(v1, v2, i, o) 401 | for key, val in pairs(v1) do 402 | if v2[key] ~= nil then 403 | o[key] = '1' 404 | end 405 | end 406 | return kt.RVSUCCESS 407 | end 408 | return svv(inmap, outmap, fn) 409 | end 410 | 411 | 412 | -- Redis-like SUNION functionality for finding union of 2 sets. 413 | -- accepts: { key1, key2, (dest) } 414 | -- returns: { ... } 415 | function sunion(inmap, outmap) 416 | local fn = function(v1, v2, i, o) 417 | for key, val in pairs(v1) do 418 | o[key] = '1' 419 | end 420 | for key, val in pairs(v2) do 421 | o[key] = '1' 422 | end 423 | return kt.RVSUCCESS 424 | end 425 | return svv(inmap, outmap, fn) 426 | end 427 | 428 | 429 | -- Redis-like SDIFF functionality for finding difference of set1 and set2. 430 | -- accepts: { key1, key2, (dest) } 431 | -- returns: { ... } 432 | function sdiff(inmap, outmap) 433 | local fn = function(v1, v2, i, o) 434 | for key, val in pairs(v1) do 435 | if v2[key] == nil then 436 | o[key] = '1' 437 | end 438 | end 439 | return kt.RVSUCCESS 440 | end 441 | return svv(inmap, outmap, fn) 442 | end 443 | 444 | 445 | -- helper function for list functions. 446 | function lkv(inmap, outmap, fn) 447 | local key = inmap.key 448 | if not key then 449 | kt.log("system", "list function missing required: 'key'") 450 | return kt.RVEINVALID 451 | end 452 | local db = _select_db(inmap) -- Allow db to be specified as argument. 453 | inmap.key = nil 454 | local value, xt = db:get(key) 455 | local value_array = {} 456 | if value then 457 | value_array = kt.arrayload(value) 458 | end 459 | local new_value, ok = fn(key, value_array, inmap, outmap) 460 | if ok then 461 | if new_value and not db:set(key, kt.arraydump(new_value), xt) then 462 | return kt.RVEINTERNAL 463 | else 464 | return kt.RVSUCCESS 465 | end 466 | else 467 | return kt.RVELOGIC 468 | end 469 | end 470 | 471 | 472 | -- Redis-like LPUSH 473 | -- accepts: { key, value } 474 | -- returns: {} 475 | function llpush(inmap, outmap) 476 | local fn = function(key, arr, inmap, outmap) 477 | local value = inmap.value 478 | if not value then 479 | kt.log("system", "missing value parameter to llpush") 480 | return nil, false 481 | end 482 | table.insert(arr, 1, value) 483 | return arr, true 484 | end 485 | return lkv(inmap, outmap, fn) 486 | end 487 | 488 | -- Redis-like RPUSH 489 | -- accepts: { key, value } 490 | -- returns: {} 491 | function lrpush(inmap, outmap) 492 | local fn = function(key, arr, inmap, outmap) 493 | local value = inmap.value 494 | if not value then 495 | kt.log("system", "missing value parameter to lrpush") 496 | return nil, false 497 | end 498 | table.insert(arr, value) 499 | return arr, true 500 | end 501 | return lkv(inmap, outmap, fn) 502 | end 503 | 504 | 505 | function _normalize_index(array_len, idx) 506 | local index = tonumber(idx or "0") + 1 507 | if index < 1 then 508 | index = array_len + index 509 | if index < 1 then return nil, false end 510 | end 511 | if index > array_len then return nil, false end 512 | return index, true 513 | end 514 | 515 | 516 | -- Redis-like LRANGE -- zero-based. 517 | -- accepts: { key, start, stop } 518 | -- returns: { i1, i2, ... } 519 | function lrange(inmap, outmap) 520 | local fn = function(key, arr, inmap, outmap) 521 | local arrsize = #arr 522 | local start = tonumber(inmap.start or "0") + 1 523 | if start < 1 then 524 | start = arrsize + start 525 | if start < 1 then 526 | return nil, true 527 | end 528 | end 529 | 530 | local stop = inmap.stop 531 | if stop then 532 | stop = tonumber(stop) 533 | if stop < 0 then 534 | stop = arrsize + stop 535 | end 536 | else 537 | stop = arrsize 538 | end 539 | 540 | for i = start, stop do 541 | outmap[i - 1] = arr[i] 542 | end 543 | return nil, true 544 | end 545 | return lkv(inmap, outmap, fn) 546 | end 547 | 548 | 549 | -- Redis-like LINDEX -- zero-based. 550 | -- accepts: { key, index } 551 | -- returns: { value } 552 | function lindex(inmap, outmap) 553 | local fn = function(key, arr, inmap, outmap) 554 | local index, ok = _normalize_index(#arr, inmap.index) 555 | if ok then 556 | local val = arr[index] 557 | outmap.value = arr[index] 558 | end 559 | return nil, true 560 | end 561 | return lkv(inmap, outmap, fn) 562 | end 563 | 564 | 565 | -- LINSERT -- zero-based. 566 | -- accepts: { key, index, value } 567 | -- returns: {} 568 | function linsert(inmap, outmap) 569 | local fn = function(key, arr, inmap, outmap) 570 | local index, ok = _normalize_index(#arr, inmap.index) 571 | if not ok then 572 | return nil, false 573 | end 574 | if not inmap.value then 575 | kt.log("info", "missing value for linsert") 576 | return nil, false 577 | end 578 | table.insert(arr, index, inmap.value) 579 | return arr, true 580 | end 581 | return lkv(inmap, outmap, fn) 582 | end 583 | 584 | 585 | -- Redis-like LPOP -- removes first elem. 586 | -- accepts: { key } 587 | -- returns: { value } 588 | function llpop(inmap, outmap) 589 | local fn = function(key, arr, inmap, outmap) 590 | outmap.value = arr[1] 591 | table.remove(arr, 1) 592 | return arr, true 593 | end 594 | return lkv(inmap, outmap, fn) 595 | end 596 | 597 | 598 | -- Redis-like RPOP -- removes last elem. 599 | -- accepts: { key } 600 | -- returns: { value } 601 | function lrpop(inmap, outmap) 602 | local fn = function(key, arr, inmap, outmap) 603 | outmap.value = arr[#arr] 604 | arr[#arr] = nil 605 | return arr, true 606 | end 607 | return lkv(inmap, outmap, fn) 608 | end 609 | 610 | 611 | -- Redis-like LLEN -- returns length of list. 612 | -- accepts: { key } 613 | -- returns: { num } 614 | function llen(inmap, outmap) 615 | local fn = function(key, arr, inmap, outmap) 616 | outmap.num = #arr 617 | return nil, true 618 | end 619 | return lkv(inmap, outmap, fn) 620 | end 621 | 622 | 623 | -- Redis-like LSET -- set item at index. 624 | -- accepts: { key, index, value } 625 | -- returns: {} 626 | function lset(inmap, outmap) 627 | local fn = function(key, arr, inmap, outmap) 628 | local idx = tonumber(inmap.index or "0") 629 | if not inmap.value then 630 | kt.log("info", "missing value for lset") 631 | return nil, false 632 | end 633 | if idx < 0 or idx >= #arr then 634 | kt.log("info", "invalid index for lset") 635 | return nil, false 636 | end 637 | arr[idx + 1] = inmap.value 638 | return arr, true 639 | end 640 | return lkv(inmap, outmap, fn) 641 | end 642 | 643 | 644 | -- Misc helpers. 645 | 646 | 647 | -- Move src to dest. 648 | -- accepts: { src, dest } 649 | -- returns: {} 650 | function move(inmap, outmap) 651 | local src = inmap.src 652 | local dest = inmap.dest 653 | if not src or not dest then 654 | kt.log("info", "missing src and/or dest key in move() call") 655 | return kt.RVEINVALID 656 | end 657 | local db = _select_db(inmap) -- Allow db to be specified as argument. 658 | local keys = { src, dest } 659 | local first = true 660 | local src_val = nil 661 | local src_xt = nil 662 | local function visit(key, value, xt) 663 | -- Operating on first key, capture value and xt and remove. 664 | if first then 665 | src_val = value 666 | src_xt = xt 667 | first = false 668 | return kt.Visitor.REMOVE 669 | end 670 | 671 | -- Operating on dest key, store value and xt. 672 | if src_val then 673 | return src_val, src_xt 674 | end 675 | return kt.Visitor.NOP 676 | end 677 | 678 | if not db:accept_bulk(keys, visit) then 679 | return kt.REINTERNAL 680 | end 681 | 682 | if not src_val then 683 | return kt.RVELOGIC 684 | end 685 | return kt.RVSUCCESS 686 | end 687 | 688 | 689 | -- List all key-value pairs. 690 | -- accepts: {} 691 | -- returns: { k=v ... } 692 | function list(inmap, outmap) 693 | local db = _select_db(inmap) -- Allow db to be specified as argument. 694 | local cur = db:cursor() 695 | cur:jump() 696 | while true do 697 | local key, value, xt = cur:get(true) 698 | if not key then break end 699 | outmap[key] = value 700 | end 701 | cur:disable() 702 | return kt.RVSUCCESS 703 | end 704 | 705 | 706 | -- Fetch a range of key-value pairs. 707 | -- accepts: { start: key, stop: key, db: idx } 708 | -- returns: { k1=v1, k2=v2, ... } 709 | function get_range(inmap, outmap) 710 | local db = _select_db(inmap) 711 | local start_key = inmap.start 712 | local stop_key = inmap.stop 713 | local cur = db:cursor() 714 | if start_key then 715 | if not cur:jump(start_key) then 716 | cur:disable() 717 | return kt.RVSUCCESS 718 | end 719 | else 720 | if not cur:jump() then 721 | cur:disable() 722 | return kt.RVSUCCESS 723 | end 724 | end 725 | local key, value 726 | while true do 727 | key = cur:get_key() 728 | if stop_key and key > stop_key then 729 | break 730 | end 731 | outmap[key] = cur:get_value() 732 | if not cur:step() then 733 | break 734 | end 735 | end 736 | cur:disable() 737 | return kt.RVSUCCESS 738 | end 739 | 740 | 741 | -- Hash one or more values. 742 | -- accepts: { val1: method1, val2: method2, ... } 743 | -- returns: { val1: hash1, val2: hash2, ... } 744 | function hash(inmap, outmap) 745 | for key, val in pairs(inmap) do 746 | if val == 'fnv' then 747 | outmap[key] = kt.hash_fnv(val) 748 | else 749 | outmap[key] = kt.hash_murmur(val) 750 | end 751 | end 752 | end 753 | 754 | 755 | -- Get a portion of a string value stored in a key. Behaves like slice operator 756 | -- does in Python. 757 | -- accepts: { key, start, stop, db } 758 | -- returns: { value } 759 | function get_part(inmap, outmap) 760 | local db = _select_db(inmap) 761 | local start_idx = inmap.start or 0 762 | local stop_idx = inmap.stop 763 | local key = inmap.key 764 | if not key then 765 | kt.log("info", "missing key in get_part() call") 766 | return kt.RVEINVALID 767 | end 768 | 769 | local value, xt = db:get(key) 770 | if value ~= nil then 771 | start_idx = tonumber(start_idx) 772 | if start_idx >= 0 then start_idx = start_idx + 1 end 773 | if stop_idx then 774 | stop_idx = tonumber(stop_idx) 775 | -- If the stop index is negative, we need to subtract 1 to get 776 | -- Python-like semantics. 777 | if stop_idx < 0 then stop_idx = stop_idx - 1 end 778 | value = string.sub(value, start_idx, stop_idx) 779 | else 780 | value = string.sub(value, start_idx) 781 | end 782 | end 783 | outmap.value = value 784 | return kt.RVSUCCESS 785 | end 786 | 787 | 788 | -- Queue helpers. 789 | 790 | -- Simple wrapper that does some basic validation and dispatches to the 791 | -- user-defined callback. 792 | function _qfn(inmap, outmap, required, fn) 793 | local db = _select_db(inmap) 794 | for i, key in pairs(required) do 795 | if not inmap[key] then 796 | kt.log("info", "queue: missing required parameter: " .. key) 797 | return kt.RVEINVALID 798 | end 799 | end 800 | return fn(db, inmap, outmap) 801 | end 802 | 803 | 804 | -- add/enqueue data to a queue 805 | -- accepts: { queue, data, db } 806 | -- returns { id } 807 | function queue_add(inmap, outmap) 808 | local fn = function(db, i, o) 809 | local id = db:increment_double(i.queue, 1) 810 | if not id then 811 | kt.log("info", "unable to determine id when adding item to queue!") 812 | return kt.RVELOGIC 813 | end 814 | local key = string.format("%s\t%012d", i.queue, id) 815 | if not db:add(key, i.data) then 816 | kt.log("info", "could not add key, already exists") 817 | return kt.RVELOGIC 818 | end 819 | o.id = id 820 | return kt.RVSUCCESS 821 | end 822 | return _qfn(inmap, outmap, {"queue", "data"}, fn) 823 | end 824 | 825 | 826 | -- add/enqueue multiple items to a queue 827 | -- accepts: { queue, 0: data0, 1: data1, ... n: dataN, db } 828 | -- returns { num } 829 | function queue_madd(inmap, outmap) 830 | local fn = function(db, i, o) 831 | local n = 0 832 | while i[tostring(n)] ~= nil do 833 | local id = db:increment_double(i.queue, 1) 834 | if not id then 835 | kt.log("info", "unable to determine id when adding item to queue!") 836 | return kt.RVELOGIC 837 | end 838 | local key = string.format("%s\t%012d", i.queue, id) 839 | if not db:add(key, i[tostring(n)]) then 840 | kt.log("info", "could not add key, already exists") 841 | return kt.RVELOGIC 842 | end 843 | n = n + 1 844 | end 845 | o.num = n 846 | return kt.RVSUCCESS 847 | end 848 | return _qfn(inmap, outmap, {"queue"}, fn) 849 | end 850 | 851 | 852 | function _queue_iter(db, queue, n, callback) 853 | -- Perform a forward iteration through the queue (up to "n" items). The 854 | -- user-defined callback returns a 2-tuple of (ok, incr) to signal that we 855 | -- should continue looping, and to increment the counter, respectively. 856 | local num = 0 857 | local cursor = db:cursor() 858 | local key = string.format("%s\t", queue) 859 | local pattern = string.format("^%s\t", queue) 860 | 861 | -- No data, we're done. 862 | if not cursor:jump(key) then 863 | cursor:disable() 864 | return num 865 | end 866 | 867 | local k, v, xt, ok, incr 868 | 869 | while n ~= 0 do 870 | -- Retrieve the key, value and xt from the cursor. If the cursor is 871 | -- invalidated then nil is returned. 872 | k, v, xt = cursor:get(false) 873 | if not k then break end 874 | 875 | -- If this is not a queue item key, we are done. 876 | if not k:match(pattern) then break end 877 | 878 | -- Pass control to the user-defined function, which is responsible for 879 | -- stepping the cursor. 880 | ok, incr = callback(cursor, k, v, num) 881 | if not ok then break end 882 | 883 | if incr then 884 | num = num + 1 885 | n = n - 1 886 | end 887 | end 888 | 889 | cursor:disable() 890 | return num 891 | end 892 | 893 | 894 | function _queue_iter_reverse(db, queue, n, callback) 895 | -- Perform a backward iteration through the queue (up to "n" items). The 896 | -- user-defined callback returns a 2-tuple of (ok, incr) to signal that we 897 | -- should continue looping, and to increment the counter, respectively. 898 | local num = 0 899 | local cursor = db:cursor() 900 | local max_key = string.format("%s\t\255", queue) 901 | local pattern = string.format("^%s\t", queue) 902 | 903 | -- No data, we're done. 904 | if not cursor:jump_back(max_key) then 905 | cursor:disable() 906 | return num 907 | end 908 | 909 | local k, v, xt, ok, incr 910 | 911 | while n ~= 0 do 912 | -- Retrieve the key, value and xt from the cursor. If the cursor is 913 | -- invalidated then nil is returned. 914 | k, v, xt = cursor:get(false) 915 | if not k then break end 916 | 917 | -- If this is a queue item key, we remove the value (which implicitly steps 918 | -- to the next key). 919 | if not k:match(pattern) then break end 920 | 921 | ok, incr = callback(cursor, k, v, num) 922 | if not ok then break end 923 | 924 | if incr then 925 | num = num + 1 926 | n = n - 1 927 | end 928 | end 929 | 930 | cursor:disable() 931 | return num 932 | end 933 | 934 | 935 | -- Remove items from a queue based on value (up-to "n" items). 936 | -- accepts: { queue, data, db, n } 937 | -- returns { num } 938 | function queue_remove(inmap, outmap) 939 | local cb = function(db, i, o) 940 | local queue = i.queue 941 | local data = i.data 942 | local n = tonumber(i.n or -1) 943 | local iter_cb = function(cursor, k, v, num) 944 | if data == v then 945 | return cursor:remove(), true 946 | else 947 | return cursor:step(), false 948 | end 949 | end 950 | outmap.num = _queue_iter(db, queue, n, iter_cb) 951 | return kt.RVSUCCESS 952 | end 953 | return _qfn(inmap, outmap, {"queue", "data"}, cb) 954 | end 955 | 956 | 957 | -- Remove items from the back of a queue, based on value (up-to "n" items). 958 | -- accepts: { queue, data, db, n } 959 | -- returns { num } 960 | function queue_rremove(inmap, outmap) 961 | local cb = function(db, i, o) 962 | local queue = i.queue 963 | local data = i.data 964 | local n = tonumber(i.n or -1) 965 | local iter_cb = function(cursor, k, v, num) 966 | if data == v then 967 | return cursor:remove(), true 968 | else 969 | return cursor:step_back(), false 970 | end 971 | end 972 | outmap.num = _queue_iter_reverse(db, queue, n, iter_cb) 973 | return kt.RVSUCCESS 974 | end 975 | return _qfn(inmap, outmap, {"queue", "data"}, cb) 976 | end 977 | 978 | 979 | -- pop/dequeue data from queue 980 | -- accepts: { queue, n, db } 981 | -- returns { idx: data, ... } 982 | function queue_pop(inmap, outmap) 983 | local cb = function(db, i, o) 984 | local n = tonumber(i.n or 1) 985 | local iter_cb = function(cursor, key, value, num) 986 | o[tostring(num)] = value 987 | return cursor:remove(), true 988 | end 989 | _queue_iter(db, i.queue, n, iter_cb) 990 | end 991 | return _qfn(inmap, outmap, {"queue"}, cb) 992 | end 993 | 994 | 995 | -- pop/dequeue data from end of queue 996 | -- accepts: { queue, n, db } 997 | -- returns { idx: data, ... } 998 | function queue_rpop(inmap, outmap) 999 | local cb = function(db, i, o) 1000 | local n = tonumber(i.n or 1) 1001 | local iter_cb = function(cursor, key, value, num) 1002 | o[tostring(num)] = value 1003 | return cursor:remove(), true 1004 | end 1005 | _queue_iter_reverse(db, i.queue, n, iter_cb) 1006 | end 1007 | return _qfn(inmap, outmap, {"queue"}, cb) 1008 | end 1009 | 1010 | 1011 | -- peek data from queue 1012 | -- accepts: { queue, n, db } 1013 | -- returns { idx: data, ... } 1014 | function queue_peek(inmap, outmap) 1015 | local cb = function(db, i, o) 1016 | local n = tonumber(i.n or 1) 1017 | local iter_cb = function(cursor, key, value, num) 1018 | o[tostring(num)] = value 1019 | return cursor:step(), true 1020 | end 1021 | _queue_iter(db, i.queue, n, iter_cb) 1022 | end 1023 | return _qfn(inmap, outmap, {"queue"}, cb) 1024 | end 1025 | 1026 | 1027 | -- peek data from end of queue 1028 | -- accepts: { queue, n, db } 1029 | -- returns { idx: data, ... } 1030 | function queue_rpeek(inmap, outmap) 1031 | local cb = function(db, i, o) 1032 | local n = tonumber(i.n or 1) 1033 | local iter_cb = function(cursor, key, value, num) 1034 | o[tostring(num)] = value 1035 | return cursor:step_back(), true 1036 | end 1037 | _queue_iter_reverse(db, i.queue, n, iter_cb) 1038 | end 1039 | return _qfn(inmap, outmap, {"queue"}, cb) 1040 | end 1041 | 1042 | 1043 | -- get queue size 1044 | -- accepts: { queue, db } 1045 | -- returns: { num } 1046 | function queue_size(inmap, outmap) 1047 | if not inmap.queue then 1048 | kt.log("info", "missing queue parameter in queue_size call") 1049 | return kt.RVEINVALID 1050 | end 1051 | 1052 | local db = _select_db(inmap) 1053 | local keys = db:match_prefix(string.format("%s\t", inmap.queue)) 1054 | outmap.num = tostring(#keys) 1055 | return kt.RVSUCCESS 1056 | end 1057 | 1058 | 1059 | -- clear queue, removing all items 1060 | -- accepts: { queue, db } 1061 | -- returns: { num } 1062 | function queue_clear(inmap, outmap) 1063 | if not inmap.queue then 1064 | kt.log("info", "missing queue parameter in queue_size call") 1065 | return kt.RVEINVALID 1066 | end 1067 | 1068 | local db = _select_db(inmap) 1069 | local keys = db:match_prefix(string.format("%s\t", inmap.queue)) 1070 | db:remove_bulk(keys) 1071 | db:remove(inmap.queue) 1072 | outmap.num = tostring(#keys) 1073 | return kt.RVSUCCESS 1074 | end 1075 | 1076 | 1077 | -- Simple hexastore graph. 1078 | 1079 | -- Python-like string split, with proper handling of edge-cases. 1080 | function nsplit(s, delim, n) 1081 | n = n or -1 1082 | local pos, length = 1, #s 1083 | local parts = {} 1084 | while pos do 1085 | local dstart, dend = string.find(s, delim, pos, true) 1086 | local part 1087 | if not dstart then 1088 | part = string.sub(s, pos) 1089 | pos = nil 1090 | elseif dend < dstart then 1091 | part = string.sub(s, pos, dstart) 1092 | if dstart < length then 1093 | pos = dstart + 1 1094 | else 1095 | pos = nil 1096 | end 1097 | else 1098 | part = string.sub(s, pos, dstart - 1) 1099 | pos = dend + 1 1100 | end 1101 | table.insert(parts, part) 1102 | n = n - 1 1103 | if n == 0 and pos then 1104 | if dend < length then 1105 | table.insert(parts, string.sub(s, pos)) 1106 | end 1107 | break 1108 | end 1109 | end 1110 | return parts 1111 | end 1112 | 1113 | function _hx_keys_for_values(s, p, o) 1114 | local perms = { 1115 | {'spo', s, p, o}, 1116 | {'pos', p, o, s}, 1117 | {'osp', o, s, p}} 1118 | local output = {} 1119 | for i = 1, #perms do 1120 | output[i] = table.concat(perms[i], '::') 1121 | end 1122 | return output 1123 | end 1124 | 1125 | function _hx_keys_for_query(s, p, o) 1126 | local parts = {} 1127 | local key = function(parts) return table.concat(parts, '::') end 1128 | 1129 | if s and p and o then 1130 | return key({"spo", s, p, o}), nil 1131 | elseif s and p then 1132 | parts = {"spo", s, p} 1133 | elseif s and o then 1134 | parts = {"osp", s, o} 1135 | elseif p and o then 1136 | parts = {"pos", p, o} 1137 | elseif s then 1138 | parts = {"spo", s} 1139 | elseif p then 1140 | parts = {"pos", p} 1141 | elseif o then 1142 | parts = {"osp", o} 1143 | end 1144 | local term = {} 1145 | for _, value in pairs(parts) do 1146 | table.insert(term, value) 1147 | end 1148 | table.insert(parts, "") 1149 | table.insert(term, "\255") 1150 | return key(parts), key(term) 1151 | end 1152 | 1153 | -- add item to hexastore 1154 | -- accepts { s, p, o } (subject, predicate, object) 1155 | function hx_add(inmap, outmap) 1156 | local db, s, p, o = _select_db(inmap), inmap.s, inmap.p, inmap.o 1157 | if not s or not p or not o then 1158 | kt.log("info", "missing s/p/o parameter in hx_add call") 1159 | return kt.REVINVALID 1160 | end 1161 | 1162 | local data = {} 1163 | for i, key in pairs(_hx_keys_for_values(s, p, o)) do 1164 | data[key] = "" 1165 | end 1166 | db:set_bulk(data) 1167 | return kt.RVSUCCESS 1168 | end 1169 | 1170 | -- remove item from hexastore 1171 | -- accepts { s, p, o } 1172 | function hx_remove(inmap, outmap) 1173 | local db, s, p, o = _select_db(inmap), inmap.s, inmap.p, inmap.o 1174 | if not s or not p or not o then 1175 | kt.log("info", "missing s/p/o parameter in hx_remove call") 1176 | return kt.REVINVALID 1177 | end 1178 | 1179 | db:remove_bulk(_hx_keys_for_values(s, p, o)) 1180 | return kt.RVSUCCESS 1181 | end 1182 | 1183 | -- query hexastore 1184 | -- accepts { s, p, o } 1185 | function hx_query(inmap, outmap) 1186 | local db, s, p, o = _select_db(inmap), inmap.s, inmap.p, inmap.o 1187 | if not s and not p and not o then 1188 | kt.log("info", "missing s/p/o parameter in hx_query call") 1189 | return kt.REVINVALID 1190 | end 1191 | 1192 | local start, stop = _hx_keys_for_query(s, p, o) 1193 | if not stop then 1194 | local value, xt = db:get(start) 1195 | if value then outmap['0'] = value end 1196 | return kt.RVSUCCESS 1197 | end 1198 | 1199 | local cursor = db:cursor() 1200 | if not cursor:jump(start) then 1201 | cursor:disable() 1202 | return kt.RVSUCCESS 1203 | end 1204 | 1205 | local i = 0 1206 | local key, value, xt 1207 | while true do 1208 | key, value, xt = cursor:get() 1209 | if key > stop then break end 1210 | _hx_key_to_table(outmap, key, i, s == nil, p == nil, o == nil) 1211 | i = i + 1 1212 | if not cursor:step() then 1213 | break 1214 | end 1215 | end 1216 | cursor:disable() 1217 | return kt.RVSUCCESS 1218 | end 1219 | 1220 | function _hx_key_to_table(tbl, key, idx, store_s, store_p, store_o) 1221 | -- internal function for adding the parts of an s/p/o key to a result table. 1222 | -- only stores the parts indicated (based on the user query). 1223 | local parts = nsplit(key, "::") 1224 | local structure = parts[1] -- e.g., "spo", "ops", "pos". 1225 | local k 1226 | for i = 1, 3 do 1227 | k = string.sub(structure, i, i) -- e.g., "s", "p" or "o". 1228 | if (k == 'o' and store_o) or (k == 's' and store_s) or (k == 'p' and store_p) then 1229 | tbl[k .. tostring(idx)] = parts[i + 1] 1230 | end 1231 | end 1232 | end 1233 | 1234 | 1235 | -- get luajit version. 1236 | function jit_version(inmap, outmap) 1237 | outmap.version = "v" .. jit.version 1238 | return kt.RVSUCCESS 1239 | end 1240 | 1241 | if kt.thid == 0 then 1242 | kt.log("system", "luajit version: " .. jit.version) 1243 | end 1244 | -------------------------------------------------------------------------------- /kt/scripts/tt.lua: -------------------------------------------------------------------------------- 1 | -- Find records whose key matches a pattern. Accepts pattern and optional 2 | -- maximum number of results. 3 | function match_pattern(key, value) 4 | value = tonumber(value) 5 | if not value then value = 0 end 6 | local res = "" 7 | function proc(tkey, tvalue) 8 | if string.match(tkey, key) then 9 | res = res .. tkey .. "\t" .. tvalue .. "\n" 10 | value = value - 1 11 | if value == 0 then return false end 12 | end 13 | return true 14 | end 15 | _foreach(proc) 16 | return res 17 | end 18 | 19 | 20 | -- Find records whose key is within a certain edit distance. Accepts pattern 21 | -- key and edit distance, which defaults to 0 (exact match) if not provided. 22 | function match_similar(key, value) 23 | value = tonumber(value) 24 | if not value then 25 | value = 0 26 | end 27 | local res = "" 28 | function proc(tkey, tvalue) 29 | if _dist(tkey, key) <= value then 30 | res = res .. tkey .. "\t" .. tvalue .. "\n" 31 | end 32 | return true 33 | end 34 | _foreach(proc) 35 | return res 36 | end 37 | 38 | 39 | -- Find records whose value is within a certain edit distance. Accepts value 40 | -- pattern and edit distance. 41 | function match_similar_value(key, value) 42 | value = tonumber(value) 43 | if not value then 44 | value = 0 45 | end 46 | local res = "" 47 | function proc(tkey, tvalue) 48 | if _dist(tvalue, key) <= value then 49 | res = res .. tkey .. "\t" .. tvalue .. "\n" 50 | end 51 | return true 52 | end 53 | _foreach(proc) 54 | return res 55 | end 56 | 57 | 58 | -- Lock a key. 59 | function lock(key, value) 60 | if _lock(key) then return "true" else return "false" end 61 | end 62 | 63 | -- Unlock a key. 64 | function unlock(key, value) 65 | if _unlock(key) then return "true" else return "false" end 66 | end 67 | 68 | -- Seize/pop implementation. 69 | function seize(key, value) 70 | _lock(key) 71 | local res = _get(key) or "" 72 | if res ~= nil then _out(key) end 73 | _unlock(key) 74 | return res 75 | end 76 | 77 | 78 | -- De-serialize and serialize a table database value to a lua table. 79 | function mapload(m) 80 | local t = {} 81 | local elems = _split(m) 82 | for i = 1, #elems, 2 do 83 | t[elems[i]] = elems[i + 1] 84 | end 85 | return t 86 | end 87 | 88 | 89 | function mapdump(t) 90 | local res = "" 91 | local glue = string.char(0) 92 | local key, value 93 | for key, value in pairs(t) do 94 | res = res .. key .. glue .. value .. glue 95 | end 96 | return res 97 | end 98 | 99 | 100 | -- Table helpers for working with values in a table database. 101 | function table_get(key, value) 102 | local tval = _get(key) 103 | if tval ~= nil then 104 | local tmap = mapload(tval) 105 | return tmap[value] or '' 106 | end 107 | return '' 108 | end 109 | 110 | function table_update(key, value) 111 | -- Value is assumed to be NULL-separated key/value. 112 | local items = _split(value) 113 | if #items < 2 then 114 | _log('expected null-separated key/value pairs for for table_set()') 115 | else 116 | local tval = _get(key) 117 | local tmap 118 | if tval == nil then 119 | tmap = {} 120 | else 121 | tmap = mapload(tval) 122 | end 123 | for i = 1, #items, 2 do 124 | tmap[items[i]] = items[i + 1] 125 | end 126 | _put(key, mapdump(tmap)) 127 | return "true" 128 | end 129 | return "false" 130 | end 131 | 132 | function table_pop(key, value) 133 | local tval = _get(key) 134 | if tval ~= nil then 135 | local tmap = mapload(tval) 136 | local ret = tmap[value] 137 | if ret ~= nil then 138 | tmap[value] = nil 139 | _put(key, mapdump(tmap)) 140 | end 141 | return ret or '' 142 | end 143 | return '' 144 | end 145 | 146 | 147 | -- Split a string. 148 | function split(key, value) 149 | if key == "" then return "" end 150 | 151 | if #value < 1 then 152 | value = nil 153 | end 154 | local elems = _split(key, value) 155 | local res = "" 156 | for i = 1, #elems do 157 | res = res .. elems[i] .. "\n" 158 | end 159 | return res 160 | end 161 | 162 | 163 | -- e.g. hash('foo bar', 'md5'), hash('checksum me', 'crc32') 164 | function hash(key, value) 165 | if #value < 1 then value = "md5" end 166 | return _hash(value, key) 167 | end 168 | 169 | -- hash the value stored in a key, if it exists. 170 | function hash_key(key, value) 171 | local tval = _get(key) 172 | if tval == nil then return '' end 173 | if #value < 1 then value = "md5" end 174 | return _hash(value, tval) 175 | end 176 | 177 | 178 | function time(key, value) 179 | return string.format("%.6f", _time()) 180 | end 181 | 182 | 183 | function ptime(key, value) 184 | _log("current time: " .. _time()) 185 | return "ok" 186 | end 187 | 188 | 189 | function getdate(key, value) 190 | -- Verify os module is available. 191 | return os.date("%Y-%m-%dT%H:%M:%S") 192 | end 193 | 194 | 195 | function glob(key, value) 196 | local paths = _glob(key) 197 | local res = "" 198 | for i = 1, #paths do 199 | res = res .. paths[i] .. "\n" 200 | end 201 | return res 202 | end 203 | 204 | 205 | -- Evaluate arbitrary user script. 206 | function script(key, value) 207 | if not _eval(key) then 208 | return nil 209 | end 210 | return "ok" 211 | end 212 | 213 | 214 | -- Queue 215 | -- enqueue a record 216 | function queue_add(key, value) 217 | local id = _adddouble(key, 1) 218 | if not id then 219 | _log("unable to determine id") 220 | return nil 221 | end 222 | key = string.format("%s\t%012d", key, id) 223 | if not _putkeep(key, value) then 224 | _log("could not add key") 225 | return nil 226 | end 227 | return "ok" 228 | end 229 | 230 | 231 | -- dequeue a record 232 | function queue_pop(key, max) 233 | max = tonumber(max) 234 | if not max or max < 1 then 235 | max = 1 236 | end 237 | key = string.format("%s\t", key) 238 | local keys = _fwmkeys(key, max) 239 | local res = "" 240 | for i = 1, #keys do 241 | local key = keys[i] 242 | local value = _get(key) 243 | if _out(key) and value then 244 | res = res .. value .. "\n" 245 | end 246 | end 247 | return res 248 | end 249 | 250 | 251 | -- blocking dequeue. 252 | function queue_bpop(key, max) 253 | res = queue_pop(key, max) 254 | while res == "" do 255 | sleep(0.1) 256 | res = queue_pop(key, max) 257 | end 258 | return res 259 | end 260 | 261 | 262 | -- get the queue size 263 | function queue_size(key) 264 | key = string.format("%s\t", key) 265 | local keys = _fwmkeys(key) 266 | return #keys 267 | end 268 | 269 | 270 | -- clear queue 271 | function queue_clear(key) 272 | key = string.format("%s\t", key) 273 | local keys = _fwmkeys(key) 274 | _misc("outlist", unpack(keys)) 275 | _out(key) 276 | return #keys 277 | end 278 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import warnings 3 | 4 | from setuptools import setup 5 | from setuptools.extension import Extension 6 | try: 7 | from Cython.Build import cythonize 8 | except ImportError: 9 | import warnings 10 | cython_installed = False 11 | warnings.warn('Cython not installed, using pre-generated C source file.') 12 | else: 13 | cython_installed = True 14 | 15 | try: 16 | from kt import __version__ 17 | except ImportError: 18 | warnings.warn('could not import kt module to determine version') 19 | __version__ = '0.0.0' 20 | 21 | 22 | if cython_installed: 23 | python_source = 'kt/_binary.pyx' 24 | else: 25 | python_source = 'kt/_binary.c' 26 | cythonize = lambda obj: obj 27 | 28 | kt = Extension( 29 | 'kt._binary', 30 | #extra_compile_args=['-g', '-O0'], 31 | #extra_link_args=['-g'], 32 | sources=[python_source]) 33 | 34 | setup( 35 | name='kt', 36 | version=__version__, 37 | description='Fast Python bindings for KyotoTycoon.', 38 | author='Charles Leifer', 39 | author_email='', 40 | packages=['kt'], 41 | ext_modules=cythonize([kt]), 42 | ) 43 | -------------------------------------------------------------------------------- /ttcommands.csv: -------------------------------------------------------------------------------- 1 | cmd,magic,args,response 2 | put,x10,ksiz:4 vsiz:4 kbuf:* vbuf:*, 3 | putkeep,x11,ksiz:4 vsiz:4 kbuf:* vbuf:*, 4 | putcat,x12,ksiz:4 vsiz:4 kbuf:* vbuf:*, 5 | putshl,x13,ksiz:4 vsiz:4 width:4 kbuf:* vbuf:*, 6 | putnr,x18,ksiz:4 vsiz:4 kbuf:* vbuf:*, 7 | out,x20,ksiz:4 kbuf:*, 8 | get,x30,ksiz:4 kbuf:*,vsiz:4 vbuf:* 9 | mget,x31,rnum:4 {ksiz:4 kbuf:*},rnum:4 {vsiz:4 vbuf:*} 10 | vsiz,x38,ksiz:4 kbuf:*,vsiz:4 11 | iterinit,x50,, 12 | iternext,x51,,ksiz:4 kbuf:* 13 | fwmkeys,x58,pfxsiz:4 maxrec:4 pfxbuf:*,rnum:4 {ksiz:4 kbuf:*} 14 | addint,x60,ksiz:4 num:4 kbuf:*,sum:4 15 | adddouble,x61,ksiz:4 integer:8 mantissa:8 kbuf:*,sumint:8 summantissa:8 16 | ext,x68,fnsiz:4 opts:4 ksiz:4 vsiz:4 fnbuf:* kbuf:* vbuf:*,rsiz:4 rbuf:* 17 | sync,x70,, 18 | optimize,x71,optsiz:4 optbuf:*, 19 | vanish,x72,, 20 | copy,x73,pathsiz:4 pathbuf:*, 21 | restore,x74,pathsiz:4 tsmicro:8 opts:4 pathbuf:*, 22 | setmst,x78,hostsiz:4 port:4 tsmicro:8 opts:4 hostbuf:*, 23 | rnum,x80,,rnum:8 24 | size,x81,,size:8 25 | stat,x88,,rsiz:4 rbuf:* 26 | misc,x90,fnsiz:4 opts:4 argc:4 fnbuf:* {asiz:4 abuf:*},rnum:4 {rsiz:4 rbuf:*} 27 | -------------------------------------------------------------------------------- /ttmisc.csv: -------------------------------------------------------------------------------- 1 | cmd,args,response,description 2 | put,"key, value",,set key/value 3 | putkeep,"key, value",,set if not exists 4 | putcat,"key, value",,append if exists 5 | putdup,"key, value",,add duplicate to front 6 | putdupback,"key, value",,add duplicate to back 7 | out,key,,delete 8 | get,key,value,get 9 | putlist,"k1,v1,kn,vn",,set multiple key/value pairs 10 | outlist,"k1,kn",,delete multiple keys 11 | getlist,"k1,kn","k1,v1,kn,vn",get multiple key/value pairs 12 | getpart,"key,start,length",value,get value at offset/length 13 | iterinit,startkey(opt),,begin iteration 14 | iternext,,"key,value",step iteration 15 | sync,,,synchronize changes 16 | optimize,optionstr,,optimize db 17 | vanish,,,clear all 18 | error,,errorstr,get error 19 | defrag,nsteps(opt),,defragment the database 20 | regex,"rgx,nrec(opt)","k1,v1,kn,vn",regex search 21 | range,"upper,nrec,lower","k1,v1,kn,vn",range search 22 | setindex,"name,type",,create index on column 23 | search,"cond1,condn","k1,v1,kn,vn",search 24 | genuid,,id,generate a unique id 25 | -------------------------------------------------------------------------------- /tuning.md: -------------------------------------------------------------------------------- 1 | ## Tokyo Tyrant 2 | 3 | `*` - in-memory hash 4 | `+` - in-memory b-tree 5 | `.tch` - on-disk hash 6 | `.tcb` - on-disk b-tree 7 | `.tcf` - fixed-length database 8 | `.tct` - table database 9 | 10 | In-memory hash: 11 | 12 | * `bnum` - number of buckets 13 | * `capnum` - capacity number of records 14 | * `capsiz` - capacity size of memory, when exceeded removed via FIFO. 15 | 16 | In-memory b-tree: 17 | 18 | * `capnum` - capacity number of records 19 | * `capsiz` - capacity size of memory, when exceeded removed via FIFO. 20 | 21 | On-disk hash: 22 | 23 | * `mode` - w=writer, r=reader, c=create, t=truncate, e=nolock default=wc 24 | * `bnum` - main tuning parameter, should be > number of records, default=131071 25 | * `apow` - alignment power, default=4, so 2^4 = 16 26 | * `fpow` - maximum elements in free block pool, default=10, so 2^10=1024 27 | * `opts` - l=large (>2gb), d=default, b=bzip2, t=tcbs 28 | * `rcnum` - record cache, default=0 (disabled) 29 | * `xmsiz` - extra mapped memory region size, default is 67108864 (64mb) 30 | * `dfunit` - unit step for auto-defragmentation, default=0 (disabled). 31 | 32 | On-disk b-tree: 33 | 34 | * `mode` 35 | * `lmemb` - number of members in each leaf node, default=128 36 | * `nmemb` - number of members in each non-leaf node, default=256 37 | * `bnum` - main tuning parameter, > 1/128 number of records, default=32749 38 | * `apow` - alignment power, default=8, so 2^8=256 39 | * `fpow` - maximum elements in free block pool, default=10, so 2^10=1024 40 | * `opts` - l=large (>2gb), d=default, b=bzip2, t=tcbs 41 | * `lcnum` - main tuning parameter, default is 1024, leaf-node cache size 42 | * `ncnum` - non-leaf nodes to cache, default=512 43 | * `xmsiz` - extra mapped memory region size, default=0 (disabled) 44 | * `dfunit` - unit step for auto-defragmentation, default=0 (disabled). 45 | 46 | Fixed-length: 47 | 48 | * `mode` 49 | * `width` - width of each value, default=255 50 | * `limsiz` - max size of database, default is 268435456 (256mb) 51 | 52 | Table: 53 | 54 | * `mode` 55 | * `bnum` - default is 131071, should be 0.5 - 4x number of records 56 | * `apow` - record alignment power, default=4, 2^4=16 57 | * `fpow` - maximum elements in free block pool, default=10, so 2^10=1024 58 | * `opts` - l=large (>2gb), d=default, b=bzip2, t=tcbs 59 | * `rcnum` - number of records to cache, default=0 (disabled) 60 | * `lcnum` - leaf-nodes to cache, default=4096 61 | * `ncnum` - non-leaf nodes to cache, default=512 62 | * `xmsiz` - extra mapped memory region size, default is 67108864 (64mb) 63 | * `dfunit` - unit step for auto-defragmentation, default=0 (disabled). 64 | * `idx` - field-name and type, separated by ":" 65 | 66 | 67 | ## Kyoto Tycoon 68 | 69 | Supported by all databases: 70 | 71 | * `log` - path to logfile 72 | * `logkinds` - debug, info, warn or error 73 | * `logpx` - prefix for each log message 74 | 75 | Other options: 76 | 77 | * `bnum` - number of buckets in the hash table 78 | * `capcnt` - set capacity by record number 79 | * `capsiz` - set capacity by memory usage 80 | * `psiz` - page size 81 | * `pccap` - page-cache capacity 82 | * `opts` - s=small, l=linear, c=compress - l=linked-list for hash collisions. 83 | * `zcomp` - compression algorithm: zlib, def (deflate), gz, lzo, lzma or arc 84 | * `zkey` - cipher key for compression (?) 85 | * `rcomp` - comparator function: lex, dec (decimal), lexdesc, decdesc 86 | * `apow` - alignment power of 2 87 | * `fpow` - maximum elements in the free-block pool 88 | * `msiz` - tune map 89 | * `dfunit` - unit step for auto-defragmentation, default=0 (disabled). 90 | 91 | Databases and available parameters: 92 | 93 | * Stash - bnum 94 | * Cache hash (`*`): opts, bnum, zcomp, capcnt, capsiz, zkey 95 | * Cache tree (`%`): opts, bnum, zcomp, zkey, psiz, rcomp, pccap 96 | * File hash (`.kch`): opts, bnum, apow, fpow, msiz, dfunit, zcomp, zkey 97 | * File tree (`.kct`): opts, bnum, apow, fpow, msiz, dfunit, zcomp, zkey, psiz, 98 | rcomp, pccap 99 | * Dir hash: (`.kcd`): opts, zcomp, zkey 100 | * Dir tree: (`.kcf`): opts, zcomp, zkey, psiz, rcomp, pccap 101 | * Plain-text: (`.kcx`): n/a 102 | 103 | #### Stash database 104 | 105 | * `bnum` - default is ~1M, should be 80%-400% of total records 106 | 107 | #### CacheHashDB 108 | 109 | * `bnum`: default ~1M. Should be 50% - 400% of total records. Collision 110 | chaining is binary search 111 | * `opts`: useful to reduce memory at expense of time effciency. Use compression 112 | if the key and value of each record is greater-than 1KB 113 | * `capcnt` and/or `capsiz`: keep memory usage constant by expiring old records 114 | 115 | #### CacheTreeDB 116 | 117 | Inherits all tuning options from the CacheHashDB, since each node of the btree 118 | is serialized as a page-buffer and treated as a record in the cache hash db. 119 | 120 | * `psiz`: default is 8192 121 | * `pccap`: default is 64MB 122 | * `rcomp`: default is lexical ordering 123 | 124 | #### HashDB 125 | 126 | * `bnum`: default ~1M. Suggested ratio is twice the total number of records, 127 | but can be anything from 100% - 400%. 128 | * `apow`: Power of the alignment of record size. Default=3, so the address of 129 | each record is aligned to a multiple of 8 (`2^3`) bytes. 130 | * `fpow`: Power of the capacity of the free block pool. Default=10, rarely 131 | needs to be modified. 132 | * `msiz`: Size of internal memory-mapped region. Default is 64MB. 133 | * `dfunit`: Unit step number of auto-defragmentation. Auto-defrag is disabled 134 | by default. 135 | 136 | apow, fpow, opts and bnum *must* be specified before a DB is opened and 137 | cannot be changed after the fact. 138 | 139 | #### TreeDB 140 | 141 | Inherits tuning parameters from the HashDB. 142 | 143 | * `psiz`: default is 8192, specified before opening db and cannot be changed 144 | * `pccap`: default is 64MB 145 | * `rcomp`: default is lexical 146 | 147 | The default alignment is 256 (2^8) and the default bucket number is ~64K. 148 | The bucket number should be calculated by the number of pages. Suggested 149 | ratio of bucket number is 10% of the number of records. 150 | --------------------------------------------------------------------------------