├── .gitignore ├── .travis.yml ├── README.md ├── codap └── __init__.py ├── examples ├── async_example.py ├── fib_example.py └── sync_example.py ├── setup.cfg ├── setup.py └── tests ├── __init__.py └── test_codap.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | eggs 9 | parts 10 | bin 11 | var 12 | sdist 13 | develop-eggs 14 | .installed.cfg 15 | 16 | # Installer logs 17 | pip-log.txt 18 | 19 | # Unit test / coverage reports 20 | .coverage 21 | .tox 22 | 23 | #Translations 24 | *.mo 25 | 26 | #Mr Developer 27 | .mr.developer.cfg 28 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | # - "2.6" doesn't discover 4 | - "2.7" 5 | - "3.2" 6 | - "3.3" 7 | - "pypy" 8 | # command to install dependencies 9 | install: 10 | - "pip install . --use-mirrors" 11 | # command to run tests 12 | script: python -m unittest discover -s tests 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | codap 2 | ===== 3 | 4 | [![Travis Build Status](https://api.travis-ci.org/lateefj/codap.png?branch=master)](https://travis-ci.org/lateefj/codap) 5 | 6 | Coroutine Data Access Patterns (codap) are a handful of libraries to make concurrent data access simpler to use. This was originally developed for web services access data in mutliple datastores (MongoDB, S3, REST, ect). The library uses async methods (thread, eventlet or gevent) which degrades gracefully. Perfering gevent, eventlet and falling back to threading. One of those things it would be really nice to have anonymous functions in Python :(. 7 | 8 | INSTALL 9 | ------- 10 | 11 | ``` 12 | pip install codap 13 | ``` 14 | 15 | 16 | Key / Value (Dictionary) 17 | ------------------------ 18 | 19 | Allows for a dictionary like access. This is great for caches, template rendering and since dictionaries are the most used data type it is easy to integrate into existing code. 20 | 21 | Example: 22 | 23 | ```python 24 | def back(db, name): 25 | return db.find(name) 26 | 27 | results = KV() 28 | results['foo'] = bar # bar is a function 29 | results.put('cats', get_photos, id, limit=4) # get_photos is a function 30 | results.put('monkey', back, db, name) 31 | render_template('my_temp.html', **results) 32 | ``` 33 | 34 | Ordered List 35 | ------------ 36 | 37 | List that returns the responses based on the order they are added. Has been used for retrieving already sorted data that needs additional information to be rendered. 38 | 39 | Example: 40 | 41 | ```python 42 | 43 | def get_stuff(db, x): 44 | return db.get(x) 45 | 46 | results = codap.Ordered() 47 | for x in xrange(0, 10): 48 | results.push(get_stuff, db, x) 49 | for r in results: # Same order as pushed 50 | r.render() 51 | ``` 52 | 53 | First Reply 54 | ----------- 55 | 56 | Based on the order of the response is the order it is added to the list. Useful for making request to multiple databases. Has been used for getting a list of files from a web service and compressing them into a single zip or tar. 57 | 58 | Example: 59 | 60 | ```python 61 | 62 | def get_data(id): 63 | return my_data[id] 64 | 65 | fr = codap.FirstReply() 66 | for ds in datasource_list: 67 | fr.push(get_data, id) 68 | data = fr[0] 69 | ``` 70 | 71 | Fibonacci Example (full): 72 | ------------------------- 73 | 74 | ```python 75 | import codap 76 | 77 | 78 | def fib(n): 79 | if n == 1: 80 | return 1 81 | elif n == 0: 82 | return 0 83 | else: 84 | return fib(n - 1) + fib(n - 2) 85 | 86 | FIB_SIZE = 30 87 | d = codap.KV() 88 | # Push a bunch of fib processing into the background 89 | for i in range(0, FIB_SIZE): 90 | d.put(i, fib, i) 91 | 92 | # Pull them out from the list 93 | for i in range(0, FIB_SIZE): 94 | assert d[i] == fib(i), 'Expected fib {0} to be {1} but was {3}'.format(i, d[i], fib(i)) 95 | 96 | d = codap.Ordered() 97 | for i in range(0, FIB_SIZE): 98 | d.push(fib, i) 99 | 100 | i = 0 101 | for f in d: 102 | assert f == fib(i), 'Expected fib {0} to be {1} but was {3}'.format(i, f, fib(i)) 103 | i += 1 104 | ``` 105 | 106 | -------------------------------------------------------------------------------- /codap/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Lateef Jackson 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | # 6 | # Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | # Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | # Neither the name of Lateef Jackson nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 9 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 10 | 11 | """ 12 | (codap) Coroutine Data Access Patterns 13 | 14 | These are some utitlites classes I built for improving response time in a web service. 15 | I used this to simplify making sequencial requests to data storage (serivices or database) 16 | concurrent so that there would be less IO wait. Warning more traditional RDBMS may have inverse 17 | performance compared to caches or datastores that are distributed. 18 | 19 | """ 20 | import sys 21 | 22 | 23 | try: 24 | # Prefer gevent as it would be fastest 25 | from gevent import spawn 26 | from gevent.queue import Queue 27 | from gevent import monkey 28 | monkey.patch_socket() # Required if we want IO to be concurrent 29 | except: 30 | try: 31 | # Eventlet we are also fans of so that would be great 32 | from eventlet import spawn 33 | from eventlet.queue import Queue 34 | import eventlet 35 | eventlet.monkey_patch(all=True) # To support concurrent IO 36 | 37 | except: 38 | # Thread worst case but this will not scale well at all 39 | if sys.version_info[0] == 3: # Python 3 40 | from queue import Queue as ThreadQueue 41 | else: 42 | from Queue import Queue as ThreadQueue 43 | from threading import Thread 44 | 45 | def thread_spawn(*args, **kwargs): 46 | """ 47 | Wrapper that acts like the coroutine libraries. Nothing really to 48 | see here. 49 | """ 50 | t = None 51 | if len(args) == 1 and not kwargs: 52 | t = Thread(target=args[0], args=()) 53 | else: 54 | t = Thread(target=args[0], args=args[1:], kwargs=kwargs) 55 | t.start() 56 | # Fall back to using threads 57 | Queue = ThreadQueue 58 | spawn = thread_spawn 59 | 60 | 61 | class KV(dict): 62 | """ 63 | This wrapper around a dict is mostly for to make getting the data 64 | however there is the ability to set the data. I have found in most 65 | cases that the function will need parameters and that is why the 66 | put function exists. 67 | """ 68 | 69 | def __init__(self, *args, **kwargs): 70 | dict.__init__(self, *args, **kwargs) 71 | 72 | def __getitem__(self, key): 73 | v = dict.__getitem__(self, key).get() 74 | dict.__getitem__(self, key).put(v) # Make sure it is there next time we ask for it 75 | return v 76 | 77 | def __setitem__(self, k, v): 78 | """ 79 | This is just a wrapper around the put function but allows the second 80 | argument to be the callback. The callback can not have any arguments 81 | or keywords so this has limited use. 82 | """ 83 | self.put(k, v) 84 | 85 | def put(self, *args, **kwargs): 86 | """ 87 | This is design to hanlde 3 situation 88 | 1. Can pass just the key as the first argument and call put on queue 89 | that is returned. 90 | 2. Pass key as fist argument and callback as second which will be called 91 | in a coroutine (or thread). 92 | 3. Pass key as first argument and callback as second and the rest of the 93 | arguments and keywords will be passed to the callback. 94 | 95 | """ 96 | key = args[0] 97 | callback = None 98 | args = args[1:] 99 | if len(args) > 0: 100 | callback = args[0] 101 | args = args[1:] 102 | q = Queue() 103 | dict.__setitem__(self, key, q) 104 | if callback: 105 | def handle(): 106 | if args: 107 | q.put(callback(*args)) 108 | elif not args and kwargs: 109 | q.put(callback(*args, **kwargs)) 110 | else: 111 | q.put(callback(*args, **kwargs)) 112 | spawn(handle) 113 | return q 114 | 115 | 116 | class Ordered: 117 | 118 | def __init__(self): 119 | self.size = 0 120 | self.kv = KV() 121 | self.index = 0 122 | 123 | def __add__(self, x): 124 | self.push(x) 125 | 126 | def __getitem__(self, i): 127 | print('{0} for in {1}'.format(i, self.kv.keys())) 128 | return self.kv[i] 129 | 130 | def __setitem__(self, k, v): 131 | self.kv[k] = v 132 | 133 | def __getslice__(self, i, j): 134 | return [self.kv[x] for x in xrange(i - 1, j)] 135 | 136 | def __delslize__(self, i, j): 137 | for x in xrange(i - 1, j): 138 | del self.kv[x] 139 | 140 | def __len__(self): 141 | return len(self.kv.keys()) 142 | 143 | def __iter__(self): 144 | return self 145 | def __next__(self): 146 | v = None 147 | if self.index < len(self.kv.keys()): 148 | v = self.kv[self.index] 149 | self.index += 1 150 | else: 151 | self.index = 0 152 | raise StopIteration 153 | return v 154 | next = __next__ # Python 2.x backport 155 | 156 | def push(self, *args, **kwargs): 157 | self.kv.put(self.size, *args, **kwargs) 158 | self.size += 1 159 | 160 | append = __add__ 161 | 162 | 163 | class FirstReply: 164 | 165 | def __init__(self): 166 | self.size = 0 167 | self.index = 0 168 | self.q = Queue() 169 | 170 | def push(self, *args, **kwargs): 171 | self.size += 1 172 | callback = args[0] 173 | args = args[1:] 174 | if callback: 175 | def handle(): 176 | if args and not kwargs: 177 | self.q.put(callback(*args)) 178 | elif not args and kwargs: 179 | self.q.put(callback()) 180 | else: 181 | self.q.put(callback(*args, **kwargs)) 182 | spawn(handle) 183 | return self.q 184 | 185 | def __len__(self): 186 | return self.size 187 | 188 | def __iter__(self): 189 | return self 190 | 191 | def __next__(self): 192 | v = None 193 | if self.index < self.size: 194 | v = self.q.get() 195 | self.index += 1 196 | else: 197 | raise StopIteration 198 | return v 199 | next = __next__ # Python 2.x backport 200 | __add__ = push 201 | append = push 202 | -------------------------------------------------------------------------------- /examples/async_example.py: -------------------------------------------------------------------------------- 1 | import time 2 | import urllib2 3 | import json 4 | import codap 5 | def order_pass(order, url): 6 | start = time.time() 7 | data = urllib2.urlopen(url) 8 | end = time.time() 9 | return data, order, end - start 10 | URL = 'https://twitter.com/status/user_timeline/lateefjackson.json?count=10 ' 11 | start = time.time() 12 | fr = codap.FirstReply() 13 | #fr = codap.Ordered() 14 | count = 0 15 | fr.push(order_pass, count, URL) 16 | for x in xrange(0, 10): 17 | count += 1 18 | fr.push(order_pass, count, URL) 19 | resp, c, t = fr.next() 20 | data = json.loads(resp.read()) 21 | end = time.time() 22 | total_statuses = len(data) 23 | print('Getting it once time is: {0} with statuses {1}'.format(t, total_statuses)) 24 | loop_time = time.time() 25 | c = 0 26 | for resp, order, t in fr: 27 | data = json.loads(resp.read()) 28 | end = time.time() 29 | print('Order: {3} getting it {2} time is: {0} with statuses {1}'.format(t, total_statuses, c, order)) 30 | c += 1 31 | total_statuses += len(data) 32 | end = time.time() 33 | print('For 10 gets {0} with total statuses: {1} '.format(end - loop_time, total_statuses)) 34 | 35 | print('Total time is {0}'.format(end - start)) 36 | -------------------------------------------------------------------------------- /examples/fib_example.py: -------------------------------------------------------------------------------- 1 | import codap 2 | 3 | 4 | def fib(n): 5 | if n == 1: 6 | return 1 7 | elif n == 0: 8 | return 0 9 | else: 10 | return fib(n - 1) + fib(n - 2) 11 | 12 | FIB_SIZE = 30 13 | d = codap.KV() 14 | # Push a bunch of fib processing into the background 15 | for i in range(0, FIB_SIZE): 16 | d.put(i, fib, i) 17 | 18 | # Pull them out from the list 19 | for i in range(0, FIB_SIZE): 20 | assert d[i] == fib(i), 'Expected fib {0} to be {1} but was {3}'.format(i, d[i], fib(i)) 21 | 22 | d = codap.Ordered() 23 | for i in range(0, FIB_SIZE): 24 | d.push(fib, i) 25 | 26 | i = 0 27 | for f in d: 28 | assert f == fib(i), 'Expected fib {0} to be {1} but was {3}'.format(i, f, fib(i)) 29 | i += 1 30 | -------------------------------------------------------------------------------- /examples/sync_example.py: -------------------------------------------------------------------------------- 1 | import time 2 | import urllib2 3 | import json 4 | 5 | URL = 'https://twitter.com/status/user_timeline/lateefjackson.json?count=10' 6 | start = time.time() 7 | resp = urllib2.urlopen(URL) 8 | data = json.loads(resp.read()) 9 | end = time.time() 10 | total_statuses = len(data) 11 | print('Getting it once time is: {0} with statuses {1}'.format(end - start, total_statuses)) 12 | loop_time = time.time() 13 | c = 0 14 | for x in xrange(0, 10): 15 | resp = urllib2.urlopen(URL) 16 | data = json.loads(resp.read()) 17 | end = time.time() 18 | print('Getting it {2} time is: {0} with statuses {1}'.format(end - start, total_statuses, c)) 19 | c += 1 20 | total_statuses += len(data) 21 | end = time.time() 22 | print('For 10 gets {0} with total statuses: {1} '.format(end - loop_time, total_statuses)) 23 | 24 | print('Total time is {0}'.format(end - start)) 25 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | DESCRIPTION = """Coroutine Data Access Patterns (mouthful I know) are some data 5 | access patterns I found useful building web applications that use gevent or 6 | evenlet. They mimic common Python data types like lists and dictionaries.""" 7 | setup( 8 | name = 'codap', 9 | version = '0.0.5', 10 | author = 'Lateef Jackson', 11 | author_email = 'lateef.jackson@gmail.com', 12 | description = (DESCRIPTION), 13 | license = 'BSD', 14 | keywords = 'gevent eventlet coroutine ', 15 | url = 'https://github.com/lateefj/codap', 16 | packages=['codap', 'tests'], 17 | classifiers=[ 18 | 'Development Status :: 3 - Alpha', 19 | 'Topic :: Utilities', 20 | 'License :: OSI Approved :: BSD License', 21 | ], 22 | ) 23 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lateefj/codap/a69121c4c1e4664d8ea79ab1215f316de67d97dd/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_codap.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import codap 4 | 5 | 6 | class TestKV(unittest.TestCase): 7 | 8 | def test_kv_get_set(self): 9 | kv = codap.KV() 10 | def two(): 11 | return 2 12 | def t(n): 13 | return n*n 14 | r = range(0, 100) 15 | for x in r: 16 | kv[x] = two 17 | for x in r: 18 | assert kv[x] == 2, 'Expectex kv[x] to be {0} but it was {1}'.format(2, kv[x]) 19 | 20 | # Make sure that passing params to callback works 21 | kv = codap.KV() 22 | for x in r: 23 | kv.put(x, t, x) 24 | for x in r: 25 | assert kv[x] == x * x, 'Expectex kv[x] to be {0} but it was {1}'.format(x * x, kv[x]) 26 | # Test to make sure returning correct queue 27 | kv = codap.KV() 28 | queues = [] 29 | for x in r: 30 | queues.append(kv.put(x, t, x)) 31 | x = 0 32 | for q in queues: 33 | # Deterministic should make sure this works 34 | v = q.get() 35 | assert v == x * x, 'Expected q.get() to be {0} but it was {1}'.format(x * x, v) 36 | x += 1 37 | 38 | # Make sure put with just key as argument works 39 | kv = codap.KV() 40 | queues = [] 41 | for x in r: 42 | q = kv.put(x) 43 | queues.append(q) 44 | q.put(two()) 45 | for q in queues: 46 | v = q.get() 47 | assert v == 2, 'Expected q.get() to be 2 but it was {0}'.format(v) 48 | 49 | 50 | import time 51 | 52 | def delay(amount): 53 | # Would like ot use something a little more asymetric like urlopen 54 | if amount > 0: 55 | time.sleep(float(amount)/100) 56 | return amount 57 | 58 | 59 | class TestOrdered(unittest.TestCase): 60 | 61 | def test_basic(self): 62 | size = 5 63 | coorder = codap.Ordered() 64 | 65 | def o(x): 66 | return x 67 | 68 | for x in reversed(range(0, size)): 69 | coorder.push(delay, x) 70 | 71 | assert len(coorder) == size, 'Expected to be the riht size' 72 | c = size - 1 73 | for v in coorder: 74 | assert v == c, 'Expected v to be {0} but was {1}'.format(c, v) 75 | c -= 1 76 | 77 | c = size - 1 78 | for v in coorder: 79 | assert v == c, 'Expected v to be {0} but was {1}'.format(c, v) 80 | c -= 1 81 | 82 | c = size - 1 83 | for v in coorder: 84 | assert v == c, 'Expected v to be {0} but was {1}'.format(c, v) 85 | c -= 1 86 | 87 | 88 | class TestFirstReply(unittest.TestCase): 89 | 90 | def test_first_reply(self): 91 | fr = codap.FirstReply() 92 | 93 | def two(): 94 | return 2 95 | for x in range(0, 100): 96 | fr.push(two) 97 | for p in fr: 98 | assert p == 2, 'Expected 2 but got {0}'.format(p) 99 | 100 | start = 0 101 | for x in reversed(range(start, 4)): 102 | fr.push(delay, x) 103 | 104 | c = start 105 | for x in fr: 106 | assert x == c, 'Expected {0} but got {1}'.format(c, x) 107 | c += 1 108 | 109 | 110 | class TestUtils(unittest.TestCase): 111 | count = 0 112 | 113 | def setUp(self): 114 | TestUtils.count = 0 115 | 116 | def test_queue_spawn(self): 117 | size = 10 118 | tq = codap.Queue() 119 | run = True 120 | def handle(q): 121 | while run: 122 | m = q.get() 123 | TestUtils.count += 1 124 | codap.spawn(handle, tq) 125 | for x in range(0, size): 126 | tq.put(x) 127 | delay(1) 128 | assert size == TestUtils.count, 'Expected size: {0} to be count: {1}'.format(size, TestUtils.count) 129 | run = False 130 | tq.put(0) 131 | 132 | 133 | 134 | --------------------------------------------------------------------------------