├── MANIFEST ├── .gitignore ├── Makefile ├── lookupy ├── __init__.py ├── dunderkey.py ├── lookupy.py └── tests.py ├── tox.ini ├── setup.py ├── LICENSE ├── examples ├── simple.py ├── har.py └── facebook.py └── README.rst /MANIFEST: -------------------------------------------------------------------------------- 1 | # file GENERATED by distutils, do NOT edit 2 | setup.py 3 | lookupy/__init__.py 4 | lookupy/dunderkey.py 5 | lookupy/lookupy.py 6 | lookupy/tests.py 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.pyc 3 | build/ 4 | 5 | # files generated by coverage.py 6 | .coverage 7 | htmlcov/ 8 | 9 | # virtualenv dir 10 | env/ 11 | 12 | # tox dir 13 | .tox/ 14 | 15 | dist/ 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | clean: 3 | find . -name '*.pyc' -delete 4 | 5 | test: 6 | nosetests -v 7 | 8 | coverage: 9 | coverage run `which nosetests` -v 10 | coverage html 11 | xdg-open htmlcov/index.html 12 | 13 | shell: 14 | python3 -i -c "from lookupy import Collection, Q" 15 | 16 | -------------------------------------------------------------------------------- /lookupy/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | lookupy 3 | ~~~~~~~ 4 | 5 | Lookupy is a Python library that provides a Django QuerySet like 6 | interface to query (select and filter) data (list of dicts) 7 | 8 | """ 9 | 10 | from .lookupy import Collection, Q 11 | 12 | __all__ = ["Collection", "Q"] 13 | 14 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py27, py32 8 | 9 | [testenv] 10 | commands = nosetests 11 | deps = 12 | nose 13 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | try: 4 | long_desc = open('./README.rst').read() 5 | except IOError: 6 | long_desc = 'See: https://github.com/naiquevin/lookupy/blob/master/README.rst' 7 | 8 | setup( 9 | name='Lookupy', 10 | version='0.1', 11 | author='Vineet Naik', 12 | author_email='naikvin@gmail.com', 13 | url='https://github.com/naiquevin/lookupy', 14 | packages=['lookupy',], 15 | license='MIT License', 16 | description='Django QuerySet inspired interface to query list of dicts', 17 | long_description=long_desc, 18 | ) 19 | 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Vineet Naik (naikvin@gmail.com) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /examples/simple.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.append(os.path.abspath('../')) 5 | 6 | from lookupy import Collection, Q 7 | 8 | data = [{'framework': 'Django', 'language': 'Python', 'type': 'full-stack'}, 9 | {'framework': 'Flask', 'language': 'Python', 'type': 'micro'}, 10 | {'framework': 'Rails', 'language': 'Ruby', 'type': 'full-stack'}, 11 | {'framework': 'Sinatra', 'language': 'Ruby', 'type': 'micro'}, 12 | {'framework': 'Zend', 'language': 'PHP', 'type': 'full-stack'}, 13 | {'framework': 'Slim', 'language': 'PHP', 'type': 'micro'}] 14 | 15 | print('Data is a list of dict') 16 | print(data) 17 | print() 18 | 19 | c = Collection(data) 20 | 21 | print('Collection wraps data') 22 | print(c) 23 | print() 24 | 25 | print('Collection provides QuerySet as it\'s ``items`` attribute') 26 | print(c) 27 | print() 28 | 29 | print('filter returns a lazy QuerySet') 30 | print(c.filter(framework__startswith='S')) 31 | print() 32 | 33 | print('items in which the framework field startswith \'S\'') 34 | print(list(c.filter(framework__startswith='S'))) 35 | print() 36 | 37 | print('items in which the framework field startswith \'S\' and language is \'Ruby\'') 38 | print(list(c.filter(framework__startswith='S', language__exact='Ruby'))) 39 | print() 40 | 41 | print('items in which language is \'Python\' *or* \'Ruby\' ') 42 | print(list(c.filter(Q(language__exact='Python') | Q(language__exact='Ruby')))) 43 | print() 44 | 45 | print('items in which language is \'Python\' *or* \'Ruby\' *and* framework name startswith \'s\' and selected to show only the \'framework field\'') 46 | result = c.filter(Q(language__exact='Python') | Q(language__exact='Ruby')) \ 47 | .filter(framework__istartswith='s') \ 48 | .select('framework') 49 | print(list(result)) 50 | print() 51 | 52 | -------------------------------------------------------------------------------- /examples/har.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from pprint import pprint 4 | import operator 5 | from functools import reduce 6 | 7 | import sys 8 | 9 | sys.path.append(os.path.abspath('../')) 10 | 11 | from lookupy import Collection, Q 12 | 13 | 14 | f = open('www.youtube.com.har') 15 | data = json.load(f) 16 | f.close() 17 | 18 | c = Collection(data['log']['entries']) 19 | 20 | print("==== All javascript assets fetched ====") 21 | js_assets = c.filter(response__content__mimeType='text/javascript') \ 22 | .select('request__url') 23 | pprint(list(js_assets)) 24 | print() 25 | 26 | print("==== URLs that were blocked ====") 27 | blocked_urls = c.filter(timings__blocked__gt=0) \ 28 | .select('request__url') 29 | pprint(list(blocked_urls)) 30 | print() 31 | 32 | print("==== GET requests that responded with 200 OK ====") 33 | get_200 = c.filter(request__method__exact='GET', 34 | response__status__exact=200) \ 35 | .select('request__url') 36 | pprint(list(get_200)) 37 | print() 38 | 39 | print("==== Requests that responded with status other than 200 ====") 40 | not_200 = c.filter(response__status__neq=200).select('request__url') 41 | pprint(list(not_200)) 42 | print() 43 | 44 | print("==== Images ====") 45 | images = c.filter(response__headers__filter=Q(name__exact='Content-Type', 46 | value__startswith='image/')) \ 47 | .select('request__url', flatten=True) 48 | pprint(list(images)) 49 | print() 50 | 51 | print("==== Any of timings > 0 ===") 52 | timings = ["blocked", "dns", "connect", "send", "wait", "receive", "ssl"] 53 | timings_lookup = reduce(operator.or_, 54 | [Q(**{'timings__{t}__gt'.format(t=t): 0}) 55 | for t in timings]) 56 | pprint(list(c.filter(timings_lookup).select('request__url', 'timings'))) 57 | print() 58 | 59 | -------------------------------------------------------------------------------- /examples/facebook.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Examples showing how lookupy can be used with data obtained from the 4 | Facebook graph API. 5 | 6 | """ 7 | 8 | import re 9 | import os 10 | import sys 11 | import requests 12 | from pprint import pprint 13 | 14 | sys.path.append(os.path.abspath('../')) 15 | 16 | from lookupy import Collection, Q 17 | 18 | try: 19 | input = raw_input 20 | except NameError: 21 | pass 22 | 23 | 24 | FB_ID = None 25 | FB_ACCESS_TOKEN = None 26 | 27 | 28 | def fetch_posts(fb_id, access_token): 29 | url = 'https://graph.facebook.com/{fb_id}'.format(fb_id=fb_id) 30 | resp = requests.get(url, params={'fields': 'posts', 31 | 'access_token': access_token}) 32 | if resp.status_code == 200: 33 | return resp.json()['posts'] 34 | else: 35 | raise Exception('Facebook API request failed with code: {status_code}'.format(status_code=resp.status_code)) 36 | 37 | 38 | if __name__ == '__main__': 39 | fb_id = input('Facebook ID: ') if FB_ID is None else FB_ID 40 | access_token = input('Facebook Access Token: ') if FB_ACCESS_TOKEN is None else FB_ACCESS_TOKEN 41 | 42 | posts = fetch_posts(fb_id, access_token) 43 | 44 | c = Collection(posts['data']) 45 | 46 | print("==== posts of type 'status' ====") 47 | statuses = c.filter(type__exact='status') \ 48 | .select('message') 49 | pprint(list(statuses)) 50 | print() 51 | 52 | print("==== posts of type 'links' ====") 53 | links = c.filter(type__exact='link') \ 54 | .select('message') 55 | pprint(list(links)) 56 | print() 57 | 58 | print("==== posts with at least 1 likes ====") 59 | liked = c.filter(likes__count__gte=1) \ 60 | .select('message', 'likes') 61 | pprint(list(liked)) 62 | print() 63 | 64 | print("==== posts about Erlang ====") 65 | about_erlang = c.filter(message__icontains='erlang') \ 66 | .select('message') 67 | pprint(list(about_erlang)) 68 | print() 69 | 70 | print("==== posts created by the Twitter app ====") 71 | via_twitter = c.filter(application__name='Twitter') \ 72 | .select('message') 73 | pprint(list(via_twitter)) 74 | print() 75 | 76 | print("=== posts having a hashtag or a mention ===") 77 | p1, p2 = map(re.compile, [r'@.+', r'#.+']) 78 | tags_mentions = c.filter(Q(message__regex=p1) 79 | | 80 | Q(message__regex=p2)) \ 81 | .select('message', 'from__name') 82 | pprint(list(tags_mentions)) 83 | print() 84 | 85 | -------------------------------------------------------------------------------- /lookupy/dunderkey.py: -------------------------------------------------------------------------------- 1 | ## This module deals with code regarding handling the double 2 | ## underscore separated keys 3 | 4 | def dunderkey(*args): 5 | """Produces a nested key from multiple args separated by double 6 | underscore 7 | 8 | >>> dunderkey('a', 'b', 'c') 9 | >>> 'a__b__c' 10 | 11 | :param *args : *String 12 | :rtype : String 13 | """ 14 | return '__'.join(args) 15 | 16 | 17 | def dunder_partition(key): 18 | """Splits a dunderkey into 2 parts 19 | 20 | The first part is everything before the final double underscore 21 | The second part is after the final double underscore 22 | 23 | >>> dunder_partition('a__b__c') 24 | >>> ('a__b', 'c') 25 | 26 | :param neskey : String 27 | :rtype : 2 Tuple 28 | 29 | """ 30 | parts = key.rsplit('__', 1) 31 | return tuple(parts) if len(parts) > 1 else (parts[0], None) 32 | 33 | 34 | def dunder_init(key): 35 | """Returns the initial part of the dunder key 36 | 37 | >>> dunder_init('a__b__c') 38 | >>> 'a__b' 39 | 40 | :param neskey : String 41 | :rtype : String 42 | """ 43 | return dunder_partition(key)[0] 44 | 45 | 46 | def dunder_last(key): 47 | """Returns the last part of the dunder key 48 | 49 | >>> dunder_last('a__b__c') 50 | >>> 'c' 51 | 52 | :param neskey : String 53 | :rtype : String 54 | """ 55 | return dunder_partition(key)[1] 56 | 57 | 58 | def dunder_get(_dict, key): 59 | """Returns value for a specified dunderkey 60 | 61 | A "dunderkey" is just a fieldname that may or may not contain 62 | double underscores (dunderscores!) for referrencing nested keys in 63 | a dict. eg:: 64 | 65 | >>> data = {'a': {'b': 1}} 66 | >>> nesget(data, 'a__b') 67 | 1 68 | 69 | key 'b' can be referrenced as 'a__b' 70 | 71 | :param _dict : (dict) 72 | :param key : (str) that represents a first level or nested key in the dict 73 | :rtype : (mixed) value corresponding to the key 74 | 75 | """ 76 | parts = key.split('__', 1) 77 | try: 78 | result = _dict[parts[0]] 79 | except KeyError: 80 | return None 81 | else: 82 | return result if len(parts) == 1 else dunder_get(result, parts[1]) 83 | 84 | 85 | def undunder_keys(_dict): 86 | """Returns dict with the dunder keys converted back to nested dicts 87 | 88 | eg:: 89 | 90 | >>> undunder_keys({'a': 'hello', 'b__c': 'world'}) 91 | {'a': 'hello', 'b': {'c': 'world'}} 92 | 93 | :param _dict : (dict) flat dict 94 | :rtype : (dict) nested dict 95 | 96 | """ 97 | def f(key, value): 98 | parts = key.split('__') 99 | return { 100 | parts[0]: value if len(parts) == 1 else f(parts[1], value) 101 | } 102 | 103 | result = {} 104 | for r in [f(k, v) for k, v in _dict.items()]: 105 | rk = list(r.keys())[0] 106 | if rk not in result: 107 | result.update(r) 108 | else: 109 | result[rk].update(r[rk]) 110 | return result 111 | 112 | 113 | def dunder_truncate(_dict): 114 | """Returns dict with dunder keys truncated to only the last part 115 | 116 | In other words, replaces the dunder keys with just last part of 117 | it. In case many identical last parts are encountered, they are 118 | not truncated further 119 | 120 | eg:: 121 | 122 | >>> dunder_truncate({'a__p': 3, 'b__c': 'no'}) 123 | {'c': 'no', 'p': 3} 124 | >>> dunder_truncate({'a__p': 'yay', 'b__p': 'no', 'c__z': 'dunno'}) 125 | {'a__p': 'yay', 'b__p': 'no', 'z': 'dunno'} 126 | 127 | :param _dict : (dict) to flatten 128 | :rtype : (dict) flattened result 129 | 130 | """ 131 | keylist = list(_dict.keys()) 132 | def decide_key(k, klist): 133 | newkey = dunder_last(k) 134 | return newkey if list(map(dunder_last, klist)).count(newkey) == 1 else k 135 | original_keys = [decide_key(key, keylist) for key in keylist] 136 | return dict(zip(original_keys, _dict.values())) 137 | 138 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Lookupy 2 | ======= 3 | 4 | Lookupy is a Python library that provides a `Django 5 | `_ `QuerySet 6 | `_ like 7 | interface to query (filter and select) data (list of dictionaries). 8 | 9 | It actually started off as a library to parse and extract useful data 10 | out of HAR (HTTP Archive) files but along the way I felt that a 11 | generic library can be useful since I often find myself trying to get 12 | data out of JSON collections such as those obtained from facebook or 13 | github APIs. I choose to imitate the Django queryset API because of my 14 | familiarity with it. 15 | 16 | I don't use this library all the time but I do find it helpful when 17 | working with deeply nested json/dicts - the kind that Facebook, Github 18 | etc. APIs return. For everyday stuff I prefer Python's built-in 19 | functional constructs such as map, filter, list comprehensions. 20 | 21 | Requirements 22 | ------------ 23 | 24 | * Python [tested for 2.7 and 3.2] 25 | * `nose `_ 26 | [optional, for running tests] 27 | * `coverage.py `_ 28 | [optional, for test coverage] 29 | * `Tox `_ 30 | [optional, for building and testing on different versions of Python] 31 | 32 | 33 | Installation 34 | ------------ 35 | 36 | The simplest way to install this library is to use pip 37 | 38 | .. code-block:: bash 39 | 40 | $ pip install lookupy 41 | 42 | **Tip!** Consider installing inside a 43 | `virtualenv `_ 44 | 45 | 46 | Quick start 47 | ----------- 48 | 49 | Since this library is based on Django QuerySets, it would help to 50 | first understand how they work. In Django, QuerySets are used to 51 | construct SQL queries to fetch data from the database. Using the 52 | filter method of the QuerySet objects is equivalent to writing the 53 | *WHERE* clause in SQL. 54 | 55 | Applying the same concept to simple collections of data (lists of 56 | dicts), lookupy can be used to extract a subset of the data depending 57 | upon some criteria that is specified using what is known as the 58 | "lookup parameters". 59 | 60 | But first, we need to construct a *Collection* object out of the 61 | data set as follows, 62 | 63 | .. code-block:: pycon 64 | 65 | >>> from lookupy import Collection, Q 66 | >>> data = [{'framework': 'Django', 'language': 'Python', 'type': 'full-stack'}, 67 | ... {'framework': 'Flask', 'language': 'Python', 'type': 'micro'}, 68 | ... {'framework': 'Rails', 'language': 'Ruby', 'type': 'full-stack'}, 69 | ... {'framework': 'Sinatra', 'language': 'Ruby', 'type': 'micro'}, 70 | ... {'framework': 'Zend', 'language': 'PHP', 'type': 'full-stack'}, 71 | ... {'framework': 'Slim', 'language': 'PHP', 'type': 'micro'}] 72 | >>> c = Collection(data) 73 | 74 | In order to filter some data out of collection, we call the *filter* 75 | method passing our lookup parameters to it. 76 | 77 | .. code-block:: pycon 78 | 79 | >>> c.filter(framework__startswith='S') 80 | 81 | 82 | >>> list(c.filter(framework__startswith='S')) 83 | [{'framework': 'Sinatra', 'type': 'micro', 'language': 'Ruby'}, 84 | {'framework': 'Slim', 'type': 'micro', 'language': 'PHP'}] 85 | 86 | 87 | A lookup parameter is basically like a conditional clause and is of 88 | the form *__=* where ** is a key in the 89 | dict and ** is one of the predefined keywords that specify 90 | how to match the ** with the actual value corresponding to the 91 | key in each dict. See `list of lookup types 92 | <#supported-lookup-types>`_ 93 | 94 | Multiple lookups passed as args are by default combined using the 95 | *and* logical operator (*or* and *not* are also supported as we will 96 | see in a bit) 97 | 98 | .. code-block:: pycon 99 | 100 | >>> list(c.filter(framework__startswith='S', language__exact='Ruby')) 101 | [{'framework': 'Sinatra', 'type': 'micro', 'language': 'Ruby'}] 102 | 103 | 104 | For *or* and *not*, we can compose a complex lookup using *Q* objects 105 | and pass them as positional arguments along with our lookup parameters 106 | as keyword args. Not surprisingly, the bitwise and (*&*), or (*|*) and 107 | inverse (*~*) are overriden to act as logical *and*, *or* and *not* 108 | respectively (just the way it works in Django). 109 | 110 | .. code-block:: pycon 111 | 112 | >>> list(c.filter(Q(language__exact='Python') | Q(language__exact='Ruby'))) 113 | [{'framework': 'Django', 'language': 'Python', 'type': 'full-stack'}, 114 | {'framework': 'Flask', 'language': 'Python', 'type': 'micro'}, 115 | {'framework': 'Rails', 'language': 'Ruby', 'type': 'full-stack'}, 116 | {'framework': 'Sinatra', 'language': 'Ruby', 'type': 'micro'}] 117 | >>> list(c.filter(~Q(language__startswith='R'), framework__endswith='go')) 118 | [{'framework': 'Django', 'language': 'Python', 'type': 'full-stack'}] 119 | 120 | Lookupy also supports having the result contain only selected fields 121 | by providing the *select* method on the QuerySet objects. 122 | 123 | Calling the filter or select methods on a QuerySet returns another 124 | QuerySet so these calls can be chained together. Internally, filtering 125 | and selecting leverage Python's generators for lazy evaluation. Also, 126 | *QuerySet* and *Collection* both implement the `iterator protocol 127 | `_ so 128 | nothing is evaluated until consumption. 129 | 130 | .. code-block:: pycon 131 | 132 | >>> result = c.filter(Q(language__exact='Python') | Q(language__exact='Ruby')) \ 133 | .filter(framework__istartswith='s')) \ 134 | .select('framework') 135 | >>> for item in result: # <-- this is where filtering will happen 136 | ... print(item) 137 | ... 138 | [{'framework': 'Sinatra'}] 139 | 140 | For nested dicts, the key in the lookup parameters can be constructed 141 | using double underscores as *request__status__exact=404*. Finally, 142 | data can also be filtered by nested collection of key-value pairs 143 | using the same *Q* object. 144 | 145 | .. code-block:: pycon 146 | 147 | >>> data = [{'a': 'python', 'b': {'p': 1, 'q': 2}, 'c': [{'name': 'version', 'value': '3.4'}, {'name': 'author', 'value': 'Guido van Rossum'}]}, 148 | ... {'a': 'erlang', 'b': {'p': 3, 'q': 4}, 'c': [{'name': 'version', 'value': 'R16B01'}, {'name': 'author', 'y': 'Joe Armstrong'}]}] 149 | >>> c = Collection(data) 150 | >>> list(c.filter(b__q__gte=4)) 151 | [{'a': 'erlang', 'c': [{'name': 'version', 'value': 'R16B01'}, {'y': 'Joe Armstrong', 'name': 'author'}], 'b': {'q': 4, 'p': 3}}] 152 | >>> list(c.filter(c__filter=Q(name='version', value__contains='.'))) 153 | [{'a': 'python', 'c': [{'name': 'version', 'value': '3.4'}, {'name': 'author', 'value': 'Guido van Rossum'}], 'b': {'q': 2, 'p': 1}}] 154 | 155 | In the last example, we used the *Q* object to filter the original 156 | dict by nested collection of key-value pairs i.e. we queried for only 157 | those languages for which the version string contains a dot 158 | (*.*). Note that this is different from filtering the nested 159 | collections themselves. To do that, we can easily construct 160 | *Collection* objects for the child collections. 161 | 162 | See the *examples* subdirectory for more usage examples. 163 | 164 | 165 | Supported lookup types 166 | ---------------------- 167 | 168 | These are the currently supported lookup types, 169 | 170 | * **exact** exact equality (default) 171 | * **neq** inequality 172 | * **contains** containment 173 | * **icontains** insensitive containment 174 | * **in** membership 175 | * **startswith** string startswith 176 | * **istartswith** insensitive startswith 177 | * **endswith** string endswith 178 | * **iendswith** insensitive endswith 179 | * **gt** greater than 180 | * **gte** greater than or equal to 181 | * **lt** less than 182 | * **lte** less than or equal to 183 | * **regex** regular expression search 184 | * **filter** nested filter 185 | 186 | 187 | Gotchas! 188 | -------- 189 | 190 | 1. If a non-existent *key* is passed to *select*, then it will be 191 | included in the result with value *None* for all results. 192 | 193 | 2. If a non-existent *key* is passed to *filter*, then the lookup will 194 | always fail. At first, this doesn't seem consistent with the first 195 | point but it's done to keep the overall behaviour predictable 196 | e.g. If a non-existent key is used with *lt* lookup with integer, 197 | say *2*, as the val, then the lookup will always fail even though 198 | *None < 2 == True* in Python 2. Best is to just avoid such 199 | situations. 200 | 201 | 3. Because of the way *select* works at the moment, if chained with 202 | *filter* it should be called only after it and not before (unless the 203 | keys used for lookup are also being selected.) I plan to fix this in 204 | later releases. 205 | 206 | 207 | Running tests 208 | ------------- 209 | 210 | .. code-block:: bash 211 | 212 | $ make test 213 | 214 | To conveniently test under all environments (Python 2.7 and 3.2), run, 215 | 216 | .. code-block:: bash 217 | 218 | $ tox 219 | 220 | 221 | Todo 222 | ---- 223 | 224 | * Measure performance for larger data sets 225 | * Implement CLI for JSON files 226 | 227 | 228 | License 229 | ------- 230 | 231 | This library is provided as-is under the 232 | `MIT License `_ 233 | -------------------------------------------------------------------------------- /lookupy/lookupy.py: -------------------------------------------------------------------------------- 1 | """ 2 | lookupy.lookupy 3 | ~~~~~~~~~~~~~~~ 4 | 5 | This module consists of all of the functionality of the package 6 | 7 | """ 8 | 9 | import re 10 | from functools import partial 11 | 12 | from .dunderkey import dunder_get, dunder_partition, undunder_keys, dunder_truncate 13 | 14 | 15 | class QuerySet(object): 16 | """Provides an interface to filter data and select specific fields 17 | from the data 18 | 19 | QuerySet is used for filtering data and also selecting only 20 | relevant fields out of it. This object is internally created which 21 | means usually you, the user wouldn't need to create it. 22 | 23 | :param data: an iterable of dicts 24 | 25 | """ 26 | 27 | def __init__(self, data): 28 | self.data = data 29 | 30 | def filter(self, *args, **kwargs): 31 | """Filters data using the lookup parameters 32 | 33 | Lookup parameters can be passed as, 34 | 35 | 1. keyword arguments of type `field__lookuptype=value` where 36 | lookuptype specifies how to "query" eg:: 37 | 38 | >>> c.items.filter(language__contains='java') 39 | 40 | above will match all items where the language field 41 | contains the substring 'java' such as 'java', 42 | 'javascript'. Each look up is treated as a conditional 43 | clause and if multiple of them are passed, they are 44 | combined using logical the ``and`` operator 45 | 46 | For nested fields, double underscore can be used eg:: 47 | 48 | >>> data = [{'a': {'b': 3}}, {'a': {'b': 10}}] 49 | >>> c = Collection(data) 50 | >>> c.items.filter(a__b__gt=5) 51 | 52 | above lookup will match the 2nd element (b > 5) 53 | 54 | For the list of supported lookup parameter, see 55 | documentation on Github 56 | 57 | 2. pos arguments of the type ``field__lookuptype=Q(...)``. 58 | These can be useful to build conditional clauses that 59 | need to be combined using logical `or` or negated using 60 | `not` 61 | 62 | >>> c.items.filter(Q(language__exact='Python') 63 | | 64 | Q(language__exact='Ruby') 65 | 66 | above query will only filter the data where language is 67 | either 'Python' or 'Ruby' 68 | 69 | For more documentation see README on Github 70 | 71 | :param args : ``Q`` objects 72 | :param kwargs : lookup parameters 73 | :rtype : QuerySet 74 | 75 | """ 76 | return self.__class__(filter_items(self.data, *args, **kwargs)) 77 | 78 | def select(self, *args, **kwargs): 79 | """Selects specific fields of the data 80 | 81 | e.g. to select just the keys 'framework' and 'type' from many 82 | keys, :: 83 | 84 | >>> c.items.select('framework', 'type') 85 | 86 | 87 | :param args : field names to select 88 | :param kwargs : optional keyword args 89 | 90 | """ 91 | flatten = kwargs.pop('flatten', False) 92 | f = dunder_truncate if flatten else undunder_keys 93 | result = (f(d) for d in include_keys(self.data, args)) 94 | return self.__class__(result) 95 | 96 | def __iter__(self): 97 | for d in self.data: 98 | yield d 99 | 100 | 101 | # QuerySet given an alias for backward compatibility 102 | Collection = QuerySet 103 | 104 | 105 | ## filter and lookup functions 106 | 107 | def filter_items(items, *args, **kwargs): 108 | """Filters an iterable using lookup parameters 109 | 110 | :param items : iterable 111 | :param args : ``Q`` objects 112 | :param kwargs : lookup parameters 113 | :rtype : lazy iterable (generator) 114 | 115 | """ 116 | q1 = list(args) if args is not None else [] 117 | q2 = [Q(**kwargs)] if kwargs is not None else [] 118 | lookup_groups = q1 + q2 119 | pred = lambda item: all(lg.evaluate(item) for lg in lookup_groups) 120 | return (item for item in items if pred(item)) 121 | 122 | 123 | def lookup(key, val, item): 124 | """Checks if key-val pair exists in item using various lookup types 125 | 126 | The lookup types are derived from the `key` and then used to check 127 | if the lookup holds true for the item:: 128 | 129 | >>> lookup('request__url__exact', 'http://example.com', item) 130 | 131 | The above will return True if item['request']['url'] == 132 | 'http://example.com' else False 133 | 134 | :param key : (str) that represents the field name to find 135 | :param val : (mixed) object to match the value in the item against 136 | :param item : (dict) 137 | :rtype : (boolean) True if field-val exists else False 138 | 139 | """ 140 | init, last = dunder_partition(key) 141 | if last == 'exact': 142 | return dunder_get(item, init) == val 143 | elif last == 'neq': 144 | return dunder_get(item, init) != val 145 | elif last == 'contains': 146 | val = guard_str(val) 147 | return iff_not_none(dunder_get(item, init), lambda y: val in y) 148 | elif last == 'icontains': 149 | val = guard_str(val) 150 | return iff_not_none(dunder_get(item, init), lambda y: val.lower() in y.lower()) 151 | elif last == 'in': 152 | val = guard_iter(val) 153 | return dunder_get(item, init) in val 154 | elif last == 'startswith': 155 | val = guard_str(val) 156 | return iff_not_none(dunder_get(item, init), lambda y: y.startswith(val)) 157 | elif last == 'istartswith': 158 | val = guard_str(val) 159 | return iff_not_none(dunder_get(item, init), lambda y: y.lower().startswith(val.lower())) 160 | elif last == 'endswith': 161 | val = guard_str(val) 162 | return iff_not_none(dunder_get(item, init), lambda y: y.endswith(val)) 163 | elif last == 'iendswith': 164 | val = guard_str(val) 165 | return iff_not_none(dunder_get(item, init), lambda y: y.lower().endswith(val.lower())) 166 | elif last == 'gt': 167 | return iff_not_none(dunder_get(item, init), lambda y: y > val) 168 | elif last == 'gte': 169 | return iff_not_none(dunder_get(item, init), lambda y: y >= val) 170 | elif last == 'lt': 171 | return iff_not_none(dunder_get(item, init), lambda y: y < val) 172 | elif last == 'lte': 173 | return iff_not_none(dunder_get(item, init), lambda y: y <= val) 174 | elif last == 'regex': 175 | return iff_not_none(dunder_get(item, init), lambda y: re.search(val, y) is not None) 176 | elif last == 'filter': 177 | val = guard_Q(val) 178 | result = guard_list(dunder_get(item, init)) 179 | return len(list(filter_items(result, val))) > 0 180 | else: 181 | return dunder_get(item, key) == val 182 | 183 | 184 | ## Classes to compose compound lookups (Q object) 185 | 186 | class LookupTreeElem(object): 187 | """Base class for a child in the lookup expression tree""" 188 | 189 | def __init__(self): 190 | self.negate = False 191 | 192 | def evaluate(self, item): 193 | raise NotImplementedError 194 | 195 | def __or__(self, other): 196 | node = LookupNode() 197 | node.op = 'or' 198 | node.add_child(self) 199 | node.add_child(other) 200 | return node 201 | 202 | def __and__(self, other): 203 | node = LookupNode() 204 | node.add_child(self) 205 | node.add_child(other) 206 | return node 207 | 208 | 209 | class LookupNode(LookupTreeElem): 210 | """A node (element having children) in the lookup expression tree 211 | 212 | Typically it's any object composed of two ``Q`` objects eg:: 213 | 214 | >>> Q(language__neq='Ruby') | Q(framework__startswith='S') 215 | >>> ~Q(language__exact='PHP') 216 | 217 | """ 218 | 219 | def __init__(self): 220 | super(LookupNode, self).__init__() 221 | self.children = [] 222 | self.op = 'and' 223 | 224 | def add_child(self, child): 225 | self.children.append(child) 226 | 227 | def evaluate(self, item): 228 | """Evaluates the expression represented by the object for the item 229 | 230 | :param item : (dict) item 231 | :rtype : (boolean) whether lookup passed or failed 232 | 233 | """ 234 | results = map(lambda x: x.evaluate(item), self.children) 235 | result = any(results) if self.op == 'or' else all(results) 236 | return not result if self.negate else result 237 | 238 | def __invert__(self): 239 | newnode = LookupNode() 240 | for c in self.children: 241 | newnode.add_child(c) 242 | newnode.negate = not self.negate 243 | return newnode 244 | 245 | 246 | class LookupLeaf(LookupTreeElem): 247 | """Class for a leaf in the lookup expression tree""" 248 | 249 | def __init__(self, **kwargs): 250 | super(LookupLeaf, self).__init__() 251 | self.lookups = kwargs 252 | 253 | def evaluate(self, item): 254 | """Evaluates the expression represented by the object for the item 255 | 256 | :param item : (dict) item 257 | :rtype : (boolean) whether lookup passed or failed 258 | 259 | """ 260 | result = all(lookup(k, v, item) for k, v in self.lookups.items()) 261 | return not result if self.negate else result 262 | 263 | def __invert__(self): 264 | newleaf = LookupLeaf(**self.lookups) 265 | newleaf.negate = not self.negate 266 | return newleaf 267 | 268 | 269 | # alias LookupLeaf to Q 270 | Q = LookupLeaf 271 | 272 | 273 | ## functions that work on the keys in a dict 274 | 275 | def include_keys(items, fields): 276 | """Function to keep only specified fields in data 277 | 278 | Returns a list of dict with only the keys mentioned in the 279 | `fields` param:: 280 | 281 | >>> include_keys(items, ['request__url', 'response__status']) 282 | 283 | :param items : iterable of dicts 284 | :param fields : (list) fieldnames to keep 285 | :rtype : lazy iterable 286 | 287 | """ 288 | return (dict((f, dunder_get(item, f)) for f in fields) for item in items) 289 | 290 | 291 | ## Exceptions 292 | 293 | class LookupyError(Exception): 294 | """Base exception class for all exceptions raised by lookupy""" 295 | pass 296 | 297 | 298 | ## utility functions 299 | 300 | def iff(precond, val, f): 301 | """If and only if the precond is True 302 | 303 | Shortcut function for precond(val) and f(val). It is mainly used 304 | to create partial functions for commonly required preconditions 305 | 306 | :param precond : (function) represents the precondition 307 | :param val : (mixed) value to which the functions are applied 308 | :param f : (function) the actual function 309 | 310 | """ 311 | return False if not precond(val) else f(val) 312 | 313 | iff_not_none = partial(iff, lambda x: x is not None) 314 | 315 | 316 | def guard_type(classinfo, val): 317 | if not isinstance(val, classinfo): 318 | raise LookupyError('Value not a {classinfo}'.format(classinfo=classinfo)) 319 | return val 320 | 321 | guard_str = partial(guard_type, str) 322 | guard_list = partial(guard_type, list) 323 | guard_Q = partial(guard_type, Q) 324 | 325 | def guard_iter(val): 326 | try: 327 | iter(val) 328 | except TypeError: 329 | raise LookupyError('Value not an iterable') 330 | else: 331 | return val 332 | 333 | 334 | if __name__ == '__main__': 335 | pass 336 | 337 | -------------------------------------------------------------------------------- /lookupy/tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | lookupy.tests 3 | ~~~~~~~~~~~~~ 4 | 5 | This module contains tests for the lookupy module written using 6 | nose to be run using:: 7 | 8 | $ nosetests -v 9 | 10 | """ 11 | 12 | import re 13 | from nose.tools import assert_list_equal, assert_equal, assert_raises 14 | 15 | from .lookupy import filter_items, lookup, include_keys, Q, QuerySet, \ 16 | Collection, LookupyError 17 | from .dunderkey import dunderkey, dunder_partition, dunder_init, dunder_last, \ 18 | dunder_get, undunder_keys, dunder_truncate 19 | 20 | 21 | entries_fixtures = [{'request': {'url': 'http://example.com', 'headers': [{'name': 'Connection', 'value': 'Keep-Alive'}]}, 22 | 'response': {'status': 404, 'headers': [{'name': 'Date', 'value': 'Thu, 13 Jun 2013 06:43:14 GMT'}, 23 | {'name': 'Content-Type', 'value': 'text/html'}]}}, 24 | {'request': {'url': 'http://example.org', 'headers': [{'name': 'Connection', 'value': 'Keep-Alive'}]}, 25 | 'response': {'status': 200, 'headers': [{'name': 'Date', 'value': 'Thu, 13 Jun 2013 06:43:14 GMT'}, 26 | {'name': 'Content-Type', 'value': 'text/html'}]}}, 27 | {'request': {'url': 'http://example.com/myphoto.jpg', 'headers': [{'name': 'Connection', 'value': 'Keep-Alive'}]}, 28 | 'response': {'status': 200, 'headers': [{'name': 'Date', 'value': 'Thu, 13 Jun 2013 06:43:14 GMT'}, 29 | {'name': 'Content-Type', 'value': 'image/jpg'}]}}] 30 | 31 | 32 | def fe(entries, *args, **kwargs): 33 | return list(filter_items(entries, *args, **kwargs)) 34 | 35 | 36 | def ik(entries, fields): 37 | return list(include_keys(entries, fields)) 38 | 39 | 40 | ## Tests 41 | 42 | 43 | 44 | 45 | def test_Collection(): 46 | c = Collection(entries_fixtures) 47 | assert_list_equal(list(c), entries_fixtures) 48 | assert_list_equal(list(c), entries_fixtures) 49 | 50 | 51 | def test_Q(): 52 | entries = entries_fixtures 53 | q1 = Q(response__status__exact=404, request__url__contains='.com') 54 | assert q1.evaluate(entries[0]) 55 | 56 | # test with negation 57 | q2 = ~Q(response__status__exact=404) 58 | assert q2.evaluate(entries[1]) 59 | # test multiple application of negation 60 | assert not (~q2).evaluate(entries[1]) 61 | 62 | q3 = Q(response__status=200) 63 | assert not (q1 & q3).evaluate(entries[0]) 64 | assert (q1 | q3).evaluate(entries[0]) 65 | assert (~(q1 & q3)).evaluate(entries[0]) 66 | 67 | assert_list_equal(list(((Q(request__url__endswith='.jpg') | Q(response__status=404)).evaluate(e) 68 | for e in entries)), 69 | [True, False, True]) 70 | 71 | assert_list_equal(list(((~Q(request__url__endswith='.jpg') | Q(response__status=404)).evaluate(e) 72 | for e in entries)), 73 | [True, True, False]) 74 | 75 | 76 | def test_lookup(): 77 | entry1, entry2, entry3 = entries_fixtures 78 | # exact -- works for strings and int 79 | assert lookup('request__url__exact', 'http://example.com', entry1) 80 | assert not lookup('request_url__exact', 'http://example.org', entry1) 81 | assert lookup('response__status__exact', 404, entry1) 82 | assert not lookup('response__status__exact', 404, entry2) 83 | assert lookup('response_unknown__exact', None, entry1) 84 | 85 | # neq -- works for strings and ints 86 | assert not lookup('request__url__neq', 'http://example.com', entry1) 87 | assert lookup('request_url__neq', 'http://example.org', entry1) 88 | assert not lookup('response__status__neq', 404, entry1) 89 | assert lookup('response__status__neq', 404, entry2) 90 | assert not lookup('response_unknown__neq', None, entry1) 91 | 92 | # contains -- works for strings, else raises error 93 | assert lookup('request__url__contains', '.com', entry1) 94 | assert not lookup('request__url__contains', 'www', entry1) 95 | assert_raises(LookupyError, lookup, 'response__status__contains', 96 | 2, entry2) 97 | assert_raises(LookupyError, lookup, 'response__unknown__contains', 98 | None, entry2) 99 | 100 | # icontains -- works for strings, else raises error 101 | assert lookup('request__url__icontains', 'EXAMPLE', entry1) 102 | assert not lookup('request__url__icontains', 'www', entry1) 103 | assert_raises(LookupyError, lookup, 'response__status__icontains', 104 | 2, entry2) 105 | assert_raises(LookupyError, lookup, 106 | 'response__unknown__icontains', None, entry2) 107 | 108 | # in -- works for strings and lists, else raises error 109 | assert lookup('request__url__in', ['http://example.com', 110 | 'http://blog.example.com'], entry1) 111 | 112 | assert lookup('response__status__in', [400, 200], entry2) 113 | assert not lookup('response__status__in', [], entry2) 114 | assert lookup('request__url__in', 'http://example.com/?q=hello', entry1) 115 | assert_raises(LookupyError, lookup, 'response__status__in', 404, entry1) 116 | 117 | # startswith -- works for strings, else raises error 118 | assert lookup('request__url__startswith', 'http://', entry1) 119 | assert not lookup('request__url__startswith', 'HTTP://', entry1) 120 | assert_raises(LookupyError, lookup, 121 | 'response__status__startswith', 4, entry1) 122 | 123 | # istartswith -- works for strings, else raises error 124 | assert lookup('request__url__istartswith', 'http://', entry1) 125 | assert lookup('request__url__istartswith', 'HTTP://', entry1) 126 | assert_raises(LookupyError, lookup, 127 | 'response__status__istartswith', 4, entry1) 128 | 129 | # endswith -- works for strings, else raises error 130 | assert lookup('request__url__endswith', '.jpg', entry3) 131 | assert not lookup('request__url__endswith', '.JPG', entry3) 132 | assert_raises(LookupyError, lookup, 'response__status__endswith', 133 | 0, entry3) 134 | 135 | # iendswith -- works for strings, else raises error 136 | assert lookup('request__url__iendswith', '.jpg', entry3) 137 | assert lookup('request__url__iendswith', '.JPG', entry3) 138 | assert_raises(LookupyError, lookup, 'response__status__iendswith', 139 | 0, entry3) 140 | 141 | # gt -- works for strings and int 142 | assert lookup('response__status__gt', 200, entry1) 143 | assert not lookup('response__status__gt', 404, entry1) 144 | assert lookup('request__url__gt', 'ftp://example.com', entry1) 145 | assert not lookup('request__url__gt', 'http://example.com', entry1) 146 | 147 | # gte -- works for strings and int 148 | assert lookup('response__status__gte', 200, entry1) 149 | assert lookup('response__status__gte', 404, entry1) 150 | assert lookup('request__url__gte', 'ftp://example.com', entry1) 151 | assert lookup('request__url__gte', 'http://example.com', entry1) 152 | 153 | # lt -- works for strings and int 154 | assert lookup('response__status__lt', 301, entry2) 155 | assert not lookup('response__status__lt', 200, entry2) 156 | assert lookup('request__url__lt', 'ws://example.com', entry2) 157 | assert not lookup('request__url__lt', 'http://example.org', entry2) 158 | 159 | # lte -- works for strings and int 160 | assert lookup('response__status__lte', 301, entry2) 161 | assert lookup('response__status__lte', 200, entry2) 162 | assert lookup('request__url__lte', 'ws://example.com', entry2) 163 | assert lookup('request__url__lte', 'http://example.org', entry2) 164 | 165 | # regex -- works for compiled patterns and strings 166 | pattern = r'^http:\/\/.+g$' 167 | assert lookup('request__url__regex', pattern, entry2) 168 | assert lookup('request__url__regex', pattern, entry3) 169 | assert not lookup('request__url__regex', pattern, entry1) 170 | compiled_pattern = re.compile(pattern) 171 | assert lookup('request__url__regex', compiled_pattern, entry2) 172 | assert lookup('request__url__regex', compiled_pattern, entry3) 173 | assert not lookup('request__url__regex', compiled_pattern, entry1) 174 | 175 | # filter -- works for Q objects, else raises error 176 | assert lookup('response__headers__filter', 177 | Q(name__exact='Content-Type', value__exact='image/jpg'), 178 | entry3) 179 | assert not lookup('response__headers__filter', 180 | Q(name__exact='Content-Type', value__exact='text/html'), 181 | entry3) 182 | assert_raises(LookupyError, lookup, 'response__headers__filter', 183 | 0, entry3) 184 | assert_raises(LookupyError, lookup, 'response__headers__filter', 185 | "hello", entry3) 186 | assert_raises(LookupyError, lookup, 'response__headers__filter', 187 | None, entry3) 188 | assert_raises(LookupyError, lookup, 'response__headers__filter', 189 | {'a': 'b'}, entry3) 190 | assert_raises(LookupyError, lookup, 'response__status__filter', 191 | Q(name__exact='Content-Type', value__exact='image/jpg'), 192 | entry3) 193 | 194 | # nothing -- works for strings and int 195 | assert lookup('request__url', 'http://example.com', entry1) 196 | assert not lookup('request_url', 'http://example.org', entry1) 197 | assert lookup('response__status', 404, entry1) 198 | assert not lookup('response__status', 404, entry2) 199 | assert lookup('response_unknown', None, entry1) 200 | 201 | 202 | def test_filter_items(): 203 | entries = entries_fixtures 204 | 205 | # when no lookup kwargs passed, all entries are returned 206 | assert_list_equal(fe(entries), entries) 207 | 208 | # simple 1st level lookups 209 | assert_list_equal(fe(entries, request__url='http://example.com'), entries[0:1]) 210 | assert_list_equal(fe(entries, response__status=200), entries[1:]) 211 | assert len(fe(entries, response__status=405)) == 0 212 | 213 | # testing compund lookups 214 | assert len(fe(entries, Q(request__url__exact='http://example.org'))) == 1 215 | assert len(fe(entries, 216 | Q(request__url__exact='http://example.org', response__status=200) 217 | | 218 | Q(request__url__endswith='.com', response__status=404))) == 2 219 | 220 | assert len(fe(entries, 221 | ~Q(request__url__exact='http://example.org', response__status__gte=500) 222 | | 223 | Q(request__url__endswith='.com', response__status=404))) == 3 224 | 225 | assert len(fe(entries, 226 | ~Q(request__url__exact='http://example.org', response__status__gte=500) 227 | | 228 | Q(request__url__endswith='.com', response__status=404), 229 | response__status__exact=200)) == 2 230 | 231 | 232 | def test_include_keys(): 233 | entries = entries_fixtures 234 | assert_list_equal(ik(entries, ['request']), 235 | [{'request': {'url': 'http://example.com', 'headers': [{'name': 'Connection', 'value': 'Keep-Alive'}]}}, 236 | {'request': {'url': 'http://example.org', 'headers': [{'name': 'Connection', 'value': 'Keep-Alive'}]}}, 237 | {'request': {'url': 'http://example.com/myphoto.jpg', 'headers': [{'name': 'Connection', 'value': 'Keep-Alive'}]}}]) 238 | 239 | assert_list_equal(ik(entries, ['response__status']), 240 | [{'response__status': 404}, 241 | {'response__status': 200}, 242 | {'response__status': 200}]) 243 | 244 | # when an empty list is passed as fields 245 | assert_list_equal(ik(entries, []), [{},{},{}]) 246 | 247 | # when a non-existent key is passed in fields 248 | assert_list_equal(ik(entries, ['response__status', 'cookies']), 249 | [{'response__status': 404, 'cookies': None}, 250 | {'response__status': 200, 'cookies': None}, 251 | {'response__status': 200, 'cookies': None}]) 252 | 253 | 254 | def test_Collection_QuerySet(): 255 | data = [{'framework': 'Django', 'language': 'Python', 'type': 'full-stack'}, 256 | {'framework': 'Flask', 'language': 'Python', 'type': 'micro'}, 257 | {'framework': 'Rails', 'language': 'Ruby', 'type': 'full-stack'}, 258 | {'framework': 'Sinatra', 'language': 'Ruby', 'type': 'micro'}, 259 | {'framework': 'Zend', 'language': 'PHP', 'type': 'full-stack'}, 260 | {'framework': 'Slim', 'language': 'PHP', 'type': 'micro'}] 261 | c = Collection(data) 262 | r1 = c.filter(framework__startswith='S') 263 | assert isinstance(r1, QuerySet) 264 | assert len(list(r1)) == 2 265 | r2 = c.filter(Q(language__exact='Python') | Q(language__exact='Ruby')) 266 | assert len(list(r2)) == 4 267 | r3 = c.filter(language='PHP') 268 | assert_list_equal(list(r3.select('framework', 'type')), 269 | [{'framework': 'Zend', 'type': 'full-stack'}, 270 | {'framework': 'Slim', 'type': 'micro'}]) 271 | r4 = c.filter(Q(language__exact='Python') | Q(language__exact='Ruby')) 272 | assert_list_equal(list(r4.select('framework')), 273 | [{'framework': 'Django'}, 274 | {'framework': 'Flask'}, 275 | {'framework': 'Rails'}, 276 | {'framework': 'Sinatra'}]) 277 | # :todo: test with flatten=True 278 | r5 = c.filter(framework__startswith='S').select('framework', 'somekey') 279 | assert_list_equal(list(r5), 280 | [{'framework': 'Sinatra', 'somekey': None}, 281 | {'framework': 'Slim', 'somekey': None}]) 282 | 283 | 284 | ## nesdict tests 285 | 286 | def test_dunderkey(): 287 | assert dunderkey('a', 'b', 'c') == 'a__b__c' 288 | assert dunderkey('a') == 'a' 289 | assert dunderkey('name', 'school_name') == 'name__school_name' 290 | 291 | 292 | def test_dunder_partition(): 293 | assert dunder_partition('a__b') == ('a', 'b') 294 | assert dunder_partition('a__b__c') == ('a__b', 'c') 295 | assert dunder_partition('a') == ('a', None) 296 | 297 | 298 | def test_dunder_init(): 299 | assert dunder_init('a__b') == 'a' 300 | assert dunder_init('a__b__c') == 'a__b' 301 | assert dunder_init('a') == 'a' 302 | 303 | 304 | def test_dunder_last(): 305 | assert dunder_last('a__b') == 'b' 306 | assert dunder_last('a__b__c') == 'c' 307 | assert dunder_last('a') == None 308 | 309 | 310 | def test_dunder_get(): 311 | d = dict([('a', 'A'), 312 | ('p', {'q': 'Q'}), 313 | ('x', {'y': {'z': 'Z'}})]) 314 | assert dunder_get(d, 'a') == 'A' 315 | assert dunder_get(d, 'p__q') == 'Q' 316 | assert dunder_get(d, 'x__y__z') == 'Z' 317 | 318 | 319 | def test_undunder_keys(): 320 | entry = {'request__url': 'http://example.com', 'request__headers': [{'name': 'Connection', 'value': 'Keep-Alive',}], 321 | 'response__status': 404, 'response__headers': [{'name': 'Date', 'value': 'Thu, 13 Jun 2013 06:43:14 GMT'}]} 322 | assert_equal(undunder_keys(entry), 323 | {'request': {'url': 'http://example.com', 'headers': [{'name': 'Connection', 'value': 'Keep-Alive',}]}, 324 | 'response': {'status': 404, 'headers': [{'name': 'Date', 'value': 'Thu, 13 Jun 2013 06:43:14 GMT'}]}}) 325 | 326 | 327 | def test_dunder_truncate(): 328 | entry = {'request__url': 'http://example.com', 'request__headers': [{'name': 'Connection', 'value': 'Keep-Alive',}], 329 | 'response__status': 404, 'response__headers': [{'name': 'Date', 'value': 'Thu, 13 Jun 2013 06:43:14 GMT'}]} 330 | assert_equal(dunder_truncate(entry), 331 | {'url': 'http://example.com', 332 | 'request__headers': [{'name': 'Connection', 'value': 'Keep-Alive',}], 333 | 'status': 404, 334 | 'response__headers': [{'name': 'Date', 'value': 'Thu, 13 Jun 2013 06:43:14 GMT'}]}) 335 | 336 | --------------------------------------------------------------------------------