├── .gitignore ├── LICENSE ├── README.md ├── moment ├── __init__.py ├── base.py ├── bitevents.py ├── collections.py ├── compat.py ├── conf.py ├── contrib │ ├── __init__.py │ └── django.py ├── counters.py ├── keys.py ├── lua.py ├── tests.py ├── timelines.py └── utils.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | venv/ 56 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Maxim Kamenkov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # redis-moment 2 | A powerful analytics python library for Redis. 3 | 4 | Installation 5 | ------------ 6 | The easiest way to install the latest version 7 | is by using pip/easy_install to pull it from PyPI: 8 | 9 | pip install redis-moment 10 | 11 | You may also use Git to clone the repository from 12 | Github and install it manually: 13 | 14 | git clone https://github.com/caxap/redis-moment.git 15 | python setup.py install 16 | 17 | Features 18 | -------- 19 | 1. Advanced data structures optimized for event crunching: 20 | - Events (Cohort analytics) 21 | - Counters 22 | - Timelines 23 | - Time Indexed Keys 24 | - Sequences 25 | 2. Partitioning by hour, day, week, month and year. 26 | 3. Pluggable serialization (default: json, pickle, msgpack) 27 | 4. Multiple Redis connections (with aliasing) 28 | 5. Key namespacing 29 | 6. Local caching (LRU lib requred) 30 | 7. Integration with Django 31 | 32 | 33 | ### Connections 34 | 35 | ```python 36 | from moment import conf 37 | 38 | # Register default connection 39 | conf.register_connection() 40 | 41 | # Register some other connection 42 | conf.register_connection(alias='analytics', host='localhost', port=6379) 43 | 44 | analytics_conn = conf.get_connection('analytics') 45 | ``` 46 | 47 | ### Sequence 48 | 49 | Use Sequence to convert symbolic identifier to sequential ids. Event component uses Sequence under the hood. Also Sequence optionaly holds cache of recenly created ids. 50 | 51 | ```python 52 | from moment import Sequence 53 | 54 | users = Sequence('users') 55 | id1 = seq.sequential_id('user1') 56 | id_one = seq.sequential_id('user1') # will not create new id, and return already assigned value 57 | id2 = seq.sequential_id('user2') 58 | 59 | assert id1 == 1 60 | assert id1 == id_one 61 | assert id2 == 2 62 | assert 'user1' in users and 'user2' in users 63 | ``` 64 | 65 | ### Events 66 | 67 | Events makes it possible to implement real-time, highly scalable analytics that can track actions for millions of users in a very little amount of memory. With events you can track active users, user retension, user churn, CTR of user actions and more. You can track events per hour, day, week, month and year. 68 | 69 | ```python 70 | from moment import record_events, DayEvent, MonthEvent 71 | 72 | # We whant to track active users by day and month. Mark `user1` & `user2` as active. 73 | record_events(['user1', 'user2'], 'users:active', ['day', 'month'], sequence='users') 74 | 75 | 76 | # Has `user1` been online today? This month? 77 | active_today = DayEvent('users:active', sequence='users') 78 | active_this_month = MonthEvent('users:active', sequence='users') 79 | 80 | assert 'user1' in active_today, 'should be active today' 81 | assert 'user1' in active_this_month, 'should be active this month' 82 | 83 | 84 | # Track some events: 85 | record_events('user1', ['event1', 'event2'], ['day', 'month'], sequence='users') 86 | record_events('user2', ['event2', 'event3'], ['day', 'month'], sequence='users') 87 | 88 | # Inspect recorded events for today 89 | e1 = DayEvent('event1', sequence='users') 90 | e2 = DayEvent('event2', sequence='users') 91 | 92 | assert len(e1) == 1 93 | assert len(e2) == 2 94 | assert 'user1' in e1 and 'user1' in e2 95 | assert 'user2' in e2 96 | assert len(e1 & e2) == 1 97 | assert len(e1 - e2) == 0 98 | ``` 99 | 100 | #### More docs comming soon... 101 | -------------------------------------------------------------------------------- /moment/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from . import conf # noqa 5 | from .base import * 6 | from .bitevents import * 7 | from .collections import * 8 | from .counters import * 9 | from .timelines import * 10 | 11 | 12 | __version__ = '0.0.6' 13 | -------------------------------------------------------------------------------- /moment/base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import calendar 5 | import inspect 6 | from datetime import datetime, date, timedelta 7 | from . import conf 8 | from .utils import not_none, add_month, iso_to_gregorian 9 | 10 | __all__ = ['Base', 'BaseHour', 'BaseDay', 'BaseMonth', 'BaseWeek', 'BaseYear'] 11 | 12 | 13 | def _key(name, namespace=None, prefix=None, delim=':'): 14 | """ 15 | Generates full redis key with `prefix` and optional `namespace`. 16 | 17 | Example :: 18 | 19 | _key('event', 'ns', 'prefix', '-') == 'prefix-ns-event' 20 | _key('event', ns) == 'spm:ns:event' 21 | _key('event') == 'spm:event' 22 | """ 23 | prefix = prefix or conf.MOMENT_KEY_PREFIX 24 | return (delim or ':').join(filter(None, [prefix, namespace, name])) 25 | 26 | 27 | def _require_defined(parent_cls, instance, name, kind='property', 28 | raise_cls=NotImplementedError): 29 | if not hasattr(instance, name): 30 | parent_name = parent_cls.__name__ # noqa 31 | child_name = instance.__class__.__name__ # noqa 32 | msg = ("`{child_name}` subclass of `{parent_name}` should define " 33 | "`{name}` {kind}.".format(**locals())) 34 | raise raise_cls(msg) 35 | 36 | 37 | class MixinPeriod(object): 38 | 39 | def next(self): 40 | return self.delta(value=1) 41 | 42 | def prev(self): 43 | return self.delta(value=-1) 44 | 45 | def delta(self, value): 46 | _require_defined(MixinPeriod, self, 'delta', 'method') 47 | 48 | 49 | class MixinClonable(object): 50 | 51 | clonable_attrs = [] 52 | 53 | def get_clonable_attrs(self): 54 | all_clonable_attrs = [] 55 | for base in inspect.getmro(self.__class__): 56 | attrs = getattr(base, 'clonable_attrs', None) 57 | if attrs: 58 | all_clonable_attrs.extend(attrs) 59 | return list(set(all_clonable_attrs)) 60 | 61 | def clone(self, **initials): 62 | names = self.get_clonable_attrs() 63 | attrs = {n: getattr(self, n) for n in names} if names else {} 64 | attrs = dict(attrs, **initials) 65 | instance = self.__class__(self.name, client=self.client) 66 | for name, value in attrs.items(): 67 | setattr(instance, name, value) 68 | return instance 69 | 70 | 71 | class MixinSerializable(object): 72 | 73 | serializer = None 74 | 75 | def dumps(self, value): 76 | return self.serializer.dumps(value) 77 | 78 | def loads(self, value): 79 | return self.serializer.loads(value) 80 | 81 | 82 | class Base(MixinClonable): 83 | 84 | def __init__(self, name, client='default'): 85 | self.name = name 86 | self.client = client 87 | 88 | def client(): 89 | def fget(self): 90 | return self._client 91 | 92 | def fset(self, client): 93 | """ Automatically resolve connection by alias. """ 94 | self._client = conf.get_connection(client) 95 | return locals() 96 | 97 | client = property(**client()) 98 | 99 | @property 100 | def key(self): 101 | """ 102 | Generates full redis key with `prefix` and optional `namespace`. 103 | """ 104 | _require_defined(Base, self, 'key_format') 105 | base_key = self.key_format.format(self=self) 106 | return _key(base_key, getattr(self, 'namespace', None)) 107 | 108 | def delete(self): 109 | self.client.delete(self.key) 110 | 111 | def expire(self, ttl): 112 | self.client.expire(self.key, ttl) 113 | 114 | def __bool__(self): 115 | return self.client.exists(self.key) 116 | 117 | __nonzero__ = __bool__ 118 | 119 | def __eq__(self, other): 120 | other_key = getattr(other, 'key', None) 121 | return other_key is not None and self.key == other_key 122 | 123 | def __ne__(self, other): 124 | return not self.__eq__(other) 125 | 126 | def __str__(self): 127 | return '<%s: %s>' % (self.__class__.__name__, self.key) 128 | 129 | __repr__ = __str__ 130 | 131 | 132 | class BaseHour(Base, MixinPeriod): 133 | 134 | # Example: 'active:2015-03-13-09' 135 | key_format = '{self.name}:{self.year:02d}-{self.month:02d}-{self.day:02d}-{self.hour:02d}' 136 | clonable_attrs = ['year', 'month', 'day', 'hour'] 137 | 138 | @classmethod 139 | def from_date(cls, name, dt=None, client='default', **kwargs): 140 | if dt is None: 141 | dt = datetime.utcnow() 142 | return cls(name, dt.year, dt.month, dt.day, dt.hour, client, **kwargs) 143 | 144 | def __init__(self, name, year=None, month=None, day=None, hour=None, 145 | client='default', **kwargs): 146 | super(BaseHour, self).__init__(name, client, **kwargs) 147 | self.set_period(year, month, day, hour) 148 | 149 | def set_period(self, year=None, month=None, day=None, hour=None): 150 | now = datetime.utcnow() 151 | self.year = not_none(year, now.year) 152 | self.month = not_none(month, now.month) 153 | self.day = not_none(day, now.day) 154 | self.hour = not_none(hour, now.hour) 155 | 156 | def delta(self, value): 157 | dt = datetime(self.year, self.month, self.day, self.hour) + timedelta(hours=value) 158 | return self.clone(year=dt.year, month=dt.month, day=dt.day, hour=dt.hour) 159 | 160 | def period_start(self): 161 | return datetime(self.year, self.month, self.day, self.hour) 162 | 163 | def period_end(self): 164 | return datetime(self.year, self.month, self.day, self.hour, 59, 59, 999999) 165 | 166 | 167 | class BaseDay(Base, MixinPeriod): 168 | 169 | # Example: 'active:2015-03-13' 170 | key_format = '{self.name}:{self.year}-{self.month:02d}-{self.day:02d}' 171 | clonable_attrs = ['year', 'month', 'day'] 172 | 173 | @classmethod 174 | def from_date(cls, name, dt=None, client='default', **kwargs): 175 | if dt is None: 176 | dt = datetime.utcnow() 177 | return cls(name, year=dt.year, month=dt.month, day=dt.day, client=client, **kwargs) 178 | 179 | def __init__(self, name, year=None, month=None, day=None, client='default', **kwargs): 180 | super(BaseDay, self).__init__(name, client, **kwargs) 181 | self.set_period(year, month, day) 182 | 183 | def set_period(self, year=None, month=None, day=None): 184 | now = datetime.utcnow() 185 | self.year = not_none(year, now.year) 186 | self.month = not_none(month, now.month) 187 | self.day = not_none(day, now.day) 188 | 189 | def delta(self, value): 190 | dt = date(self.year, self.month, self.day) + timedelta(days=value) 191 | return self.clone(year=dt.year, month=dt.month, day=dt.day) 192 | 193 | def period_start(self): 194 | return datetime(self.year, self.month, self.day) 195 | 196 | def period_end(self): 197 | return datetime(self.year, self.month, self.day, 23, 59, 59, 999999) 198 | 199 | 200 | class BaseMonth(Base, MixinPeriod): 201 | 202 | # Example: 'active:2015-03' 203 | key_format = '{self.name}:{self.year}-{self.month:02d}' 204 | clonable_attrs = ['year', 'month'] 205 | 206 | @classmethod 207 | def from_date(cls, name, dt=None, client='default', **kwargs): 208 | if dt is None: 209 | dt = datetime.utcnow() 210 | return cls(name, dt.year, dt.month, client, **kwargs) 211 | 212 | def __init__(self, name, year=None, month=None, client='default', **kwargs): 213 | super(BaseMonth, self).__init__(name, client, **kwargs) 214 | self.set_period(year, month) 215 | 216 | def set_period(self, year=None, month=None): 217 | now = datetime.utcnow() 218 | self.year = not_none(year, now.year) 219 | self.month = not_none(month, now.month) 220 | 221 | def delta(self, value): 222 | year, month = add_month(self.year, self.month, value) 223 | return self.clone(year=year, month=month) 224 | 225 | def period_start(self): 226 | return datetime(self.year, self.month, 1) 227 | 228 | def period_end(self): 229 | _, day = calendar.monthrange(self.year, self.month) 230 | return datetime(self.year, self.month, day, 23, 59, 59, 999999) 231 | 232 | 233 | class BaseWeek(Base, MixinPeriod): 234 | 235 | # Example: 'active:2015-W35' 236 | key_format = '{self.name}:{self.year}-W{self.week:02d}' 237 | clonable_attrs = ['year', 'week'] 238 | 239 | @classmethod 240 | def from_date(cls, name, dt=None, client='default', **kwargs): 241 | if dt is None: 242 | dt = datetime.utcnow() 243 | dt_year, dt_week, _ = dt.isocalendar() 244 | return cls(name, dt_year, dt_week, client, **kwargs) 245 | 246 | def __init__(self, name, year=None, week=None, client='default', **kwargs): 247 | super(BaseWeek, self).__init__(name, client, **kwargs) 248 | self.set_period(year, week) 249 | 250 | def set_period(self, year=None, week=None): 251 | now = datetime.utcnow() 252 | now_year, now_week, _ = now.isocalendar() 253 | self.year = not_none(year, now_year) 254 | self.week = not_none(week, now_week) 255 | 256 | def delta(self, value): 257 | dt = iso_to_gregorian(self.year, self.week + value, 1) 258 | year, week, _ = dt.isocalendar() 259 | return self.__class__(self.name, year, week, self.client) 260 | 261 | def period_start(self): 262 | s = iso_to_gregorian(self.year, self.week, 1) # mon 263 | return datetime(s.year, s.month, s.day) 264 | 265 | def period_end(self): 266 | e = iso_to_gregorian(self.year, self.week, 7) # mon 267 | return datetime(e.year, e.month, e.day, 23, 59, 59, 999999) 268 | 269 | 270 | class BaseYear(Base, MixinPeriod): 271 | 272 | # Example: 'active:2015-W35' 273 | key_format = '{self.name}:{self.year}' 274 | clonable_attrs = ['year'] 275 | 276 | @classmethod 277 | def from_date(cls, name, dt=None, client='default', **kwargs): 278 | if dt is None: 279 | dt = datetime.utcnow() 280 | return cls(name, dt.year, client, **kwargs) 281 | 282 | def __init__(self, name, year=None, client='default', **kwargs): 283 | super(BaseYear, self).__init__(name, client, **kwargs) 284 | self.set_period(year) 285 | 286 | def set_period(self, year=None): 287 | now = datetime.utcnow() 288 | self.year = not_none(year, now.year) 289 | 290 | def delta(self, value): 291 | return self.clone(year=self.year + value) 292 | 293 | def period_start(self): 294 | return datetime(self.year, 1, 1) 295 | 296 | def period_end(self): 297 | return datetime(self.year, 12, 31, 23, 59, 59, 999999) 298 | -------------------------------------------------------------------------------- /moment/bitevents.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import itertools 4 | from datetime import datetime 5 | from . import conf 6 | from .base import _key, Base, BaseHour, BaseDay, BaseWeek, BaseMonth, BaseYear 7 | from .collections import BaseSequence 8 | from .lua import msetbit 9 | 10 | 11 | __all__ = ['EVENT_NAMESPACE', 'EVENT_ALIASES', 'SEQUENCE_NAMESPACE', 12 | 'record_events', 'delete_temporary_bitop_keys', 'Sequence', 13 | 'Event', 'HourEvent', 'DayEvent', 'MonthEvent', 'WeekEvent', 14 | 'YearEvent', 'Or', 'And', 'Xor', 'Not', 'LDiff'] 15 | 16 | 17 | EVENT_NAMESPACE = 'evt' 18 | SEQUENCE_NAMESPACE = 'seq' 19 | 20 | 21 | def record_events(uuids, event_names, event_types=None, dt=None, client='default', 22 | sequence=None): 23 | """ 24 | Records events for hours, days, weeks and months. 25 | 26 | Examples:: 27 | 28 | seq = Sequence('sequence1') 29 | record_events('foo_id', 'event1') 30 | record_events('foo_id', 'event1', 'day', sequence=seq) 31 | record_events('foo_id', 'event1', MonthEvent, 'sequence1') 32 | record_events('foo_id', ['event1', 'event2'], [DayEvent, MonthEvent], seq) 33 | record_events('foo_id', ['event1', 'event2'], ['day', 'month'], 'sequence1') 34 | """ 35 | client = conf.get_connection(client) 36 | 37 | if not isinstance(uuids, (list, tuple, set)): 38 | uuids = [uuids] 39 | 40 | if not isinstance(event_names, (list, tuple, set)): 41 | event_names = [event_names] 42 | 43 | # `event_types` may be string, event class or events list 44 | if event_types is None: 45 | event_types = [DayEvent] 46 | elif (isinstance(event_types, basestring) or 47 | not isinstance(event_types, (list, tuple, set))): 48 | event_types = [event_types] 49 | # Resolve event aliasses 50 | event_types = [EVENT_ALIASES.get(t, t) for t in event_types] 51 | 52 | if dt is None: 53 | dt = datetime.utcnow() 54 | 55 | events = [] 56 | for name, ev_type in itertools.product(event_names, event_types): 57 | events.append(ev_type.from_date(name, dt, client, sequence=sequence)) 58 | 59 | first = events[0] 60 | keys = [ev.key for ev in events] 61 | # TODO: make single `msetbit` call 62 | for uuid in uuids: 63 | if len(events) == 1: 64 | # For single event just set bit directly 65 | first.record(uuid) 66 | else: 67 | # Because sequence the same for all events 68 | sid = first.sequential_id(uuid) 69 | msetbit(keys=keys, args=([sid, 1] * len(keys)), client=client) 70 | 71 | return events 72 | 73 | 74 | class Sequence(BaseSequence): 75 | cache_size = 100 76 | namespace = SEQUENCE_NAMESPACE 77 | key_format = '{self.name}' 78 | 79 | 80 | class MixinBitwise(object): 81 | 82 | def __invert__(self): 83 | return Not(self.client, self) 84 | 85 | def __or__(self, other): 86 | return Or(self.client, self, other) 87 | 88 | def __and__(self, other): 89 | return And(self.client, self, other) 90 | 91 | def __xor__(self, other): 92 | return Xor(self.client, self, other) 93 | 94 | def __sub__(self, other): 95 | return LDiff(self.client, self, other) 96 | 97 | 98 | class Event(Base, MixinBitwise): 99 | namespace = EVENT_NAMESPACE 100 | key_format = '{self.name}' 101 | clonable_attrs = ['sequence'] 102 | 103 | def __init__(self, name, client='default', sequence=None): 104 | super(Event, self).__init__(name, client) 105 | self.sequence = sequence 106 | 107 | def sequence(): 108 | def fget(self): 109 | return self._sequence 110 | 111 | def fset(self, sequence): 112 | """ Automatically create `Sequence` instance by name. """ 113 | if isinstance(sequence, basestring): 114 | sequence = Sequence(sequence, self.client) 115 | self._sequence = sequence 116 | 117 | return locals() 118 | 119 | sequence = property(**sequence()) 120 | 121 | def sequential_id(self, uuid): 122 | if self.sequence is not None: 123 | return self.sequence.sequential_id(uuid) 124 | try: 125 | return int(uuid) 126 | except (ValueError, TypeError): 127 | raise ValueError("A `Sequence` instance is required " 128 | "to use non integer uuid `%s`." % (uuid,)) 129 | 130 | def is_recorded(self, uuid): 131 | if self.sequence is not None and uuid not in self.sequence: 132 | return False 133 | sid = self.sequential_id(uuid) 134 | return bool(self.client.getbit(self.key, sid)) 135 | 136 | def record(self, uuid): 137 | return self.client.setbit(self.key, self.sequential_id(uuid), 1) 138 | 139 | def count(self): 140 | return self.client.bitcount(self.key) 141 | 142 | def delete(self, cascade=False): 143 | self.client.delete(self.key) 144 | if cascade and self.sequence is not None: 145 | self.sequence.delete() 146 | 147 | def __len__(self): 148 | return self.count() 149 | 150 | def __contains__(self, uuid): 151 | return self.is_recorded(uuid) 152 | 153 | def __eq__(self, other): 154 | other_key = getattr(other, 'key', None) 155 | other_sequence = getattr(other, 'sequence', None) 156 | return (other_key is not None and 157 | self.key == other_key and 158 | self.sequence == other_sequence) 159 | 160 | 161 | class HourEvent(BaseHour, Event): 162 | pass 163 | 164 | 165 | class DayEvent(BaseDay, Event): 166 | pass 167 | 168 | 169 | class WeekEvent(BaseWeek, Event): 170 | pass 171 | 172 | 173 | class MonthEvent(BaseMonth, Event): 174 | pass 175 | 176 | 177 | class YearEvent(BaseYear, Event): 178 | 179 | def __init__(self, name, year=None, client='default', sequence=None): 180 | super(YearEvent, self).__init__(name, client, year) 181 | self.sequence = sequence 182 | 183 | def months(self): 184 | if not hasattr(self, '_months'): 185 | month = lambda i: MonthEvent(self.name, self.year, i, self.client, self.sequence) 186 | self._months = Or(*[month(i) for i in range(1, 13)]) 187 | return self._months 188 | 189 | @property 190 | def key(self): 191 | return self.months.key 192 | 193 | def delete(self, cascade=False): 194 | self.client.delete(self.key) 195 | if cascade: 196 | self.months.delete(cascade=cascade) 197 | 198 | 199 | class BitOperation(Event): 200 | """ 201 | Base class for bit operations (AND, OR, XOR, NOT). 202 | 203 | Please note that each bit operation creates a new key prefixed with 204 | `spm:bitop_`. Bit operations can be nested. 205 | 206 | Examples:: 207 | 208 | s1 = Sequence('events') 209 | m2 = Month('event1', 2015, 2, s1) 210 | m3 = Month('event1', 2015, 3, s1) 211 | m2 & m3 == And(m2, m3) 212 | """ 213 | def __init__(self, op_name, client_or_event, *events): 214 | if hasattr(client_or_event, 'key'): 215 | events = list(events) 216 | events.insert(0, client_or_event) 217 | client = 'default' 218 | else: 219 | client = client_or_event 220 | 221 | cls_name = self.__class__.__name__ 222 | assert events, \ 223 | "At least one event should be given to perform `%s` operation." % (cls_name,) 224 | 225 | sequences = [ev.sequence for ev in events] 226 | s1 = sequences[0] 227 | for s in sequences[1:]: 228 | if s != s1: 229 | raise ValueError("Event sequences mismatch (%s != %s)" % (s1, s)) 230 | 231 | name = 'bitop_{0}'.format(op_name) 232 | super(BitOperation, self).__init__(name, client, sequences[0]) 233 | 234 | self.op_name = op_name 235 | self.events = events 236 | self.event_keys = [ev.key for ev in events] 237 | self.evaluate() 238 | 239 | @property 240 | def key(self): 241 | k = '{0.name}:({1})' 242 | return _key(k.format(self, '~'.join(self.event_keys))) 243 | 244 | def evaluate(self): 245 | self.client.bitop(self.op_name, self.key, *self.event_keys) 246 | 247 | def delete(self, cascade=False): 248 | self.client.delete(self.key) 249 | if cascade: 250 | for ev in self.events: 251 | if isinstance(ev, BitOperation): 252 | ev.delete(cascade=cascade) 253 | 254 | def clone(self, **initials): 255 | raise NotImplementedError( 256 | 'Method `clone()` is not implemented for `BitOperation` class.') 257 | 258 | 259 | class And(BitOperation): 260 | 261 | def __init__(self, client_or_event, *events): 262 | super(And, self).__init__('AND', client_or_event, *events) 263 | 264 | 265 | class Or(BitOperation): 266 | 267 | def __init__(self, client_or_event, *events): 268 | super(Or, self).__init__('OR', client_or_event, *events) 269 | 270 | 271 | class Xor(BitOperation): 272 | 273 | def __init__(self, client_or_event, *events): 274 | super(Xor, self).__init__('XOR', client_or_event, *events) 275 | 276 | 277 | class Not(BitOperation): 278 | 279 | def __init__(self, client_or_event, *events): 280 | super(Not, self).__init__('NOT', client_or_event, *events) 281 | 282 | 283 | # TODO: rewrite to lua script 284 | class LDiff(BitOperation): 285 | """ 286 | Left diff bitwise operation: 287 | 288 | LDiff(A, B, C) == A - (B & C) == A & ~(B & C) 289 | """ 290 | def __init__(self, client_or_event, *events): 291 | assert len(events) > 1, \ 292 | "At least two events should be given to perform `LDiff` operation." 293 | super(LDiff, self).__init__('_LDIFF', client_or_event, *events) 294 | 295 | def evaluate(self): 296 | left, tail = self.events[0], self.events[1:] 297 | if len(tail) > 1: 298 | right = Not(And(self.client, *tail)) 299 | else: 300 | right = Not(self.client, tail[0]) 301 | self.client.bitop('AND', self.key, left.key, right.key) 302 | 303 | 304 | EVENT_ALIASES = { 305 | 'hour': HourEvent, 306 | 'day': DayEvent, 307 | 'week': WeekEvent, 308 | 'month': MonthEvent, 309 | #'year': YearEvent, 310 | } 311 | 312 | 313 | def delete_temporary_bitop_keys(client='default', dryrun=False): 314 | """ Delete all temporary keys that are used when using bit operations. """ 315 | client = conf.get_connection(client) 316 | pattertn = '{}:bitop_*'.format(EVENT_NAMESPACE) 317 | keys = client.keys(_key(pattertn)) 318 | if not dryrun and len(keys) > 0: 319 | client.delete(*keys) 320 | return keys 321 | -------------------------------------------------------------------------------- /moment/collections.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import absolute_import 5 | 6 | from . import conf 7 | from .base import Base, MixinSerializable 8 | from .compat import lru 9 | from .lua import ( 10 | sequential_id as _sequential_id, multiset_union_update, 11 | multiset_intersection_update 12 | ) 13 | 14 | __all__ = ['BaseSequence', 'BaseDict', 'BaseCounter'] 15 | 16 | 17 | _NONE = object() 18 | 19 | 20 | class BaseSequence(Base): 21 | """ 22 | Tracks sequential ids for symbolic identifiers. Also optionaly holds 23 | cache of recenly created ids. 24 | 25 | Examples:: 26 | 27 | seq = Sequence('sequence1') 28 | sid = seq.sequential_id('foo') 29 | sid == 1 30 | 'foo' in seq 31 | """ 32 | cache_size = None 33 | clonable_attrs = ['cache_size'] 34 | 35 | def __init__(self, name, client='default', cache_size=None): 36 | super(BaseSequence, self).__init__(name, client) 37 | self.cache_size = cache_size or self.cache_size 38 | 39 | @property 40 | def cache(self): 41 | if self.cache_size and lru: 42 | if not hasattr(self, '_cache'): 43 | self._cache = lru.LRU(self.cache_size) 44 | return self._cache 45 | 46 | def sequential_id(self, uuid, force=False): 47 | cache = self.cache 48 | if not force and cache: 49 | try: 50 | return cache[uuid] 51 | except KeyError: 52 | pass 53 | new_id = _sequential_id(self.key, uuid, self.client) 54 | if cache: 55 | cache[uuid] = new_id 56 | return new_id 57 | 58 | def has_uuid(self, uuid, force=False): 59 | cache = self.cache 60 | if not force and cache: 61 | try: 62 | # Is uuid was cached before? 63 | return cache[uuid] is not None 64 | except KeyError: 65 | pass 66 | # Seq id is zero-based. 67 | return self.client.zscore(self.key, uuid) is not None 68 | 69 | def count(self): 70 | return self.client.zcard(self.key) 71 | 72 | def delete(self): 73 | self.client.delete(self.key) 74 | self.flush_cache() 75 | 76 | def flush_cache(self): 77 | try: 78 | del self._cache 79 | except AttributeError: 80 | pass 81 | 82 | def __contains__(self, uuid): 83 | return self.has_uuid(uuid) 84 | 85 | def __len__(self): 86 | return self.count() 87 | 88 | 89 | class BaseDict(MixinSerializable, Base): 90 | clonable_attrs = ['serializer'] 91 | 92 | def __init__(self, name, client='default', serializer=None): 93 | super(BaseDict, self).__init__(name, client) 94 | self.serializer = conf.get_serializer(serializer) 95 | 96 | def __len__(self): 97 | return self.client.hlen(self.key) 98 | 99 | def __contains__(self, key): 100 | return self.client.hexists(self.key, key) 101 | 102 | def __iter__(self): 103 | return self.iterkeys() 104 | 105 | def __setitem__(self, key, value): 106 | self.client.hset(self.key, key, self.dumps(value)) 107 | 108 | def __getitem__(self, key): 109 | value = self.get(key) 110 | if value is None: 111 | raise KeyError(key) 112 | return value 113 | 114 | def __delitem__(self, key): 115 | if self.client.hdel(self.key, key) == 0: 116 | raise KeyError(key) 117 | 118 | def get(self, key, default=None): 119 | value = self.client.hget(self.key, key) 120 | if value is not None: 121 | return self.loads(value) 122 | return default 123 | 124 | def update(self, *args, **kwargs): 125 | for o in args: 126 | self._update(o) 127 | self._update(kwargs) 128 | 129 | def _update(self, other): 130 | if hasattr(other, 'items'): 131 | for k, v in other.items(): 132 | self[k] = v 133 | else: 134 | for k, v in other: 135 | self[k] = v 136 | 137 | def keys(self): 138 | return self.client.hkeys(self.key) 139 | 140 | def values(self): 141 | return [self.loads(v) for v in self.client.hvals(self.key)] 142 | 143 | def items(self): 144 | data = self.client.hgetall(self.key) 145 | return [(k, self.loads(v)) for k, v in data.items()] 146 | 147 | def iterkeys(self): 148 | return iter(self.keys()) 149 | 150 | def itervalues(self): 151 | return iter(self.values()) 152 | 153 | def iteritems(self): 154 | return iter(self.items()) 155 | 156 | def setdefault(self, key, value=None): 157 | if self.client.hsetnx(self.key, key, self.dumps(value)) == 1: 158 | return value 159 | return self.get(key) 160 | 161 | def has_key(self, key): 162 | return key in self 163 | 164 | def copy(self): 165 | return self.__class__(self.name, self.client, self.serializer) 166 | 167 | def clear(self): 168 | self.delete() 169 | 170 | def pop(self, key, default=_NONE): 171 | 172 | with self.client.pipeline() as pipe: 173 | pipe.hget(self.key, key) 174 | pipe.hdel(self.key, key) 175 | value, existed = pipe.execute() 176 | 177 | if not existed: 178 | if default is _NONE: 179 | raise KeyError(key) 180 | return default 181 | return self.loads(value) 182 | 183 | 184 | class BaseCounter(BaseDict): 185 | 186 | def __init__(self, name, client='default', serializer=None): 187 | super(BaseCounter, self).__init__(name, client, None) 188 | 189 | def dumps(self, value): 190 | return str(int(value)) 191 | 192 | def loads(self, value): 193 | return int(value) 194 | 195 | def _flatten(self, iterable, **kwargs): 196 | for k, v in self._merge(iterable, **kwargs): 197 | yield k 198 | yield v 199 | 200 | def _merge(self, iterable=None, **kwargs): 201 | if iterable: 202 | try: 203 | items = iterable.iteritems() 204 | except AttributeError: 205 | for k in iterable: 206 | kwargs[k] = kwargs.get(k, 0) + 1 207 | else: 208 | for k, v in items: 209 | kwargs[k] = kwargs.get(k, 0) + v 210 | return kwargs.items() 211 | 212 | def _update(self, iterable, multiplier, **kwargs): 213 | for k, v in self._merge(iterable, **kwargs): 214 | self.client.hincrby(self.key, k, v * multiplier) 215 | 216 | def update(self, iterable=None, **kwargs): 217 | self._update(iterable, 1, **kwargs) 218 | 219 | def subtract(self, iterable=None, **kwargs): 220 | self._update(iterable, -1, **kwargs) 221 | 222 | def intersection_update(self, iterable=None, **kwargs): 223 | args = self._flatten(iterable, **kwargs) 224 | multiset_intersection_update(keys=[self.key], args=args, 225 | client=self.client) 226 | 227 | def union_update(self, iterable=None, **kwargs): 228 | args = self._flatten(iterable, **kwargs) 229 | multiset_union_update(keys=[self.key], args=args, 230 | client=self.client) 231 | 232 | def elements(self): 233 | for k, count in self.iteritems(): 234 | for i in range(count): 235 | yield k 236 | 237 | def most_common(self, n=None): 238 | values = sorted(self.iteritems(), key=lambda v: v[1], reverse=True) 239 | if n: 240 | values = values[:n] 241 | return values 242 | 243 | def most_common_percent(self, n=None, precision=None): 244 | values = self.most_common() 245 | total = float(sum([v for _, v in values])) 246 | if n: 247 | values = values[:n] 248 | values = [(k, float(v) / total * 100) for k, v in values] 249 | if precision is not None: 250 | values = [(k, round(v, precision)) for k, v in values] 251 | return values 252 | 253 | def total(self): 254 | return sum(self.values()) 255 | 256 | def __iadd__(self, other): 257 | self.update(other) 258 | return self 259 | 260 | def __isub__(self, other): 261 | self.subtract(other) 262 | return self 263 | 264 | def __iand__(self, other): 265 | self.intersection_update(other) 266 | return self 267 | 268 | def __ior__(self, other): 269 | self.union_update(other) 270 | return self 271 | -------------------------------------------------------------------------------- /moment/compat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __all__ = ['lru', 'msgpack', 'json', 'pickle', 'pickle_hi'] 5 | 6 | 7 | try: 8 | import msgpack 9 | except ImportError: 10 | msgpack = None # noqa 11 | 12 | try: 13 | import lru 14 | except ImportError: 15 | lru = None # noqa 16 | 17 | try: 18 | import ujson as json 19 | except ImportError: 20 | try: 21 | import cjson as json # noqa 22 | except ImportError: 23 | import json # noqa 24 | 25 | 26 | import pickle 27 | 28 | 29 | class pickle_hi: 30 | 31 | @staticmethod 32 | def dumps(val): 33 | return pickle.dumps(val, pickle.HIGHEST_PROTOCOL) 34 | 35 | loads = staticmethod(pickle.loads) 36 | -------------------------------------------------------------------------------- /moment/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | import redis 6 | from redis.connection import PythonParser 7 | from redis.connection import ConnectionPool 8 | 9 | from .compat import json, pickle_hi, pickle, msgpack 10 | 11 | 12 | __all__ = ['get_serializer', 'register_connection', 'get_connection'] 13 | 14 | 15 | MOMENT_KEY_PREFIX = 'spm' 16 | MOMENT_SERIALIZER = 'json' 17 | 18 | 19 | _serializers = { 20 | 'json': json, 21 | 'pickle': pickle, 22 | 'pickle_hi': pickle_hi, 23 | } 24 | if msgpack: 25 | _serializers['msgpack'] = msgpack 26 | 27 | 28 | def get_serializer(alias): 29 | alias = alias or MOMENT_SERIALIZER 30 | 31 | if hasattr(alias, 'loads'): 32 | return alias 33 | try: 34 | return _serializers[alias] 35 | except KeyError: 36 | raise LookupError("Serializer `{}` not configured.".format(alias)) 37 | 38 | 39 | _connections = {} 40 | 41 | 42 | def register_connection(alias='default', host='localhost', port=6379, **kwargs): 43 | global _connections 44 | 45 | kwargs.setdefault('parser_class', PythonParser) 46 | kwargs.setdefault('db', 0) 47 | 48 | pool = ConnectionPool(host=host, port=port, **kwargs) 49 | conn = redis.StrictRedis(connection_pool=pool) 50 | 51 | _connections[alias] = conn 52 | return conn 53 | 54 | 55 | def get_connection(alias='default'): 56 | global _connections 57 | 58 | if isinstance(alias, redis.StrictRedis): 59 | return alias 60 | 61 | try: 62 | return _connections[alias] 63 | except KeyError: 64 | raise LookupError("Connection `{}` not configured.".format(alias)) 65 | -------------------------------------------------------------------------------- /moment/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | -------------------------------------------------------------------------------- /moment/contrib/django.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import absolute_import 5 | 6 | __doc__ = """ 7 | 8 | Integration with django settings: 9 | 10 | # settings.py 11 | 12 | MOMENT_KEY_PREFIX = 'spm' 13 | MOMENT_SERIALIZER = 'msgpack' 14 | MOMENT_REDIS = { 15 | 'default': { 16 | 'host': 'localhost', 17 | 'port': 6379, 18 | 'db': 1, 19 | 'socket_timeout': 3, 20 | 'max_connections': 10, 21 | } 22 | } 23 | 24 | INSTALLED_APPS.append('moment.contrib.django') 25 | """ 26 | 27 | from .. import conf 28 | from django.conf import settings 29 | 30 | 31 | MOMENT_REDIS = getattr(settings, 'MOMENT_REDIS', None) 32 | MOMENT_KEY_PREFIX = getattr(settings, 'MOMENT_KEY_PREFIX', None) 33 | MOMENT_SERIALIZER = getattr(settings, 'MOMENT_SERIALIZER', None) 34 | 35 | 36 | if MOMENT_KEY_PREFIX: 37 | conf.MOMENT_KEY_PREFIX = MOMENT_KEY_PREFIX 38 | 39 | if MOMENT_SERIALIZER: 40 | conf.MOMENT_SERIALIZER = MOMENT_SERIALIZER 41 | 42 | if MOMENT_REDIS: 43 | for alias, conn_conf in MOMENT_REDIS.items(): 44 | conf.register_connection(alias, **conn_conf) 45 | -------------------------------------------------------------------------------- /moment/counters.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import itertools 5 | from datetime import datetime 6 | from . import conf 7 | from .collections import BaseCounter 8 | from .base import BaseHour, BaseDay, BaseWeek, BaseMonth, BaseYear 9 | 10 | __all__ = ['COUNTER_NAMESPACE', 'COUNTER_ALIASES', 'update_counters', 11 | 'Counter', 'HourCounter', 'DayCounter', 'WeekCounter', 12 | 'MonthCounter', 'YearCounter'] 13 | 14 | 15 | COUNTER_NAMESPACE = 'cnt' 16 | 17 | 18 | def update_counters(counter_names, iterable=None, counter_types=None, dt=None, 19 | client='default'): 20 | """ 21 | Updates counters for hours, days, weeks and months. Default counter 22 | type is `Day`. 23 | 24 | Examples:: 25 | 26 | update_counters('counter1', {'value1': 1, 'value2': 1}) 27 | update_counters(['counter1', 'counter2'], {'value1': 2, 'value2': 3}) 28 | update_counters(['counter1', 'counter2'], ['value1', 'value2']) 29 | update_counters('counter1', 'value1') 30 | """ 31 | client = conf.get_connection(client) 32 | 33 | if isinstance(counter_names, basestring): 34 | counter_names = [counter_names] 35 | 36 | if isinstance(iterable, basestring): 37 | iterable = [iterable] 38 | 39 | # `counter_types` may be string, event class or events list 40 | if counter_types is None: 41 | counter_types = [DayCounter] 42 | elif (isinstance(counter_types, basestring) or 43 | not isinstance(counter_types, (list, tuple, set))): 44 | counter_types = [counter_types] 45 | # Resolve counters aliasses 46 | counter_types = [COUNTER_ALIASES.get(t, t) for t in counter_types] 47 | 48 | if dt is None: 49 | dt = datetime.utcnow() 50 | 51 | # TODO: use pipe and multi commands 52 | counters = [] 53 | for name, cn_type in itertools.product(counter_names, counter_types): 54 | counters.append(cn_type.from_date(name, dt, client)) 55 | 56 | for counter in counters: 57 | counter.update(iterable) 58 | 59 | return counters 60 | 61 | 62 | class Counter(BaseCounter): 63 | namespace = COUNTER_NAMESPACE 64 | key_format = '{self.name}' 65 | 66 | 67 | class HourCounter(BaseHour, Counter): 68 | pass 69 | 70 | 71 | class DayCounter(BaseDay, Counter): 72 | pass 73 | 74 | 75 | class WeekCounter(BaseWeek, Counter): 76 | pass 77 | 78 | 79 | class MonthCounter(BaseMonth, Counter): 80 | pass 81 | 82 | 83 | class YearCounter(BaseYear, Counter): 84 | pass 85 | 86 | 87 | COUNTER_ALIASES = { 88 | 'hour': HourCounter, 89 | 'day': DayCounter, 90 | 'week': WeekCounter, 91 | 'month': MonthCounter, 92 | 'year': YearCounter, 93 | } 94 | -------------------------------------------------------------------------------- /moment/keys.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import time 5 | from . import conf 6 | from .base import ( 7 | _key, Base, MixinSerializable, BaseHour, BaseDay, BaseWeek, BaseMonth, 8 | BaseYear 9 | ) 10 | from .timelines import _totimerange 11 | 12 | 13 | __all__ = ['TIME_INDEX_KEY_NAMESAPCE', 'TimeIndexedKey', 'HourIndexedKey', 14 | 'DayIndexedKey', 'WeekIndexedKey', 'MonthIndexedKey', 15 | 'YearIndexedKey'] 16 | 17 | 18 | TIME_INDEX_KEY_NAMESAPCE = 'tik' 19 | 20 | 21 | class TimeIndexedKey(MixinSerializable, Base): 22 | """ 23 | Key/value storage where keys indexed by time. This allows you continuously 24 | process newly created/updated keys. 25 | 26 | Examples :: 27 | 28 | index = TimeIndexedKey('users') 29 | 30 | # Save users profiles: 31 | index.set('uid1', user_data1, update_index=True) 32 | index.set('uid2', user_data2, update_index=True) 33 | 34 | # Get user ids added in the last 10 sec. (sorted by time added) 35 | result = index.keys(time.time() - 10, limit=2) 36 | for uid, timestamp in result: 37 | print 'User {0} added at {1}'.format(uid, timestamp) 38 | 39 | # Get assotiated data for these ids 40 | result = index.values('uid1', 'uid2') 41 | for uid, data in result: 42 | print 'User {0} -> {1}'.format(uid, data) 43 | 44 | """ 45 | namespace = TIME_INDEX_KEY_NAMESAPCE 46 | clonable_attrs = ['serializer'] 47 | key_format = '{self.name}' 48 | index_key_format = '{self.name}_index' 49 | 50 | def __init__(self, name, client='default', serializer=None): 51 | super(TimeIndexedKey, self).__init__(name, client) 52 | self.serializer = conf.get_serializer(serializer) 53 | 54 | @property 55 | def index_key(self): 56 | base_key = self.index_key_format.format(self=self) 57 | return _key(base_key, self.namespace) 58 | 59 | def value_key(self, key): 60 | return '{0}:{1}'.format(self.key, key) 61 | 62 | def __len__(self): 63 | return self.count() 64 | 65 | def __contains__(self, key): 66 | value_key = self.value_key(key) 67 | return self.client.exists(value_key) 68 | 69 | def __setitem__(self, key, value): 70 | self.set(key, value, update_index=True) 71 | 72 | def __getitem__(self, key): 73 | value, timestamp = self.get(key) 74 | if value is None: 75 | raise KeyError(key) 76 | return value, timestamp 77 | 78 | def __delitem__(self, key): 79 | existed = self.remove(key) 80 | if not existed: 81 | raise KeyError(key) 82 | 83 | def set(self, key, value, timestamp=None, update_index=None): 84 | """ By default we trying to create index if it doesn't exist. """ 85 | 86 | # If `update_index` is True force to update index 87 | if update_index: 88 | return self._set(key, value, timestamp) 89 | 90 | # If `update_index` is None create index if it not exists. 91 | index_time = self.client.zscore(self.index_key, key) 92 | if index_time is None and update_index is None: 93 | return self._set(key, value, timestamp) 94 | 95 | # Else just update assotiated value 96 | value_key, value = self.value_key(key), self.dumps(value) 97 | self.client.set(value_key, value) 98 | return index_time 99 | 100 | def _set(self, key, value, timestamp=None): 101 | timestamp = timestamp or time.time() 102 | value_key, value = self.value_key(key), self.dumps(value) 103 | 104 | with self.client.pipeline() as pipe: 105 | pipe.multi() 106 | pipe.zrem(self.index_key, key) 107 | pipe.zadd(self.index_key, timestamp, key) 108 | pipe.set(value_key, value) 109 | pipe.execute() 110 | return timestamp 111 | 112 | def get(self, key): 113 | value_key = self.value_key(key) 114 | 115 | with self.client.pipeline() as pipe: 116 | pipe.zscore(self.index_key, key) 117 | pipe.get(value_key) 118 | timestamp, value = pipe.execute() 119 | 120 | if value is not None: 121 | return self.loads(value), timestamp 122 | return value, timestamp 123 | 124 | def remove(self, key): 125 | value_key = self.value_key(key) 126 | with self.client.pipeline() as pipe: 127 | pipe.multi() 128 | pipe.delete(value_key) 129 | pipe.zrem(self.index_key, key) 130 | existed, _ = pipe.execute() 131 | return existed 132 | 133 | def values(self, *keys): 134 | assert keys, 'Al least one key should be given.' 135 | value_keys = [self.value_key(k) for k in keys] 136 | values = self.client.mget(*value_keys) 137 | result = [] 138 | for key, value in zip(keys, values): 139 | if value is not None: 140 | value = self.loads(value) 141 | result.append((key, value)) 142 | return result 143 | 144 | def keys(self, start_time=None, end_time=None, limit=None, 145 | with_timestamp=False): 146 | start_time, end_time = _totimerange(start_time, end_time) 147 | offset = None if limit is None else 0 148 | items = self.client.zrangebyscore(self.index_key, start_time, end_time, 149 | offset, limit, with_timestamp) 150 | return items 151 | 152 | def timerange(self, start_time=None, end_time=None, limit=None): 153 | keys_with_timestamp = self.keys(start_time, end_time, limit, True) 154 | value_keys = [self.value_key(k) for k, _ in keys_with_timestamp] 155 | values = self.client.mget(*value_keys) 156 | result = [] 157 | for (key, timestamp), value in zip(keys_with_timestamp, values): 158 | if value is not None: 159 | value = self.loads(value) 160 | result.append((key, value, timestamp)) 161 | return result 162 | 163 | def count_timerange(self, start_time=None, end_time=None): 164 | start_time, end_time = _totimerange(start_time, end_time) 165 | return self.client.zcount(self.index_key, start_time, end_time) 166 | 167 | def delete_timerange(self, start_time=None, end_time=None): 168 | start_time, end_time = _totimerange(start_time, end_time) 169 | keys = self.keys(start_time, end_time) 170 | value_keys = [self.value_key(k) for k in keys] 171 | 172 | if value_keys: 173 | with self.client.pipeline() as pipe: 174 | pipe.delete(*value_keys) 175 | pipe.zremrangebyscore(self.index_key, start_time, end_time) 176 | pipe.execute() 177 | else: 178 | self.client.zremrangebyscore(self.index_key, start_time, end_time) 179 | 180 | def has_key(self, key): 181 | return key in self 182 | 183 | def count(self): 184 | return self.client.zcard(self.index_key) 185 | 186 | def delete(self): 187 | value_key_pattern = self.value_key('*') 188 | keys = self.client.keys(value_key_pattern) 189 | self.client.delete(self.index_key, *keys) 190 | 191 | 192 | class HourIndexedKey(BaseHour, TimeIndexedKey): 193 | pass 194 | 195 | 196 | class DayIndexedKey(BaseDay, TimeIndexedKey): 197 | pass 198 | 199 | 200 | class WeekIndexedKey(BaseWeek, TimeIndexedKey): 201 | pass 202 | 203 | 204 | class MonthIndexedKey(BaseMonth, TimeIndexedKey): 205 | pass 206 | 207 | 208 | class YearIndexedKey(BaseYear, TimeIndexedKey): 209 | pass 210 | -------------------------------------------------------------------------------- /moment/lua.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | __all__ = ['LazzyScript', 'monotonic_zadd', 'sequential_id', 'msetbit', 4 | 'multiset_union_update', 'multiset_intersection_update'] 5 | 6 | 7 | class LazzyScript(object): 8 | 9 | def __init__(self, script, client=None): 10 | self.script = script.read() if hasattr(script, 'read') else script 11 | self.client = client 12 | self._func = None 13 | 14 | if client: 15 | self.load() 16 | 17 | def load(self, client=None, force=False): 18 | client = client or self.client 19 | if force or not self._func: 20 | if not client: 21 | msg = "Redis client should be given explicitly to call `LazzyScript`." 22 | raise AssertionError(msg) 23 | self._func = client.register_script(self.script) 24 | 25 | def __call__(self, keys=[], args=[], client=None): 26 | client = client or self.client 27 | if not self._func: 28 | self.load(client) 29 | return self._func(keys=keys, args=args, client=client) 30 | 31 | 32 | monotonic_zadd = LazzyScript(""" 33 | local sequential_id = redis.call('zscore', KEYS[1], ARGV[1]) 34 | if not sequential_id then 35 | sequential_id = redis.call('zcard', KEYS[1]) 36 | redis.call('zadd', KEYS[1], sequential_id, ARGV[1]) 37 | end 38 | return sequential_id 39 | """) 40 | 41 | 42 | def sequential_id(key, identifier, client=None): 43 | """Map an arbitrary string identifier to a set of sequential ids""" 44 | return int(monotonic_zadd(keys=[key], args=[identifier], client=client)) 45 | 46 | 47 | msetbit = LazzyScript(""" 48 | for index, value in ipairs(KEYS) do 49 | redis.call('setbit', value, ARGV[(index - 1) * 2 + 1], ARGV[(index - 1) * 2 + 2]) 50 | end 51 | return redis.status_reply('ok') 52 | """) 53 | 54 | 55 | first_key_with_bit_set = LazzyScript(""" 56 | for index, value in ipairs(KEYS) do 57 | local bit = redis.call('getbit', value, ARGV[1]) 58 | if bit == 1 then 59 | return value 60 | end 61 | end 62 | return false 63 | """) 64 | 65 | 66 | multiset_intersection_update = LazzyScript(""" 67 | local keys_values = redis.call('HGETALL', KEYS[1]) 68 | local all = {} 69 | for i = 1, #keys_values, 2 do 70 | all[keys_values[i]] = keys_values[i+1] 71 | end 72 | redis.call('DEL', KEYS[1]) 73 | for i = 1, #ARGV, 2 do 74 | local current = tonumber(all[ARGV[i]]) 75 | local new = tonumber(ARGV[i+1]) 76 | if new > 0 and current then 77 | redis.call('HSET', KEYS[1], ARGV[i], math.min(new, current)) 78 | end 79 | end 80 | """) 81 | 82 | 83 | multiset_union_update = LazzyScript(""" 84 | for i = 1, #ARGV, 2 do 85 | local current = tonumber(redis.call('HGET', KEYS[1], ARGV[i])) 86 | local new = tonumber(ARGV[i+1]) 87 | if new > 0 and (not current or new > current) then 88 | redis.call('HSET', KEYS[1], ARGV[i], new) 89 | end 90 | end 91 | """) 92 | -------------------------------------------------------------------------------- /moment/tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import time 5 | import uuid 6 | import unittest 7 | 8 | from . import conf 9 | from . import timelines 10 | from . import keys 11 | 12 | 13 | client = conf.register_connection() 14 | 15 | 16 | ############################################################################## 17 | # Timeline Tests 18 | ############################################################################## 19 | 20 | class TimelineTestCase(unittest.TestCase): 21 | 22 | timeline_class = timelines.Timeline 23 | 24 | def setup_timeline(self): 25 | self.timeline = self.timeline_class('test') 26 | self.start_time = int(time.time()) 27 | self.items = [] 28 | 29 | for i, t in enumerate(range(self.start_time, self.start_time + 10)): 30 | item = {'index': i} 31 | self.items.append((item, t)) 32 | self.timeline.add(item, timestamp=t) 33 | 34 | def teardown_timeline(self): 35 | self.timeline.delete() 36 | 37 | def assert_ranges_equal(self, items1, items2): 38 | self.assertEqual(len(items1), len(items2)) 39 | for (item1, t1), (item2, t2) in zip(items1, items2): 40 | self.assertEqual(t1, t2) 41 | self.assertEqual(item1, item2) 42 | 43 | def setUp(self): 44 | self.setup_timeline() 45 | 46 | def tearDown(self): 47 | self.teardown_timeline() 48 | 49 | def test_count(self): 50 | self.assertEqual(self.timeline.count(), len(self.items)) 51 | self.assertEqual(len(self.timeline), len(self.items)) 52 | 53 | def test_tail(self): 54 | last1, t1 = self.items[-1] 55 | last2, t2 = self.timeline.tail()[0] 56 | self.assertEqual(t1, t2) 57 | self.assertEqual(last1, last2) 58 | 59 | tail1 = self.items[-2:] 60 | tail2 = self.timeline.tail(2) 61 | self.assert_ranges_equal(tail1, tail2) 62 | 63 | def test_head(self): 64 | first1, t1 = self.items[0] 65 | first2, t2 = self.timeline.head()[0] 66 | self.assertEqual(t1, t2) 67 | self.assertEqual(first1, first2) 68 | 69 | head1 = self.items[:2] 70 | head2 = self.timeline.head(2) 71 | self.assert_ranges_equal(head1, head2) 72 | 73 | def test_range(self): 74 | self.assert_ranges_equal(self.items[2:4], self.timeline.range(2, 3)) 75 | self.assert_ranges_equal(self.items[2:], self.timeline.range(2)) 76 | self.assert_ranges_equal(self.items[2:-2], self.timeline.range(2, -3)) 77 | self.assert_ranges_equal(self.items[-4:-1], self.timeline.range(-4, -2)) 78 | 79 | def test_delete_range(self): 80 | items1 = self.items[:2] + self.items[4:] 81 | self.timeline.delete_range(2, 3) 82 | items2 = self.timeline.items() 83 | self.assertEqual(len(items2), len(self.items) - 2) 84 | self.assert_ranges_equal(items1, self.timeline.items()) 85 | 86 | def test_timerange(self): 87 | items1 = self.items[2:5] 88 | start, end = self.items[2], self.items[4] 89 | start_ts, end_ts = start[1], end[1] 90 | items2 = self.timeline.timerange(start_ts, end_ts) 91 | self.assert_ranges_equal(items1, items2) 92 | 93 | def test_count_timerange(self): 94 | items1 = self.items[2:5] 95 | start, end = self.items[2], self.items[4] 96 | start_ts, end_ts = start[1], end[1] 97 | self.assertEqual(len(items1), self.timeline.count_timerange(start_ts, end_ts)) 98 | 99 | def test_delete_timerange(self): 100 | items1 = self.items[:2] + self.items[4:] 101 | start, end = self.items[2], self.items[3] 102 | start_ts, end_ts = start[1], end[1] 103 | self.timeline.delete_timerange(start_ts, end_ts) 104 | items2 = self.timeline.items() 105 | self.assertEqual(len(items2), len(self.items) - 2) 106 | self.assert_ranges_equal(items1, self.timeline.items()) 107 | 108 | 109 | class HourTimelineTestCase(TimelineTestCase): 110 | timeline_class = timelines.HourTimeline 111 | 112 | 113 | class DayTimelineTestCase(TimelineTestCase): 114 | timeline_class = timelines.DayTimeline 115 | 116 | 117 | class WeekTimelineTestCase(TimelineTestCase): 118 | timeline_class = timelines.WeekTimeline 119 | 120 | 121 | class MonthTimelineTestCase(TimelineTestCase): 122 | timeline_class = timelines.MonthTimeline 123 | 124 | 125 | class YearTimelineTestCase(TimelineTestCase): 126 | timeline_class = timelines.YearTimeline 127 | 128 | 129 | ############################################################################## 130 | # Time Indexed Keys Tests 131 | ############################################################################## 132 | 133 | class TimeIndexedKeyTestCase(unittest.TestCase): 134 | 135 | key_class = keys.TimeIndexedKey 136 | 137 | def setup_indexed_key(self): 138 | self.tik = self.key_class('test') 139 | self.start_time = int(time.time()) 140 | self.items = [] 141 | 142 | for i, ts in enumerate(range(self.start_time, self.start_time + 10)): 143 | key = str(uuid.uuid4().hex)[:5] 144 | value = {'index': i} 145 | self.items.append((key, value, ts)) 146 | self.tik.set(key, value, timestamp=ts, update_index=True) 147 | 148 | def teardown_indexed_key(self): 149 | self.tik.delete() 150 | 151 | def assert_ranges_equal(self, items1, items2): 152 | self.assertEqual(len(items1), len(items2)) 153 | for (k1, v1, t1), (k2, v2, t2) in zip(items1, items2): 154 | self.assertEqual((k1, v1, t1), (k2, v2, t2)) 155 | 156 | def setUp(self): 157 | self.setup_indexed_key() 158 | 159 | def tearDown(self): 160 | self.teardown_indexed_key() 161 | 162 | def test_get_set(self): 163 | now = int(time.time()) 164 | key = str(uuid.uuid4().hex)[:5] 165 | val = {'foo': 1} 166 | self.tik.set(key, val, timestamp=now, update_index=True) 167 | val1, t1 = self.tik.get(key) 168 | val2, t2 = self.tik[key] 169 | self.assertEqual(now, t1, t2) 170 | self.assertEqual(val, val1, val2) 171 | 172 | val = {'bar': 2} 173 | past = now - 10 174 | self.tik.set(key, val, timestamp=past, update_index=False) 175 | val1, t1 = self.tik.get(key) 176 | val2, t2 = self.tik[key] 177 | self.assertEqual(now, t1, t2) 178 | self.assertEqual(val, val1, val2) 179 | 180 | val = {'baz': 3} 181 | self.tik.set(key, val, timestamp=past, update_index=True) 182 | val1, t1 = self.tik.get(key) 183 | val2, t2 = self.tik[key] 184 | self.assertEqual(past, t1, t2) 185 | self.assertEqual(val, val1, val2) 186 | 187 | self.tik[key] = val = {'abc': 4} 188 | val1, t1 = self.tik.get(key) 189 | val2, t2 = self.tik[key] 190 | self.assertNotEqual(t1, past, now) 191 | self.assertEqual(t1, t2) 192 | self.assertEqual(val, val1, val2) 193 | 194 | def test_remove(self): 195 | now = int(time.time()) 196 | key = str(uuid.uuid4().hex)[:5] 197 | val = {'foo': 1} 198 | 199 | self.tik.set(key, val, timestamp=now, update_index=True) 200 | self.assertNotEqual(len(self.tik), len(self.items)) 201 | self.tik.remove(key) 202 | self.assertEqual(len(self.tik), len(self.items)) 203 | 204 | self.tik.set(key, val, timestamp=now, update_index=True) 205 | self.assertNotEqual(len(self.tik), len(self.items)) 206 | del self.tik[key] 207 | self.assertEqual(len(self.tik), len(self.items)) 208 | 209 | def test_count(self): 210 | self.assertEqual(self.tik.count(), len(self.items)) 211 | self.assertEqual(len(self.tik), len(self.items)) 212 | 213 | def test_keys(self): 214 | keys1 = self.tik.keys() 215 | keys2 = [i[0] for i in self.items] 216 | self.assertEqual(keys1, keys2) 217 | 218 | keys1 = self.tik.keys(limit=5) 219 | keys2 = [i[0] for i in self.items[:5]] 220 | self.assertEqual(keys1, keys2) 221 | 222 | keys1 = self.tik.keys(with_timestamp=True) 223 | keys2 = [(i[0], i[2]) for i in self.items] 224 | self.assertEqual(keys1, keys2) 225 | 226 | start_ts, end_ts = self.items[1][2], self.items[4][2] 227 | keys1 = self.tik.keys(start_ts, end_ts) 228 | keys2 = [i[0] for i in self.items[1:5]] 229 | self.assertEqual(keys1, keys2) 230 | 231 | keys1 = self.tik.keys(start_ts, end_ts, limit=2, with_timestamp=True) 232 | keys2 = [(i[0], i[2]) for i in self.items[1:3]] 233 | self.assertEqual(keys1, keys2) 234 | 235 | def test_values(self): 236 | keys = [k for k, _, _ in self.items] 237 | data1 = self.tik.values(*keys) 238 | data2 = [(k, v) for k, v, _ in self.items] 239 | self.assertEqual(data1, data2) 240 | 241 | def test_timerange(self): 242 | items1 = self.tik.timerange() 243 | self.assert_ranges_equal(items1, self.items) 244 | 245 | items1 = self.tik.timerange(limit=5) 246 | self.assert_ranges_equal(items1, self.items[:5]) 247 | 248 | start_ts, end_ts = self.items[1][2], self.items[4][2] 249 | items1 = self.tik.timerange(start_ts, end_ts) 250 | self.assert_ranges_equal(items1, self.items[1:5]) 251 | 252 | items1 = self.tik.timerange(start_ts, end_ts, limit=2) 253 | self.assert_ranges_equal(items1, self.items[1:3]) 254 | 255 | items1 = self.tik.timerange(start_ts, limit=2) 256 | self.assert_ranges_equal(items1, self.items[1:3]) 257 | 258 | def test_count_timerange(self): 259 | self.assertEqual(self.tik.count_timerange(), len(self.items)) 260 | 261 | start_ts, end_ts = self.items[1][2], self.items[4][2] 262 | count = self.tik.count_timerange(start_ts, end_ts) 263 | self.assertEqual(count, 4) 264 | 265 | count = self.tik.count_timerange(start_ts) 266 | self.assertEqual(count, len(self.items) - 1) 267 | 268 | def test_delete_timerange(self): 269 | start_ts, end_ts = self.items[1][2], self.items[4][2] 270 | self.tik.delete_timerange(start_ts, end_ts) 271 | self.assertEqual(len(self.tik), len(self.items) - 4) 272 | 273 | 274 | class HourIndexedKeyTestCase(TimeIndexedKeyTestCase): 275 | key_class = keys.HourIndexedKey 276 | 277 | 278 | class DayIndexedKeyTestCase(TimeIndexedKeyTestCase): 279 | key_class = keys.DayIndexedKey 280 | 281 | 282 | class WeekIndexedKeyTestCase(TimeIndexedKeyTestCase): 283 | key_class = keys.WeekIndexedKey 284 | 285 | 286 | class MonthIndexedKeyTestCase(TimeIndexedKeyTestCase): 287 | key_class = keys.MonthIndexedKey 288 | 289 | 290 | class YaerIndexedKeyTestCase(TimeIndexedKeyTestCase): 291 | key_class = keys.YearIndexedKey 292 | 293 | 294 | if __name__ == '__main__': 295 | unittest.main() 296 | -------------------------------------------------------------------------------- /moment/timelines.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import time 5 | from . import conf 6 | from .base import Base, BaseHour, BaseDay, BaseWeek, BaseMonth, BaseYear 7 | from .collections import MixinSerializable 8 | 9 | __all__ = ['TIMELINE_NAMESPACE', 'TIMELINE_ALIASES', 'Timeline', 10 | 'HourTimeline', 'DayTimeline', 'WeekTimeline', 11 | 'MonthTimeline', 'YearTimeline'] 12 | 13 | 14 | TIMELINE_NAMESPACE = 'tln' 15 | 16 | 17 | def _totimerange(start_time, end_time): 18 | if start_time is None: 19 | start_time = '-inf' 20 | if end_time is None: 21 | end_time = '+inf' 22 | return start_time, end_time 23 | 24 | 25 | class Timeline(Base, MixinSerializable): 26 | namespace = TIMELINE_NAMESPACE 27 | key_format = '{self.name}' 28 | clonable_attrs = ['serializer'] 29 | 30 | def __init__(self, name, client='default', serializer=None): 31 | super(Timeline, self).__init__(name, client) 32 | self.serializer = conf.get_serializer(serializer) 33 | 34 | def encode(self, data, timestamp): 35 | return {'d': data, 't': timestamp} 36 | 37 | def decode(self, value): 38 | return value.get('d'), value.get('t') 39 | 40 | def add(self, *items, **kwargs): 41 | """ 42 | Add new item to `timeline` 43 | 44 | Examples :: 45 | 46 | tl = Timeline('events') 47 | tl.add('event1', 'event2', timestamp=time.time()) 48 | """ 49 | assert items, 'At least one item should be given.' 50 | 51 | timestamp = kwargs.get('timestamp') or time.time() 52 | args = [] 53 | for item in items: 54 | args.append(timestamp) 55 | args.append(self.dumps(self.encode(item, timestamp))) 56 | self.client.zadd(self.key, *args) 57 | return timestamp 58 | 59 | def timerange(self, start_time=None, end_time=None, limit=None): 60 | start_time, end_time = _totimerange(start_time, end_time) 61 | offset = None if limit is None else 0 62 | items = self.client.zrangebyscore(self.key, start_time, 63 | end_time, offset, limit) 64 | return [self.decode(self.loads(i)) for i in items] 65 | 66 | def delete_timerange(self, start_time=None, end_time=None): 67 | start_time, end_time = _totimerange(start_time, end_time) 68 | return self.client.zremrangebyscore(self.key, start_time, end_time) 69 | 70 | def count_timerange(self, start_time=None, end_time=None): 71 | start_time, end_time = _totimerange(start_time, end_time) 72 | return self.client.zcount(self.key, start_time, end_time) 73 | 74 | def range(self, start=0, end=-1): 75 | items = self.client.zrange(self.key, start, end) 76 | return [self.decode(self.loads(i)) for i in items] 77 | 78 | def delete_range(self, start=0, end=-1): 79 | return self.client.zremrangebyrank(self.key, start, end) 80 | 81 | def head(self, limit=1): 82 | return self.range(0, limit - 1) 83 | 84 | def tail(self, limit=1): 85 | return self.range(-limit, -1) 86 | 87 | def items(self): 88 | return self.range() 89 | 90 | def count(self): 91 | return self.client.zcard(self.key) 92 | 93 | def __len__(self): 94 | return self.count() 95 | 96 | 97 | class HourTimeline(BaseHour, Timeline): 98 | pass 99 | 100 | 101 | class DayTimeline(BaseDay, Timeline): 102 | pass 103 | 104 | 105 | class WeekTimeline(BaseWeek, Timeline): 106 | pass 107 | 108 | 109 | class MonthTimeline(BaseMonth, Timeline): 110 | pass 111 | 112 | 113 | class YearTimeline(BaseYear, Timeline): 114 | pass 115 | 116 | 117 | TIMELINE_ALIASES = { 118 | 'hour': HourTimeline, 119 | 'day': DayTimeline, 120 | 'week': WeekTimeline, 121 | 'month': MonthTimeline, 122 | 'year': YearTimeline, 123 | } 124 | -------------------------------------------------------------------------------- /moment/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from datetime import date, timedelta 5 | 6 | 7 | def add_month(year, month, delta): 8 | """ 9 | Helper function which adds `delta` months to current `(year, month)` tuple 10 | and returns a new valid tuple `(year, month)` 11 | """ 12 | year, month = divmod(year * 12 + month + delta, 12) 13 | if month == 0: 14 | month = 12 15 | year = year - 1 16 | return year, month 17 | 18 | 19 | def not_none(*keys): 20 | """ Helper function returning first value which is not None. """ 21 | for key in keys: 22 | if key is not None: 23 | return key 24 | 25 | 26 | def iso_year_start(iso_year): 27 | """ The gregorian calendar date of the first day of the given ISO year. """ 28 | fourth_jan = date(iso_year, 1, 4) 29 | delta = timedelta(fourth_jan.isoweekday() - 1) 30 | return fourth_jan - delta 31 | 32 | 33 | def iso_to_gregorian(iso_year, iso_week, iso_day): 34 | """ Gregorian calendar date for the given ISO year, week and day. """ 35 | year_start = iso_year_start(iso_year) 36 | return year_start + timedelta(days=iso_day - 1, weeks=iso_week - 1) 37 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from setuptools import setup, find_packages 5 | 6 | from moment import __version__ 7 | 8 | 9 | setup( 10 | name='redis-moment', 11 | version=__version__, 12 | author='Max Kamenkov', 13 | author_email='mkamenkov@gmail.com', 14 | description='A Powerful Analytics Python Library for Redis', 15 | url='https://github.com/caxap/redis-moment', 16 | packages=find_packages(), 17 | install_requires=[ 18 | 'redis', 19 | 'msgpack-python>=0.4.6' 20 | ], 21 | zip_safe=False, 22 | include_package_data=True, 23 | test_suite='moment.tests', 24 | license='MIT', 25 | classifiers=[ 26 | 'Development Status :: 3 - Alpha', 27 | 'Environment :: Web Environment', 28 | 'Intended Audience :: Developers', 29 | 'License :: OSI Approved :: BSD License', 30 | 'Operating System :: POSIX', 31 | 'Programming Language :: Python', 32 | 'Topic :: System :: Distributed Computing', 33 | 'Topic :: Software Development :: Object Brokering', 34 | ] 35 | ) 36 | --------------------------------------------------------------------------------