├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST ├── Makefile ├── README.markdown ├── coopy ├── __init__.py ├── base.py ├── decorators.py ├── error.py ├── fileutils.py ├── foundation.py ├── journal.py ├── network │ ├── __init__.py │ ├── default_select.py │ ├── linux_epoll.py │ ├── network.py │ └── osx_kqueue.py ├── restore.py ├── snapshot.py ├── tests │ ├── __init__.py │ └── utils.py ├── utils.py └── validation.py ├── docs ├── Makefile ├── _static │ ├── default.css │ └── grid.css ├── _templates │ └── layout.html ├── basics.rst ├── changelog.rst ├── client_server.rst ├── conf.py ├── index.rst ├── installation.rst ├── make.bat ├── method_decorators.rst ├── replication.rst ├── roadmap.rst ├── snapshots.rst ├── tests.rst ├── todo.rst ├── tutorial.rst ├── usage.rst ├── use_clock.rst └── util_api.rst ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── domain.py ├── functional ├── __init__.py └── test_coopy.py └── unit ├── __init__.py ├── test_base.py ├── test_fileutils.py ├── test_foundation.py ├── test_journal.py ├── test_network.py ├── test_network_select.py ├── test_snapshot.py ├── test_utils.py └── test_validation.py /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | *.pyc 3 | htmlcov/ 4 | build/ 5 | build_history/ 6 | coopy.log* 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.6" 4 | - "2.7" 5 | - "3.3" 6 | - "pypy" 7 | # command to run tests 8 | script: make test 9 | notifications: 10 | irc: "irc.freenode.org#loogica" 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009/2012, Loogica - Felipe João Pontes da Cruz - felipecruz@loogica.net 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of copycat nor the names of its contributors may be used 15 | to endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | README 2 | setup.py 3 | coopy/__init__.py 4 | coopy/base.py 5 | coopy/decorators.py 6 | coopy/fileutils.py 7 | coopy/foundation.py 8 | coopy/journal.py 9 | coopy/network.py 10 | coopy/restore.py 11 | coopy/snapshot.py 12 | coopy/utils.py 13 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: clean 2 | py.test --verbose . 3 | coverage: 4 | py.test --cov-report html --cov . 5 | clean: 6 | @rm -rf build/ 7 | @rm -rf htmlcov/ 8 | @rm -rf dist/ 9 | @rm -rf *egg-info/ 10 | @rm -f coopy.log* 11 | @find . -name '*.py[co,log,dat]' | xargs rm -f 12 | @find . -name '__pycache__' | xargs rm -rf 13 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # coopy 2 | 3 | **coopy** is a simple, transparent, non-intrusive persistence library for python language. It's released under BSD License 4 | 5 | * **Simple** - you don't have to learn an API. You can use it with just one line of code. 6 | * **Transparent** - you don't need to call any API functions, just your Object methods. 7 | * **Non-Intrusive** - no inheritance, no interface.. only pure-python-business code. 8 | 9 | It is based on the techniques of system snapshotting and transaction journalling. In the prevalent model, the object data is kept in memory in native object format, rather than being marshalled to an RDBMS or other data storage system. A snapshot of data is regularly saved to disk, and in addition to this, all changes are serialised and a log of transactions is also stored on disk. 10 | 11 | http://en.wikipedia.org/wiki/Object_prevalence 12 | 13 | ## Status 14 | 15 | Current version - 0.4.3beta 16 | 17 | coopy is compatible with py2.6, py2.7, py3.2, py3.3 and pypy. 18 | 19 | CI builds: 20 | 21 | [![Build Status](https://secure.travis-ci.org/felipecruz/coopy.png)](http://travis-ci.org/felipecruz/coopy) 22 | 23 | ## Install 24 | 25 | ```sh 26 | $ pip install coopy 27 | ``` 28 | 29 | or 30 | 31 | 32 | ```sh 33 | $ git clone https://github.com/felipecruz/coopy.git 34 | $ cd coopy 35 | $ python setup.py install 36 | ``` 37 | 38 | ## Using 39 | 40 | Simple, transparent and non-intrusive. Note that ``Todo`` could be any class 41 | that you want to persist state across method calls that modifies it's internal 42 | state. 43 | 44 | ```python 45 | 46 | from coopy.base import init_persistent_system 47 | class Todo(object): 48 | def __init__(self): 49 | self.tasks = [] 50 | 51 | def add_task(self, name, description): 52 | task = dict(name=name, description=description) 53 | self.tasks.append(task) 54 | 55 | persistent_todo_list = init_persistent_system(Todo()) 56 | persistent_todo_list.add_task("Some Task Name", "A Task Description") 57 | ``` 58 | 59 | ## Restrictions 60 | 61 | This should not affect end-user code 62 | 63 | To get datetime or date objects you need to get from an internal clock. 64 | Check [How to use Clock](http://coopy.readthedocs.org/en/latest/use_clock.html) 65 | 66 | ## Documentation 67 | 68 | http://coopy.readthedocs.org 69 | 70 | ## Cases 71 | 72 | ### RioBus 73 | 74 | http://riobus.loogica.net/ 75 | 76 | All (~1800) Bus lines from RJ (State), Brazil, each Bus Line with a list 77 | of tuples (street, city_name, direction). All in memory. Running since Sep 2012. 78 | 79 | System(Domain) Class: https://github.com/loogica/riobus/blob/master/riobus.py 80 | 81 | ## Tests 82 | 83 | First time: 84 | 85 | `pip install -r requirements.txt` 86 | 87 | To actually run the tests: 88 | 89 | `make test` 90 | 91 | ## Coverage Report 92 | 93 | First time: 94 | 95 | `pip install -r requirements.txt` 96 | 97 | And then: 98 | 99 | `make coverage` 100 | 101 | # LICENSE 102 | 103 | ``` 104 | Copyright (c) 2009/2012, Loogica - Felipe João Pontes da Cruz - felipecruz@loogica.net 105 | All rights reserved. 106 | 107 | Redistribution and use in source and binary forms, with or without modification, 108 | are permitted provided that the following conditions are met: 109 | 110 | 1. Redistributions of source code must retain the above copyright notice, 111 | this list of conditions and the following disclaimer. 112 | 113 | 2. Redistributions in binary form must reproduce the above copyright 114 | notice, this list of conditions and the following disclaimer in the 115 | documentation and/or other materials provided with the distribution. 116 | 117 | 3. Neither the name of copycat nor the names of its contributors may be used 118 | to endorse or promote products derived from this software without 119 | specific prior written permission. 120 | 121 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 122 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 123 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 124 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 125 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 126 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 127 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 128 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 129 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 130 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 131 | ``` 132 | 133 | Contribute 134 | ---------- 135 | 136 | You know! 137 | 138 | Fork, Pull Request :) 139 | 140 | Contact 141 | ------- 142 | 143 | felipecruz@loogica.net 144 | -------------------------------------------------------------------------------- /coopy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felipecruz/coopy/507221f2dead6145e884dea99999d603ee175178/coopy/__init__.py -------------------------------------------------------------------------------- /coopy/base.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 6, 2011 3 | 4 | @author: felipe j. cruz 5 | ''' 6 | __author__ = "Felipe Cruz " 7 | __license__ = "BSD" 8 | __version__ = "0.4.6b" 9 | 10 | import six 11 | import os 12 | import logging 13 | import threading 14 | import types 15 | 16 | if six.PY3: 17 | import _thread as thread 18 | else: 19 | import thread 20 | 21 | from logging.handlers import RotatingFileHandler 22 | from datetime import datetime 23 | 24 | from coopy import fileutils 25 | from coopy.foundation import Action, RecordClock, Publisher 26 | from coopy.journal import DiskJournal 27 | from coopy.restore import restore 28 | from coopy.snapshot import SnapshotManager, SnapshotTimer 29 | from coopy.utils import method_or_none, action_check, inject 30 | from coopy.validation import validate_system 31 | 32 | from coopy.network.default_select import CopyNet, CopyNetSlave 33 | 34 | CORE_LOG_PREFIX = '[CORE] ' 35 | 36 | def logging_config(basedir="./"): 37 | log_file_path = os.path.join(basedir, "coopy.log") 38 | result = logging.getLogger("coopy") 39 | result.setLevel(logging.DEBUG) 40 | 41 | handler = RotatingFileHandler(log_file_path, "a", 1000000, 10) 42 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s ' \ 43 | '- %(message)s', '%a, %d %b %Y %H:%M:%S') 44 | handler.setFormatter(formatter) 45 | result.addHandler(handler) 46 | 47 | logging_config() 48 | logger = logging.getLogger("coopy") 49 | 50 | 51 | def init_persistent_system(obj, basedir=None): 52 | # a system object is needed in order to coopy work 53 | if not obj: 54 | raise Exception(CORE_LOG_PREFIX + 55 | "Must input a valid object if there's no snapshot files") 56 | 57 | # if obj is a class, change obj to an insance 58 | if isinstance(obj, type): 59 | obj = obj() 60 | 61 | validate_system(obj) 62 | 63 | # first step is to check basedir argument. if isn't defined 64 | # coopy will create a directory name based on system class 65 | if not basedir: 66 | basedir = fileutils.obj_to_dir_name(obj) 67 | 68 | # convert some string to a valid directory name 69 | basedir = fileutils.name_to_dir(basedir) 70 | 71 | system_data_path = os.getcwd() 72 | 73 | # check if basedir exists, if not, create it 74 | try: 75 | os.listdir(basedir) 76 | except os.error: 77 | os.mkdir(basedir) 78 | 79 | # if no snapshot files, create first one with a 'empty' system 80 | if not fileutils.last_snapshot_file(basedir): 81 | logger.info(CORE_LOG_PREFIX + "No snapshot files..") 82 | SnapshotManager(basedir).take_snapshot(obj) 83 | 84 | # measure restore time 85 | start = datetime.utcnow() 86 | logger.info(CORE_LOG_PREFIX + "coopy init....") 87 | 88 | # inject clock on system object 89 | inject(obj, '_clock', RecordClock()) 90 | inject(obj, '_system_data_path', system_data_path) 91 | 92 | # RestoreHelper will recover a system state based on snapshots and 93 | # transations files extracting actions executed previously and 94 | # re-executing them again 95 | obj = restore(obj, basedir) 96 | 97 | end = datetime.utcnow() 98 | delta = end - start 99 | logger.info(CORE_LOG_PREFIX + "spent " + str(delta) + "microseconds") 100 | 101 | journal = DiskJournal(basedir, system_data_path) 102 | journal.setup() 103 | 104 | snapshot_manager = SnapshotManager(basedir) 105 | 106 | return CoopyProxy(obj, [journal], snapshot_manager=snapshot_manager) 107 | 108 | def init_system(obj, subscribers): 109 | # return a coopy proxy instead of a system instance 110 | proxy = CoopyProxy(obj, subscribers) 111 | return proxy 112 | 113 | class CoopyProxy(): 114 | def __init__(self, obj, subscribers, snapshot_manager=None): 115 | self.obj = obj 116 | self.publisher = Publisher(subscribers) 117 | self.lock = threading.RLock() 118 | self.master = None 119 | self.slave = None 120 | self.snapshot_manager = snapshot_manager 121 | self.snapshot_timer = None 122 | 123 | def start_snapshot_manager(self, snapshot_time): 124 | import time 125 | self.snapshot_timer = SnapshotTimer(snapshot_time, self) 126 | time.sleep(snapshot_time) 127 | self.snapshot_timer.start() 128 | 129 | def start_master(self, port=8012, password=None, ipc=False): 130 | self.server = CopyNet(self.obj, 131 | port=port, 132 | password=password, 133 | ipc=ipc) 134 | self.server.start() 135 | self.publisher.register(self.server) 136 | self.slave = None 137 | 138 | def start_slave(self, host, port, password=None, ipc=None): 139 | self.server = None 140 | self.slave = CopyNetSlave(self.obj, 141 | self, 142 | host=host, 143 | password=password, 144 | port=port, 145 | ipc=ipc) 146 | self.slave.start() 147 | 148 | def __getattr__(self, name): 149 | method = method_or_none(self.obj, name) 150 | 151 | if not method: 152 | return getattr(self.obj, name) 153 | 154 | (readonly,unlocked,abort_exception) = action_check(method) 155 | 156 | #TODO: re-write 157 | if not readonly and hasattr(self, 'slave') and self.slave: 158 | raise Exception('This is a slave/read-only instance.') 159 | 160 | def method(*args, **kwargs): 161 | exception = None 162 | try: 163 | if not unlocked: 164 | self.lock.acquire(1) 165 | 166 | #record all calls to clock.now() 167 | self.obj._clock = RecordClock() 168 | 169 | thread_ident = thread.get_ident() 170 | action = Action(thread_ident, 171 | name, 172 | datetime.now(), 173 | args, 174 | kwargs) 175 | system = None 176 | if not readonly: 177 | self.publisher.publish_before(action) 178 | 179 | try: 180 | system = action.execute_action(self.obj) 181 | except Exception as e: 182 | logger.debug(CORE_LOG_PREFIX + 'Error: ' + str(e)) 183 | if abort_exception: 184 | logger.debug(CORE_LOG_PREFIX + 185 | 'Aborting action' + str(action)) 186 | if not abort_exception: 187 | self.publisher.publish_exception(action) 188 | exception = e 189 | 190 | #restore clock 191 | action.results = self.obj._clock.results 192 | 193 | if not readonly and not abort_exception: 194 | self.publisher.publish(action) 195 | 196 | finally: 197 | if not unlocked: 198 | self.lock.release() 199 | 200 | if exception: 201 | raise exception 202 | 203 | return system 204 | return method 205 | 206 | def basedir_abspath(self): 207 | disk_journals = (journal for journal in self.publisher.subscribers 208 | if hasattr(journal, 'basedir')) 209 | return [os.path.join(os.path.abspath(os.getcwd()), journal.basedir) 210 | for journal in disk_journals] 211 | 212 | def take_snapshot(self): 213 | if self.slave: 214 | self.slave.acquire() 215 | 216 | self.lock.acquire() 217 | if self.snapshot_manager: 218 | self.snapshot_manager.take_snapshot(self.obj) 219 | self.lock.release() 220 | 221 | if self.slave: 222 | self.slave.release() 223 | 224 | def close(self): 225 | self.publisher.close() 226 | logging.shutdown() 227 | if self.snapshot_timer: 228 | self.snapshot_timer.stop() 229 | 230 | def shutdown(self): 231 | if self.master: 232 | self.server.close() 233 | if self.slave: 234 | self.slave.close() 235 | -------------------------------------------------------------------------------- /coopy/decorators.py: -------------------------------------------------------------------------------- 1 | def unlocked(func): 2 | func.__unlocked = True 3 | return func 4 | 5 | def readonly(func): 6 | func.__readonly = True 7 | return func 8 | 9 | def abort_exception(func): 10 | func.__abort_exception = True 11 | return func -------------------------------------------------------------------------------- /coopy/error.py: -------------------------------------------------------------------------------- 1 | class PrevalentError(Exception): 2 | """This error represents issues related to the nature of a prevalent 3 | system. 4 | """ 5 | def __init__(self, *args, **kwargs): 6 | super(PrevalentError, self).__init__(*args, **kwargs) 7 | 8 | -------------------------------------------------------------------------------- /coopy/fileutils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import types 4 | import logging 5 | import logging.config 6 | 7 | logger = logging.getLogger("coopy") 8 | 9 | DIGITS = 15 10 | MAX_LOGFILE_SIZE = 1024 * 100 * 10 # 1 MB 11 | 12 | LOG_PREFIX = "transaction_" 13 | LOG_SUFIX = ".log" 14 | 15 | SNAPSHOT_PREFIX = "snapshot_" 16 | SNAPSHOT_SUFIX = ".dat" 17 | 18 | SNAPSHOT_REGEX = re.compile("snapshot_(\d{15})\.dat", re.IGNORECASE) 19 | LOG_REGEX = re.compile("transaction_(\d{15})\.log", re.IGNORECASE) 20 | FILE_REGEX = re.compile("(transaction|snapshot)_(\d{15})\..{3}", re.IGNORECASE) 21 | 22 | class RotateFileWrapper(): 23 | def __init__(self, file, basedir, system_data_path, max_size=MAX_LOGFILE_SIZE): 24 | self.file = file 25 | self.basedir = basedir 26 | self.max_size = max_size 27 | self.system_data_path = system_data_path 28 | 29 | def write(self, data): 30 | current_cwd = os.getcwd() 31 | restore_cwd = current_cwd == self.system_data_path 32 | if not restore_cwd: 33 | os.chdir(self.system_data_path) 34 | 35 | self.file.write(data) 36 | 37 | ''' certifies that data will be written - performance issues? ''' 38 | self.file.flush() 39 | 40 | if os.path.getsize(self.file.name) > self.max_size: 41 | file_name = next_log_file(self.basedir) 42 | logger.debug("Opening: " + file_name) 43 | self.file.flush() 44 | os.fsync(self.file.fileno()) 45 | self.file.close() 46 | self.file = open(file_name,'wb') 47 | self.pickler.clear_memo() 48 | 49 | 50 | if not restore_cwd: 51 | os.chdir(current_cwd) 52 | 53 | def __getattr__(self, name): 54 | if name == 'closed': 55 | return self.file.closed 56 | elif name == "flush": 57 | import os 58 | self.file.flush() 59 | os.fsync(self.file.fileno()) 60 | else: 61 | return getattr(self.file, name) 62 | 63 | @property 64 | def name(self): 65 | return self.file.name 66 | 67 | def close(self): 68 | self.file.close() 69 | 70 | def set_pickler(self, pickler): 71 | self.pickler = pickler 72 | 73 | def number_as_string(number): 74 | s_number = str(number) 75 | return s_number.zfill(DIGITS) 76 | 77 | 78 | def list_coopy_files(basedir, regex): 79 | files = os.listdir(basedir) 80 | if not files: 81 | return None 82 | files = filter(regex.search, files) 83 | if not files: 84 | return None 85 | return sorted(files) 86 | 87 | 88 | def list_snapshot_files(basedir): 89 | return list_coopy_files(basedir, SNAPSHOT_REGEX) 90 | 91 | 92 | def list_log_files(basedir): 93 | return list_coopy_files(basedir, LOG_REGEX) 94 | 95 | 96 | def check_files(files): 97 | if not files: 98 | return None 99 | return files[len(files)-1] 100 | 101 | 102 | def last_snapshot_file(basedir): 103 | files = list_coopy_files(basedir, SNAPSHOT_REGEX) 104 | return check_files(files) 105 | 106 | def last_log_file(basedir): 107 | files = list_coopy_files(basedir, LOG_REGEX) 108 | return check_files(files) 109 | 110 | def lastest_snapshot_number(basedir): 111 | last_file = last_snapshot_file(basedir) 112 | if not last_file: 113 | return 1 114 | number = SNAPSHOT_REGEX.match(last_file) 115 | return int(number.group(1)) 116 | 117 | 118 | def lastest_log_number(basedir): 119 | last_file = last_log_file(basedir) 120 | if not last_file: 121 | return 1 122 | number = LOG_REGEX.match(last_file) 123 | return int(number.group(1)) 124 | 125 | 126 | def to_log_file(basedir, number): 127 | return basedir + LOG_PREFIX + number_as_string(number) + LOG_SUFIX; 128 | 129 | 130 | def to_snapshot_file(basedir, number): 131 | return basedir + SNAPSHOT_PREFIX + number_as_string(number) + SNAPSHOT_SUFIX; 132 | 133 | 134 | def get_number(filename): 135 | number = FILE_REGEX.match(filename) 136 | return int(number.group(2)) 137 | 138 | 139 | def next_number(basedir): 140 | n1 = lastest_log_number(basedir) 141 | n2 = lastest_snapshot_number(basedir) 142 | if n1 > n2: 143 | return n1+1 144 | else: 145 | return n2+1 146 | 147 | 148 | def next_snapshot_file(basedir): 149 | number = next_number(basedir) 150 | return to_snapshot_file(basedir, number) 151 | 152 | 153 | def next_log_file(basedir): 154 | number = next_number(basedir) 155 | return to_log_file(basedir, number) 156 | 157 | 158 | def last_log_files(basedir): 159 | files = list_log_files(basedir) 160 | if not files: 161 | return None 162 | cont = 0 163 | last_snap = lastest_snapshot_number(basedir) 164 | file_names = [] 165 | for file in files: 166 | if get_number(file) > last_snap: 167 | cont += 1 168 | file_names.append(basedir + file) 169 | return file_names 170 | 171 | def name_to_dir(basedir): 172 | if len(basedir) > 0: 173 | basedir = basedir.replace('\\','/') 174 | if not basedir.endswith('/'): 175 | basedir += '/' 176 | else: 177 | basedir = './' 178 | return basedir 179 | 180 | def obj_to_dir_name(obj): 181 | if isinstance(obj, type): 182 | basedir = obj.__name__.lower() 183 | else: 184 | basedir = obj.__class__.__name__.lower() 185 | return basedir 186 | -------------------------------------------------------------------------------- /coopy/foundation.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, date, time 2 | from time import localtime 3 | 4 | from .validation import FORBIDDEN_OBJECTS, DATETIME_FUNCS, DATE_FUNCS 5 | 6 | class RecordClock(object): 7 | def __init__(self): 8 | self.results = [] 9 | 10 | def __getattr__(self, name): 11 | if name in DATETIME_FUNCS: 12 | return getattr(datetime, name) 13 | elif name in DATE_FUNCS: 14 | return getattr(date, name) 15 | return object.__getattribute__(self, name) 16 | 17 | def now(self): 18 | dt = datetime.now() 19 | self.results.append(dt) 20 | return dt 21 | 22 | def utcnow(self): 23 | dt = datetime.utcnow() 24 | self.results.append(dt) 25 | return dt 26 | 27 | def today(self): 28 | dt = date.today() 29 | self.results.append(dt) 30 | return dt 31 | 32 | class RestoreClock(object): 33 | def __init__(self, results): 34 | #reverse order! 35 | self.results = results[::-1] 36 | 37 | def __getattr__(self, name): 38 | if name in DATETIME_FUNCS or name in DATE_FUNCS: 39 | return self.pop_and_check(name) 40 | return object.__getattribute__(self, name) 41 | 42 | def pop_and_check(self, method_name): 43 | last_element = self.results[-1] 44 | if method_name in DATETIME_FUNCS: 45 | if not isinstance(last_element, datetime): 46 | raise TypeError("This call returns a %s" % 47 | (datetime.__class__)) 48 | return lambda: self.results.pop() 49 | elif method_name in DATE_FUNCS: 50 | if not isinstance(last_element, date): 51 | raise TypeError("This call returns a %s" % 52 | (date.__class__)) 53 | return lambda: self.results.pop() 54 | 55 | 56 | class Action(object): 57 | def __init__(self, caller_id, action, timestamp, args, kwargs): 58 | self.caller_id = caller_id 59 | self.action = action 60 | self.args = args 61 | self.kwargs = kwargs 62 | self.results = None 63 | 64 | # TODO do we need it? 65 | self.timestamp = timestamp 66 | 67 | def __str__(self): 68 | return "action %s \ncaller_id %s \nargs %s" % \ 69 | (self.action, str(self.caller_id), self.args) 70 | 71 | def execute_action(self, system): 72 | #TODO not reliable 73 | method = getattr(system, self.action) 74 | return method(*self.args, **self.kwargs) 75 | 76 | class Publisher: 77 | def __init__(self, subscribers): 78 | self.subscribers = subscribers 79 | 80 | def register(self, subscriber): 81 | self.subscribers.append(subscriber) 82 | 83 | def publish(self, message): 84 | for subscriber in self.subscribers: 85 | subscriber.receive(message) 86 | 87 | def publish_before(self, message): 88 | for subscriber in self.subscribers: 89 | subscriber.receive_before(message) 90 | 91 | def publish_exception(self, message): 92 | for subscriber in self.subscribers: 93 | subscriber.receive_exception(message) 94 | 95 | def close(self): 96 | for subscriber in self.subscribers: 97 | subscriber.close() 98 | -------------------------------------------------------------------------------- /coopy/journal.py: -------------------------------------------------------------------------------- 1 | import six 2 | import os 3 | import pickle 4 | from coopy import fileutils 5 | 6 | if six.PY3: 7 | from pickle import Pickler 8 | else: 9 | from cPickle import Pickler 10 | 11 | class DiskJournal(): 12 | def __init__(self, basedir, system_data_path): 13 | ''' 14 | set basedir and declare file attribute 15 | ''' 16 | self.basedir = basedir 17 | self.file = None 18 | self.system_data_path = system_data_path 19 | 20 | def setup(self): 21 | ''' 22 | configure file attribute, create pickler 23 | ''' 24 | self.file = self.current_journal_file(self.basedir) 25 | self.pickler = Pickler(self.file, pickle.HIGHEST_PROTOCOL) 26 | self.file.set_pickler(self.pickler) 27 | 28 | def current_journal_file(self, basedir): 29 | ''' 30 | get current file on basedir(last created one) 31 | and return a Rotate wrapper. 32 | 33 | if current file size > 1Mb (configured on fileutils) crate 34 | another file with the next name 35 | ''' 36 | 37 | last_file_name = fileutils.last_log_file(self.basedir) 38 | 39 | if last_file_name and os.path.getsize(self.basedir + last_file_name) \ 40 | < fileutils.MAX_LOGFILE_SIZE: 41 | 42 | file = fileutils.RotateFileWrapper( 43 | open(self.basedir + last_file_name, 'ab'), 44 | self.basedir, 45 | self.system_data_path) 46 | else: 47 | file = fileutils.RotateFileWrapper( 48 | open(fileutils.next_log_file(self.basedir), 'wb'), 49 | self.basedir, 50 | self.system_data_path) 51 | 52 | return file 53 | 54 | def receive_before(self, message): 55 | pass 56 | 57 | def receive(self, message): 58 | ''' 59 | receive a message and pickle 60 | ''' 61 | self.pickler.dump(message) 62 | 63 | def receive_exception(self, message): 64 | pass 65 | 66 | def close(self): 67 | ''' 68 | close file instance 69 | ''' 70 | if not self.file.closed: 71 | self.file.close() 72 | -------------------------------------------------------------------------------- /coopy/network/__init__.py: -------------------------------------------------------------------------------- 1 | from coopy.network import * 2 | -------------------------------------------------------------------------------- /coopy/network/default_select.py: -------------------------------------------------------------------------------- 1 | import six 2 | from select import select 3 | import os 4 | import socket 5 | import zlib 6 | import struct 7 | 8 | if six.PY3: 9 | import pickle 10 | from queue import Queue 11 | else: 12 | import cPickle as pickle 13 | from Queue import Queue 14 | import threading 15 | import logging 16 | import sys 17 | 18 | 19 | from coopy.network.network import prepare_data, CopyNetPacket, CopyNetClient,\ 20 | CopyNetSnapshotThread 21 | 22 | COPYNET_MASTER_PREFIX = '[coopynet - master]' 23 | COPYNET_SLAVE_PREFIX = '[coopynet - slave]' 24 | 25 | l = logging.getLogger("coopy") 26 | 27 | _minfo = lambda x: l.info("%s %s" % (COPYNET_MASTER_PREFIX, x)) 28 | _sinfo = lambda x: l.info("%s %s" % (COPYNET_SLAVE_PREFIX, x)) 29 | 30 | _mdebug = lambda x: l.debug("%s %s" % (COPYNET_MASTER_PREFIX, x)) 31 | _sdebug = lambda x: l.debug("%s %s" % (COPYNET_SLAVE_PREFIX, x)) 32 | 33 | _mwarn = lambda x: l.warn("%s %s" % (COPYNET_MASTER_PREFIX, x)) 34 | _swarn = lambda x: l.warn("%s %s" % (COPYNET_SLAVE_PREFIX, x)) 35 | 36 | COPYNET_SOCK = '/tmp/coopy.sock' 37 | COPYNET_HEADER = '!Ic' 38 | 39 | _HEADER_SIZE = struct.calcsize(COPYNET_HEADER) 40 | 41 | #TODO: re-write all code below!!! :) (and tests) 42 | 43 | class CopyNet(threading.Thread): 44 | def __init__(self, 45 | obj, 46 | host=None, 47 | port=5466, 48 | max_clients=5, 49 | password=b'copynet', 50 | ipc=False): 51 | 52 | threading.Thread.__init__(self) 53 | self.obj = obj 54 | self.host = host 55 | self.port = port 56 | self.max_clients = max_clients 57 | self.password = password 58 | self.ipc = ipc 59 | 60 | self.clientmap = {} 61 | self.queues = {} 62 | 63 | if not self.ipc: 64 | self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 65 | else: 66 | self.server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 67 | 68 | if not host: 69 | try: 70 | host = socket.gethostbyname(socket.gethostname()) 71 | except Exception as e: 72 | _mwarn("Error on gethostname %s" % (str(e))) 73 | host = "127.0.0.1" 74 | 75 | self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 76 | 77 | if not self.ipc: 78 | self.server.bind((host, port)) 79 | else: 80 | self.server.bind(COPYNET_SOCK) 81 | 82 | self.server.listen(max_clients) 83 | _minfo("Listening to %d" % max_clients) 84 | 85 | def close(self): 86 | self.running = False 87 | try: 88 | self.server.shutdown(socket.SHUT_RDWR) 89 | except socket.error: 90 | pass 91 | self.server.close() 92 | if self.ipc: 93 | os.remove(COPYNET_SOCK) 94 | 95 | def receive(self, message): 96 | _mdebug('Receive') 97 | 98 | if len(self.clientmap) == 0: 99 | return 100 | 101 | (header, data) = prepare_data(message) 102 | self.broadcast(header, data) 103 | 104 | def broadcast(self, header, data): 105 | _mdebug('Broadcast') 106 | 107 | for copynetclient in self.clientmap.values(): 108 | if copynetclient.state == 'r': 109 | copynetclient.client.sendall(header) 110 | copynetclient.client.sendall(data) 111 | elif copynetclient.state == 'b': 112 | self.queues[copynetclient.client].put_nowait( 113 | CopyNetPacket(header,data)) 114 | else: 115 | _mdebug('Unknow client state') 116 | 117 | def send_direct(self, client, message): 118 | (header, data) = prepare_data(message) 119 | client.sendall(header) 120 | client.sendall(data) 121 | 122 | def check_if_authorized_client(self, client): 123 | password = client.recv(20) 124 | 125 | if self.password != password.rstrip(): 126 | unauthdata = struct.pack(COPYNET_HEADER, 0, b'n') 127 | client.sendall(unauthdata) 128 | client.close() 129 | _minfo('Client rejected') 130 | return False 131 | 132 | return True 133 | 134 | def initialize_client(self, client, address): 135 | self.clientmap[client] = CopyNetClient(client, address, 'r') 136 | self.queues[client] = Queue(999999) 137 | 138 | def disconnect(self, sock): 139 | _mdebug('Master received data. Close client') 140 | sock.close() 141 | del self.clientmap[sock] 142 | 143 | def run(self): 144 | self.running = True 145 | 146 | while self.running: 147 | 148 | try: 149 | to_read, to_write, exception_mode = \ 150 | select([self.server] + list(self.clientmap.keys()), [], [], 5) 151 | except socket.error as e: 152 | _mdebug("Select error") 153 | self.running = False 154 | break 155 | 156 | for sock in to_read: 157 | if sock == self.server: 158 | try: 159 | client, address = self.server.accept() 160 | except Exception as e: 161 | _mdebug('Cannot accept client: %s' % e.message) 162 | continue 163 | 164 | _minfo('Server: got connection %d from %s' % 165 | (client.fileno(), address)) 166 | 167 | if not self.check_if_authorized_client(client): 168 | continue 169 | 170 | _minfo('Client connected') 171 | self.initialize_client(client, address) 172 | #CopyNetSnapshotThread(self.clientmap[client], self.obj).start() 173 | else: 174 | self.disconnect(sock) 175 | 176 | for sock in to_write: 177 | while not self.queues[sock].empty(): 178 | self.send_direct(sock, self.queues[sock].get_nowait()) 179 | 180 | self.server.close() 181 | 182 | class CopyNetSlave(threading.Thread): 183 | def __init__(self, 184 | parent, 185 | host='localhost', 186 | port=5466, 187 | password=b'copynet', 188 | ipc=False): 189 | 190 | threading.Thread.__init__(self) 191 | self.parent = parent 192 | self.host = host 193 | self.port = port 194 | self.running = False 195 | self.lock = threading.RLock() 196 | 197 | try: 198 | if not ipc: 199 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 200 | self.sock.connect((host, self.port)) 201 | else: 202 | self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 203 | self.sock.connect(COPYNET_SOCK) 204 | 205 | self.sock.sendall(password) 206 | _sdebug('Slave connected to server@%d' % self.port) 207 | except socket.error as e: 208 | _sdebug("Error connecting to server %s" % (e)) 209 | raise Exception("Network Error: Could not connect to: %s:%d" % 210 | (host, port)) 211 | 212 | def acquire(self): 213 | self.lock.acquire(1) 214 | 215 | def release(self): 216 | self.lock.release() 217 | 218 | def close(self): 219 | self.running = False 220 | self.sock.close() 221 | 222 | def run(self): 223 | self.running = True 224 | 225 | while self.running: 226 | try: 227 | to_read, to_write, exception_mode = \ 228 | select([self.sock], [],[], 1) 229 | except Exception as e: 230 | _mdebug('Error on select %s..\nShutting down' % (e)) 231 | self.running = False 232 | break 233 | 234 | _mdebug("waitin for data..") 235 | 236 | for sock in to_read: 237 | if sock == self.sock: 238 | try: 239 | data = self.sock.recv(_HEADER_SIZE) 240 | except Exception as e: 241 | _mdebug('Exception %s' % e) 242 | data = None 243 | 244 | if not data: 245 | _mdebug('Shutting down') 246 | self.running = False 247 | break 248 | else: 249 | try: 250 | (psize, stype) = struct.unpack(COPYNET_HEADER, data) 251 | except struct.error as e: 252 | self.running = False 253 | self.sock.close() 254 | raise Exception("Unexpected error") 255 | if stype == 'n': 256 | self.sock.close() 257 | raise Exception("Not authorized") 258 | 259 | _sdebug('Reading %d bytes' % (psize)) 260 | buf = '' 261 | 262 | while len(buf) < psize: 263 | buf += self.sock.recv(4096) 264 | 265 | un_data = zlib.decompress(buf) 266 | 267 | self.lock.acquire(1) 268 | if stype == 'a': 269 | _sdebug('Receiving action') 270 | action = picke.loads(str(un_data)) 271 | action.execute_action(self.parent.obj) 272 | _sdebug(str(action)) 273 | else: 274 | _sdebug('Receiving system') 275 | self.parent.obj = pickle.loads(str(un_data)) 276 | self.lock.release() 277 | 278 | else: 279 | _sdebug("Unexcpected") 280 | 281 | _mdebug("Slave close") 282 | self.sock.close() 283 | -------------------------------------------------------------------------------- /coopy/network/linux_epoll.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felipecruz/coopy/507221f2dead6145e884dea99999d603ee175178/coopy/network/linux_epoll.py -------------------------------------------------------------------------------- /coopy/network/network.py: -------------------------------------------------------------------------------- 1 | import six 2 | import zlib 3 | import struct 4 | 5 | if six.PY3: 6 | import pickle 7 | else: 8 | import cPickle as pickle 9 | 10 | import threading 11 | import logging 12 | 13 | from coopy import foundation 14 | from coopy.foundation import Action 15 | 16 | COPYNET_MASTER_PREFIX = '[coopynet - master]' 17 | COPYNET_SLAVE_PREFIX = '[coopynet - slave]' 18 | COPYNET_SOCK = '/tmp/coopy.sock' 19 | COPYNET_HEADER = '!Ic' 20 | 21 | l = logging.getLogger("coopy") 22 | 23 | def prepare_data(data, stype=b's'): 24 | if (isinstance(data, Action) or isinstance(data, foundation.Action)): 25 | stype = b'a' 26 | 27 | data = pickle.dumps(data) 28 | compressed_data = zlib.compress(data) 29 | value = len(compressed_data) 30 | header = struct.pack(COPYNET_HEADER, value, stype) 31 | 32 | return (header, compressed_data) 33 | 34 | class CopyNetClient(): 35 | def __init__(self, client, address, state): 36 | self.client = client 37 | self.address = address 38 | self.state = state 39 | 40 | class CopyNetPacket(): 41 | def __init__(self, header, data): 42 | self.header = header 43 | self.data = data 44 | 45 | class CopyNetSnapshotThread(threading.Thread): 46 | def __init__(self, net_client, obj): 47 | threading.Thread.__init__ (self) 48 | self.net_client = net_client 49 | self.obj = obj 50 | 51 | def run(self): 52 | (header,data) = prepare_data(self.obj) 53 | self.net_client.client.sendall(header) 54 | self.net_client.client.sendall(data) 55 | self.net_client.state = 'r' 56 | -------------------------------------------------------------------------------- /coopy/network/osx_kqueue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felipecruz/coopy/507221f2dead6145e884dea99999d603ee175178/coopy/network/osx_kqueue.py -------------------------------------------------------------------------------- /coopy/restore.py: -------------------------------------------------------------------------------- 1 | import six 2 | import logging 3 | 4 | from coopy import fileutils 5 | from coopy.snapshot import SnapshotManager 6 | from coopy.foundation import RestoreClock 7 | 8 | if six.PY3: 9 | from pickle import Unpickler, UnpicklingError 10 | BadPickleGet = UnpicklingError 11 | else: 12 | from cPickle import Unpickler, BadPickleGet 13 | 14 | logger = logging.getLogger("coopy") 15 | LOG_PREFIX = '[RESTORE] ' 16 | 17 | def restore(system, basedir): 18 | #save current clock 19 | current_clock = system._clock 20 | 21 | #restore from snapshot 22 | system = SnapshotManager(basedir).recover_snapshot() 23 | 24 | files = fileutils.last_log_files(basedir) 25 | logger.debug(LOG_PREFIX + "Files found: " + str(files)) 26 | if not files: 27 | return system 28 | 29 | actions = [] 30 | for file in files: 31 | logger.debug(LOG_PREFIX + "Opening " + str(file)) 32 | unpickler = Unpickler(open(file,'rb')) 33 | try: 34 | while True: 35 | action = unpickler.load() 36 | logger.debug(LOG_PREFIX + action.action) 37 | actions.append(action) 38 | except BadPickleGet: 39 | logger.error(LOG_PREFIX + "Error unpickling %s" % (str(file))) 40 | except EOFError: 41 | pass 42 | 43 | if not actions: 44 | return system 45 | 46 | logger.debug(LOG_PREFIX + "Actions re-execution") 47 | for action in actions: 48 | try: 49 | if hasattr(action, 'results'): 50 | system._clock = RestoreClock(action.results) 51 | action.execute_action(system) 52 | except Exception as e: 53 | logger.debug(LOG_PREFIX + 'Error executing: %s' % (str(action))) 54 | logger.debug(LOG_PREFIX + 'Exception: %s' % (str(e))) 55 | 56 | system._clock = current_clock 57 | return system 58 | -------------------------------------------------------------------------------- /coopy/snapshot.py: -------------------------------------------------------------------------------- 1 | import six 2 | import threading 3 | import logging 4 | import pickle 5 | from coopy import fileutils as fu 6 | 7 | from os import path 8 | 9 | if six.PY3: 10 | from pickle import Pickler, Unpickler 11 | else: 12 | from cPickle import Pickler, Unpickler 13 | 14 | logger = logging.getLogger("coopy") 15 | 16 | class SnapshotManager(object): 17 | def __init__(self, basedir): 18 | if len(basedir) > 0: 19 | basedir = basedir.replace('\\','/') 20 | if not basedir.endswith('/'): 21 | basedir += '/' 22 | else: 23 | basedir = './' 24 | 25 | self.basedir = basedir 26 | 27 | def take_snapshot(self, object): 28 | file = open(fu.next_snapshot_file(self.basedir), "wb") 29 | logger.debug("Taking snapshot on: " + file.name) 30 | pickler = Pickler(file, pickle.HIGHEST_PROTOCOL) 31 | pickler.dump(object) 32 | file.flush() 33 | file.close() 34 | 35 | def recover_snapshot(self): 36 | file = open(path.join(self.basedir,fu.last_snapshot_file(self.basedir)),"rb") 37 | if not file: 38 | return None 39 | logger.debug("Recovering snapshot from: " + file.name) 40 | unpickler = Unpickler(file) 41 | return unpickler.load() 42 | 43 | 44 | class SnapshotTimer(threading.Thread): 45 | def __init__(self, snapshot_time, proxy): 46 | threading.Thread.__init__ (self) 47 | self.snapshot_time = snapshot_time 48 | self.proxy = proxy 49 | self.finished = threading.Event() 50 | 51 | def run(self): 52 | while not self.finished.isSet(): 53 | self.proxy.take_snapshot() 54 | self.finished.wait(self.snapshot_time) 55 | 56 | def stop(self): 57 | self.finished.set() 58 | self.join() 59 | -------------------------------------------------------------------------------- /coopy/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felipecruz/coopy/507221f2dead6145e884dea99999d603ee175178/coopy/tests/__init__.py -------------------------------------------------------------------------------- /coopy/tests/utils.py: -------------------------------------------------------------------------------- 1 | from coopy.utils import inject 2 | from coopy.foundation import RecordClock 3 | 4 | class TestClock(object): 5 | def __init__(self, date): 6 | self.date = date 7 | 8 | def now(self): 9 | return self.date 10 | 11 | def utcnow(self): 12 | return self.date 13 | 14 | def today(self): 15 | return self.date 16 | 17 | class TestSystemMixin(object): 18 | def enable_clock(self, system): 19 | inject(system, '_clock', RecordClock()) 20 | 21 | def mock_clock(self, system, date): 22 | test_clock = TestClock(date) 23 | inject(system, '_clock', test_clock) 24 | -------------------------------------------------------------------------------- /coopy/utils.py: -------------------------------------------------------------------------------- 1 | def method_or_none(instance, name): 2 | method = getattr(instance, name) 3 | if (name[0:2] == '__' and name[-2:] == '__') or \ 4 | not callable(method) : 5 | return None 6 | return method 7 | 8 | def action_check(obj): 9 | return (hasattr(obj, '__readonly'), 10 | hasattr(obj, '__unlocked'), 11 | hasattr(obj, '__abort_exception')) 12 | 13 | def inject(obj, name, dependency): 14 | obj.__dict__[name] = dependency 15 | -------------------------------------------------------------------------------- /coopy/validation.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import inspect 3 | import logging 4 | 5 | from .error import PrevalentError 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | FORBIDDEN_OBJECTS = ('datetime', 'date') 10 | 11 | DATE_FUNCS = ('today',) 12 | 13 | DATETIME_FUNCS = ('now', 'utcnow') 14 | 15 | FORBIDDEN_FUNCS = DATE_FUNCS + DATETIME_FUNCS 16 | 17 | class NodeVisitor(ast.NodeVisitor): 18 | def generic_visit(self, node): 19 | self._continue(node) 20 | 21 | def visit_Call(self, node): 22 | if not hasattr(node.func, 'value'): 23 | ''' 24 | Ignore calls from nodes with no 'value' attribute, 25 | like constructors calls MyClass() 26 | ''' 27 | return 28 | if isinstance(node.func.value, ast.Attribute): 29 | ''' 30 | In this case we can't be sure if it's a call from a date or 31 | datetime instance 32 | ''' 33 | if node.func.value.attr == "_clock": 34 | ''' 35 | A call to self._clock.method() wich is ok. 36 | ''' 37 | return 38 | elif node.func.attr in FORBIDDEN_FUNCS: 39 | logger.warn("Dont call now(), utcnow() nor today() from date" 40 | " or datetime objects. Use the clock instead.") 41 | elif isinstance(node.func.value, ast.Subscript): 42 | ''' 43 | Ignore var[something] pattern 44 | ''' 45 | return 46 | elif isinstance(node.func.value, ast.Str): 47 | ''' 48 | ignore ''.join([]) patterns 49 | ''' 50 | return 51 | elif hasattr(node.func.value, 'id'): 52 | if node.func.value.id in FORBIDDEN_OBJECTS and \ 53 | node.func.attr in FORBIDDEN_FUNCS: 54 | ''' 55 | Bad calls: date.today(), datetime.now(), datetime.utcnow() 56 | ''' 57 | raise Exception("This function calls %s.%s()- use clock.%s()" % \ 58 | (node.func.value.id, node.func.attr, node.func.attr)) 59 | 60 | self._continue(node) 61 | 62 | def _continue(self, stmt): 63 | '''Helper: parse a node's children''' 64 | super(NodeVisitor, self).generic_visit(stmt) 65 | 66 | def validate_date_datetime_calls(function): 67 | tree = ast.parse(function) 68 | node = NodeVisitor() 69 | try: 70 | node.visit(tree) 71 | return True 72 | except Exception as e: 73 | return False 74 | 75 | def unindent_source(source_lines): 76 | source = source_lines[0] 77 | unindent_level = len(source) - len(source.lstrip()) 78 | return "\n".join([line[unindent_level:] for line in source_lines]) 79 | 80 | def validate_system(system_instance): 81 | for (method_name, method) in inspect.getmembers(system_instance, 82 | predicate=inspect.ismethod): 83 | if method_name.startswith("__") and method_name.endswith("__"): 84 | continue 85 | fixed_source = unindent_source(inspect.getsourcelines(method)[0]) 86 | valid = validate_date_datetime_calls(fixed_source) 87 | if not valid: 88 | raise PrevalentError("%s contains methods with invalid calls "\ 89 | "on method %s:\n%s" % (system_instance, 90 | method_name, 91 | fixed_source)) 92 | return True 93 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " pickle to make pickle files" 22 | @echo " json to make JSON files" 23 | @echo " htmlhelp to make HTML files and a HTML help project" 24 | @echo " qthelp to make HTML files and a qthelp project" 25 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 26 | @echo " changes to make an overview of all changed/added/deprecated items" 27 | @echo " linkcheck to check all external links for integrity" 28 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 29 | 30 | clean: 31 | -rm -rf $(BUILDDIR)/* 32 | 33 | html: 34 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 35 | @echo 36 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 37 | 38 | dirhtml: 39 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 40 | @echo 41 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 42 | 43 | pickle: 44 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 45 | @echo 46 | @echo "Build finished; now you can process the pickle files." 47 | 48 | json: 49 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 50 | @echo 51 | @echo "Build finished; now you can process the JSON files." 52 | 53 | htmlhelp: 54 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 55 | @echo 56 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 57 | ".hhp project file in $(BUILDDIR)/htmlhelp." 58 | 59 | qthelp: 60 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 61 | @echo 62 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 63 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 64 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/coopy.qhcp" 65 | @echo "To view the help file:" 66 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/coopy.qhc" 67 | 68 | latex: 69 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 70 | @echo 71 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 72 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 73 | "run these through (pdf)latex." 74 | 75 | changes: 76 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 77 | @echo 78 | @echo "The overview file is in $(BUILDDIR)/changes." 79 | 80 | linkcheck: 81 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 82 | @echo 83 | @echo "Link check complete; look for any errors in the above output " \ 84 | "or in $(BUILDDIR)/linkcheck/output.txt." 85 | 86 | doctest: 87 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 88 | @echo "Testing of doctests in the sources finished, look at the " \ 89 | "results in $(BUILDDIR)/doctest/output.txt." 90 | -------------------------------------------------------------------------------- /docs/_static/default.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Sphinx stylesheet -- default theme 3 | * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 4 | */ 5 | 6 | @import url("basic.css"); 7 | 8 | /* -- page layout ----------------------------------------------------------- */ 9 | 10 | body { 11 | font-family: arial,helvetica,clean,sans-serif; 12 | font-size: 100%; 13 | background-color: white; 14 | color: #000; 15 | margin: 0; 16 | padding: 0; 17 | } 18 | 19 | div.document { 20 | background-color: white; 21 | margin-left: auto; 22 | margin-right: auto; 23 | width: 960px; 24 | } 25 | 26 | div.documentwrapper { 27 | float: left; 28 | width: 100%; 29 | } 30 | 31 | div.bodywrapper { 32 | margin: 0 0 0 230px; 33 | } 34 | 35 | div.body { 36 | background-color: white; 37 | color: black; 38 | padding: 0 20px 30px 20px; 39 | } 40 | div.bodywrapper { 41 | margin: 0 230px 0 0; 42 | } 43 | 44 | div.footer { 45 | color: black; 46 | width: 100%; 47 | padding: 9px 0 9px 0; 48 | text-align: center; 49 | font-size: 75%; 50 | } 51 | 52 | div.footer a { 53 | color: black; 54 | text-decoration: underline; 55 | } 56 | 57 | div.related { 58 | background-color: white; 59 | line-height: 30px; 60 | color: black; 61 | } 62 | 63 | div.related a { 64 | color: red; 65 | } 66 | 67 | div.sphinxsidebar { 68 | float: right; 69 | } 70 | 71 | div.sphinxsidebar h3 { 72 | font-family: arial,helvetica,clean,sans-serif; 73 | color: red; 74 | font-size:1em; 75 | font-weight: normal; 76 | margin:1em 0 0.5em; 77 | padding:0.1em 0 0.1em 0.5em; 78 | background-color:#CCCCCC; 79 | border:1px solid #FF0000; 80 | } 81 | 82 | div.sphinxsidebar h3 a { 83 | color: red; 84 | } 85 | 86 | div.sphinxsidebar h4 { 87 | font-family: arial,helvetica,clean,sans-serif; 88 | color: red; 89 | font-size:1em; 90 | font-weight: normal; 91 | margin:1em 0 0.5em; 92 | padding:0.1em 0 0.1em 0.5em; 93 | background-color:#CCCCCC; 94 | border:1px solid #FF0000; 95 | } 96 | 97 | div.sphinxsidebar p { 98 | color: black; 99 | } 100 | 101 | div.sphinxsidebar p.topless { 102 | margin: 5px 10px 10px 10px; 103 | } 104 | 105 | div.sphinxsidebar ul { 106 | margin: 10px; 107 | padding: 0; 108 | color: black; 109 | } 110 | 111 | div.sphinxsidebar a { 112 | color: red; 113 | } 114 | 115 | div.sphinxsidebar input { 116 | border: 1px solid red; 117 | font-family: sans-serif; 118 | font-size: 1em; 119 | } 120 | 121 | /* -- body styles ----------------------------------------------------------- */ 122 | 123 | a { 124 | color: red; 125 | text-decoration: none; 126 | } 127 | 128 | a:hover { 129 | text-decoration: underline; 130 | } 131 | 132 | div.body p, div.body dd, div.body li { 133 | text-align: justify; 134 | line-height: 130%; 135 | } 136 | 137 | div.body h1, 138 | div.body h2, 139 | div.body h3, 140 | div.body h4, 141 | div.body h5, 142 | div.body h6 { 143 | font-family: arial,helvetica,clean,sans-serif; 144 | background-color: white; 145 | font-weight: normal; 146 | color: red; 147 | border-bottom: 1px solid #ccc; 148 | margin: 20px -20px 10px -20px; 149 | padding: 3px 0 3px 10px; 150 | } 151 | 152 | div.body h1 { margin-top: 0; font-size: 200%; } 153 | div.body h2 { font-size: 160%; } 154 | div.body h3 { font-size: 140%; } 155 | div.body h4 { font-size: 120%; } 156 | div.body h5 { font-size: 110%; } 157 | div.body h6 { font-size: 100%; } 158 | 159 | a.headerlink { 160 | color: red; 161 | font-size: 0.8em; 162 | padding: 0 4px 0 4px; 163 | text-decoration: none; 164 | } 165 | 166 | a.headerlink:hover { 167 | background-color: red; 168 | color: white; 169 | } 170 | 171 | div.body p, div.body dd, div.body li { 172 | text-align: justify; 173 | line-height: 130%; 174 | } 175 | 176 | div.admonition p.admonition-title + p { 177 | display: inline; 178 | } 179 | 180 | div.note { 181 | background-color: #eee; 182 | border: 1px solid #ccc; 183 | } 184 | 185 | div.seealso { 186 | background-color: #ffc; 187 | border: 1px solid #ff6; 188 | } 189 | 190 | div.topic { 191 | background-color: #eee; 192 | } 193 | 194 | div.warning { 195 | background-color: #ffe4e4; 196 | border: 1px solid #f66; 197 | } 198 | 199 | p.admonition-title { 200 | display: inline; 201 | } 202 | 203 | p.admonition-title:after { 204 | content: ":"; 205 | } 206 | 207 | pre { 208 | padding: 5px; 209 | background-color: #EEE; 210 | color: black; 211 | line-height: 120%; 212 | border: 1px solid #ac9; 213 | border-left: none; 214 | border-right: none; 215 | } 216 | 217 | tt { 218 | background-color: #ecf0f3; 219 | padding: 0 1px 0 1px; 220 | font-size: 0.95em; 221 | } 222 | -------------------------------------------------------------------------------- /docs/_static/grid.css: -------------------------------------------------------------------------------- 1 | /* 2 | Variable Grid System. 3 | Learn more ~ http://www.spry-soft.com/grids/ 4 | Based on 960 Grid System - http://960.gs/ 5 | 6 | Licensed under GPL and MIT. 7 | */ 8 | 9 | 10 | /* Containers 11 | ----------------------------------------------------------------------------------------------------*/ 12 | .container_12 { 13 | margin-left: auto; 14 | margin-right: auto; 15 | width: 960px; 16 | } 17 | 18 | /* Grid >> Global 19 | ----------------------------------------------------------------------------------------------------*/ 20 | 21 | .grid_1, 22 | .grid_2, 23 | .grid_3, 24 | .grid_4, 25 | .grid_5, 26 | .grid_6, 27 | .grid_7, 28 | .grid_8, 29 | .grid_9, 30 | .grid_10, 31 | .grid_11, 32 | .grid_12 { 33 | display:inline; 34 | float: left; 35 | position: relative; 36 | margin-left: 10px; 37 | margin-right: 10px; 38 | } 39 | 40 | /* Grid >> Children (Alpha ~ First, Omega ~ Last) 41 | ----------------------------------------------------------------------------------------------------*/ 42 | 43 | .alpha { 44 | margin-left: 0; 45 | } 46 | 47 | .omega { 48 | margin-right: 0; 49 | } 50 | 51 | /* Grid >> 12 Columns 52 | ----------------------------------------------------------------------------------------------------*/ 53 | 54 | .container_12 .grid_1 { 55 | width:60px; 56 | } 57 | 58 | .container_12 .grid_2 { 59 | width:140px; 60 | } 61 | 62 | .container_12 .grid_3 { 63 | width:220px; 64 | } 65 | 66 | .container_12 .grid_4 { 67 | width:300px; 68 | } 69 | 70 | .container_12 .grid_5 { 71 | width:380px; 72 | } 73 | 74 | .container_12 .grid_6 { 75 | width:460px; 76 | } 77 | 78 | .container_12 .grid_7 { 79 | width:540px; 80 | } 81 | 82 | .container_12 .grid_8 { 83 | width:620px; 84 | } 85 | 86 | .container_12 .grid_9 { 87 | width:700px; 88 | } 89 | 90 | .container_12 .grid_10 { 91 | width:780px; 92 | } 93 | 94 | .container_12 .grid_11 { 95 | width:860px; 96 | } 97 | 98 | .container_12 .grid_12 { 99 | width:940px; 100 | } 101 | 102 | 103 | 104 | /* Prefix Extra Space >> 12 Columns 105 | ----------------------------------------------------------------------------------------------------*/ 106 | 107 | .container_12 .prefix_1 { 108 | padding-left:80px; 109 | } 110 | 111 | .container_12 .prefix_2 { 112 | padding-left:160px; 113 | } 114 | 115 | .container_12 .prefix_3 { 116 | padding-left:240px; 117 | } 118 | 119 | .container_12 .prefix_4 { 120 | padding-left:320px; 121 | } 122 | 123 | .container_12 .prefix_5 { 124 | padding-left:400px; 125 | } 126 | 127 | .container_12 .prefix_6 { 128 | padding-left:480px; 129 | } 130 | 131 | .container_12 .prefix_7 { 132 | padding-left:560px; 133 | } 134 | 135 | .container_12 .prefix_8 { 136 | padding-left:640px; 137 | } 138 | 139 | .container_12 .prefix_9 { 140 | padding-left:720px; 141 | } 142 | 143 | .container_12 .prefix_10 { 144 | padding-left:800px; 145 | } 146 | 147 | .container_12 .prefix_11 { 148 | padding-left:880px; 149 | } 150 | 151 | 152 | 153 | /* Suffix Extra Space >> 12 Columns 154 | ----------------------------------------------------------------------------------------------------*/ 155 | 156 | .container_12 .suffix_1 { 157 | padding-right:80px; 158 | } 159 | 160 | .container_12 .suffix_2 { 161 | padding-right:160px; 162 | } 163 | 164 | .container_12 .suffix_3 { 165 | padding-right:240px; 166 | } 167 | 168 | .container_12 .suffix_4 { 169 | padding-right:320px; 170 | } 171 | 172 | .container_12 .suffix_5 { 173 | padding-right:400px; 174 | } 175 | 176 | .container_12 .suffix_6 { 177 | padding-right:480px; 178 | } 179 | 180 | .container_12 .suffix_7 { 181 | padding-right:560px; 182 | } 183 | 184 | .container_12 .suffix_8 { 185 | padding-right:640px; 186 | } 187 | 188 | .container_12 .suffix_9 { 189 | padding-right:720px; 190 | } 191 | 192 | .container_12 .suffix_10 { 193 | padding-right:800px; 194 | } 195 | 196 | .container_12 .suffix_11 { 197 | padding-right:880px; 198 | } 199 | 200 | 201 | 202 | /* Push Space >> 12 Columns 203 | ----------------------------------------------------------------------------------------------------*/ 204 | 205 | .container_12 .push_1 { 206 | left:80px; 207 | } 208 | 209 | .container_12 .push_2 { 210 | left:160px; 211 | } 212 | 213 | .container_12 .push_3 { 214 | left:240px; 215 | } 216 | 217 | .container_12 .push_4 { 218 | left:320px; 219 | } 220 | 221 | .container_12 .push_5 { 222 | left:400px; 223 | } 224 | 225 | .container_12 .push_6 { 226 | left:480px; 227 | } 228 | 229 | .container_12 .push_7 { 230 | left:560px; 231 | } 232 | 233 | .container_12 .push_8 { 234 | left:640px; 235 | } 236 | 237 | .container_12 .push_9 { 238 | left:720px; 239 | } 240 | 241 | .container_12 .push_10 { 242 | left:800px; 243 | } 244 | 245 | .container_12 .push_11 { 246 | left:880px; 247 | } 248 | 249 | 250 | 251 | /* Pull Space >> 12 Columns 252 | ----------------------------------------------------------------------------------------------------*/ 253 | 254 | .container_12 .pull_1 { 255 | left:-80px; 256 | } 257 | 258 | .container_12 .pull_2 { 259 | left:-160px; 260 | } 261 | 262 | .container_12 .pull_3 { 263 | left:-240px; 264 | } 265 | 266 | .container_12 .pull_4 { 267 | left:-320px; 268 | } 269 | 270 | .container_12 .pull_5 { 271 | left:-400px; 272 | } 273 | 274 | .container_12 .pull_6 { 275 | left:-480px; 276 | } 277 | 278 | .container_12 .pull_7 { 279 | left:-560px; 280 | } 281 | 282 | .container_12 .pull_8 { 283 | left:-640px; 284 | } 285 | 286 | .container_12 .pull_9 { 287 | left:-720px; 288 | } 289 | 290 | .container_12 .pull_10 { 291 | left:-800px; 292 | } 293 | 294 | .container_12 .pull_11 { 295 | left:-880px; 296 | } 297 | 298 | 299 | 300 | 301 | /* Clear Floated Elements 302 | ----------------------------------------------------------------------------------------------------*/ 303 | 304 | /* http://sonspring.com/journal/clearing-floats */ 305 | 306 | .clear { 307 | clear: both; 308 | display: block; 309 | overflow: hidden; 310 | visibility: hidden; 311 | width: 0; 312 | height: 0; 313 | } 314 | 315 | /* http://perishablepress.com/press/2008/02/05/lessons-learned-concerning-the-clearfix-css-hack */ 316 | 317 | .clearfix:after { 318 | clear: both; 319 | content: ' '; 320 | display: block; 321 | font-size: 0; 322 | line-height: 0; 323 | visibility: hidden; 324 | width: 0; 325 | height: 0; 326 | } 327 | 328 | .clearfix { 329 | display: inline-block; 330 | } 331 | 332 | * html .clearfix { 333 | height: 1%; 334 | } 335 | 336 | .clearfix { 337 | display: block; 338 | } 339 | -------------------------------------------------------------------------------- /docs/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {%- block doctype -%} 2 | 4 | {%- endblock %} 5 | {%- set reldelim1 = reldelim1 is not defined and ' »' or reldelim1 %} 6 | {%- set reldelim2 = reldelim2 is not defined and ' |' or reldelim2 %} 7 | 8 | {%- macro relbar() %} 9 |
10 |

{{ _('Navigation') }}

11 |
    12 | {%- for rellink in rellinks %} 13 |
  • 14 | {{ rellink[3] }} 16 | {%- if not loop.first %}{{ reldelim2 }}{% endif %}
    • 17 | {%- endfor %} 18 | {%- block rootrellink %} 19 |
  • {{ shorttitle|e }}{{ reldelim1 }}
    • 20 | {%- endblock %} 21 | {%- for parent in parents %} 22 |
  • {{ parent.title }}{{ reldelim1 }}
    • 23 | {%- endfor %} 24 | {%- block relbaritems %} {% endblock %} 25 |
26 |
27 | {%- endmacro %} 28 | 29 | {%- macro sidebar() %} 30 | {%- if not embedded %}{% if not theme_nosidebar|tobool %} 31 |
32 |
33 | {%- block sidebarlogo %} 34 | {%- if logo %} 35 | 38 | {%- endif %} 39 | {%- endblock %} 40 | {%- block sidebartoc %} 41 | {%- if display_toc %} 42 |

{{ _('Table Of Contents') }}

43 | {{ toc }} 44 | {%- endif %} 45 | {%- endblock %} 46 | {%- block sidebarrel %} 47 | {%- if prev %} 48 |

{{ _('Previous topic') }}

49 |

{{ prev.title }}

51 | {%- endif %} 52 | {%- if next %} 53 |

{{ _('Next topic') }}

54 |

{{ next.title }}

56 | {%- endif %} 57 | {%- endblock %} 58 | {%- block sidebarsourcelink %} 59 | {%- if show_source and has_source and sourcename %} 60 |

{{ _('This Page') }}

61 | 65 | {%- endif %} 66 | {%- endblock %} 67 | {%- if customsidebar %} 68 | {% include customsidebar %} 69 | {%- endif %} 70 | {%- block sidebarsearch %} 71 | {%- if pagename != "search" %} 72 | 84 | 85 | {%- endif %} 86 | {%- endblock %} 87 |
88 |
89 | {%- endif %}{% endif %} 90 | {%- endmacro %} 91 | 92 | 93 | 94 | 95 | {{ metatags }} 96 | {%- if not embedded %} 97 | {%- set titlesuffix = " — "|safe + docstitle|e %} 98 | {%- else %} 99 | {%- set titlesuffix = "" %} 100 | {%- endif %} 101 | {{ title|striptags }}{{ titlesuffix }} 102 | 103 | 104 | 105 | {%- if not embedded %} 106 | 115 | {%- for scriptfile in script_files %} 116 | 117 | {%- endfor %} 118 | {%- if use_opensearch %} 119 | 122 | {%- endif %} 123 | {%- if favicon %} 124 | 125 | {%- endif %} 126 | {%- endif %} 127 | {%- block linktags %} 128 | {%- if hasdoc('about') %} 129 | 130 | {%- endif %} 131 | {%- if hasdoc('genindex') %} 132 | 133 | {%- endif %} 134 | {%- if hasdoc('search') %} 135 | 136 | {%- endif %} 137 | {%- if hasdoc('copyright') %} 138 | 139 | {%- endif %} 140 | 141 | {%- if parents %} 142 | 143 | {%- endif %} 144 | {%- if next %} 145 | 146 | {%- endif %} 147 | {%- if prev %} 148 | 149 | {%- endif %} 150 | {%- endblock %} 151 | {%- block extrahead %} {% endblock %} 152 | 153 | 154 | {%- block header %}{% endblock %} 155 |
156 | {%- block relbar1 %}{{ relbar() }}{% endblock %} 157 | 158 | {%- block sidebar1 %} {# possible location for sidebar #} {% endblock %} 159 | 160 |
161 | {%- block document %} 162 |
163 | {%- if not embedded %}{% if not theme_nosidebar|tobool %} 164 |
165 | {%- endif %}{% endif %} 166 |
167 | {% block body %} {% endblock %} 168 |
169 | {%- if not embedded %}{% if not theme_nosidebar|tobool %} 170 |
171 | {%- endif %}{% endif %} 172 |
173 | {%- endblock %} 174 | 175 | {%- block sidebar2 %}{{ sidebar() }}{% endblock %} 176 |
177 |
178 |
179 | 180 | {%- block relbar2 %}{{ relbar() }}{% endblock %} 181 | 182 | {%- block footer %} 183 |
184 | {%- if hasdoc('copyright') %} 185 | {% trans path=pathto('copyright'), copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %} 186 | {%- else %} 187 | {% trans copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %} 188 | {%- endif %} 189 | {%- if last_updated %} 190 | {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} 191 | {%- endif %} 192 | {%- if show_sphinx %} 193 | {% trans sphinx_version=sphinx_version|e %}Created using Sphinx {{ sphinx_version }}.{% endtrans %} 194 | {%- endif %} 195 |
196 | {%- endblock %} 197 | 198 | 199 | -------------------------------------------------------------------------------- /docs/basics.rst: -------------------------------------------------------------------------------- 1 | coopy basics 2 | =================================== 3 | 4 | **coopy** returns to you a proxy to your object. Everytime you call some method on this proxy, coopy will log to disk this operation, so it can be re-executed later on a restore process. This behaviour assures that you object will have their state persisted. 5 | 6 | So far, you know that you are manipulating a proxy object and when you call methods on this object, this invocation will be written to disk. We call **log file** the files that contains all operations executed on your object. This log files are created on what we call **basedir**. You can specify basedir or coopy will lowercase your object class name and create a directory with this name to store all log files. 7 | 8 | **coopy logger** is responsible to receive this methods invocations, create **Action** objects and serialize to disk. It automatically handles file rotations, like python logging RotateFileHandler, in order to keep log files not too big. 9 | 10 | As your application is running, your log file number will be increasing and restore process can start to run slowly, becase it'll open many log files. To avoid that you can take **snapshots** from your object. 11 | **Snapshot file** is a copy of your objects in memory serialized trhough the disk. As you take a snapshot, all log files older than this snapshots can be deleted if you want. Take snapshots will also speed up the restore process, because is much more fast open 1 file and deserialize to memory than open 10 files to execute each action inside of them. 12 | 13 | Now, you know everything about how information are stored. Let's see how this information are restored. 14 | 15 | **Restore process** is what coopy do to restore your object state. It checks for **log files** and **snapshot files** on your **basedir** to look to the last snapshot taken and all log files created after. It'll deserialize this **snapshot file** and then open all log files to re-execute all **Actions** that were executed after the snapshot was created. This will assure that your object will have the same state as your object had once in the past when your program was terminated or maybe killed. 16 | 17 | The bascis of **coopy** is covered here 18 | 19 | * You are manipulating a proxy object that delegates memory execution to your **domain** object 20 | * Once you call a method on proxy, this call turns into a **Action** object and then serialized to disk. 21 | * **Log files** contain **Action** objects to be re-executed 22 | * You can take **Snapshots** of your object to increase your **restore process** and have a small number of files on your **basedir** 23 | * Everytime you use **coopy** it'll look to your **basedir** and restore your object state with the files there 24 | 25 | All this is done using python cPickle module. 26 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. _changelog: 2 | 3 | Changelog 4 | ========= 5 | 6 | Nothing so far. 7 | -------------------------------------------------------------------------------- /docs/client_server.rst: -------------------------------------------------------------------------------- 1 | .. _client_server: 2 | 3 | Client-Server 4 | ------------- 5 | 6 | When you want to detach client from server, you can use coopy + Pyro (or xmlrpclib) in order to have a client and a server (running coopy). 7 | 8 | This is useful when you want to have only one machine dedicated to have it's ram memory filled with python objects. 9 | 10 | Note, that this example uses Pyro.core.ObjBase instead of Pyro.core.SynchronizedObjBase, because by default, coopy proxy (wiki object) is already thread-safe unless you decorate your business methods with @unlocked decorator. 11 | 12 | Server Code:: 13 | 14 | #coopy code 15 | wiki = coopy.init_system(Wiki(), "pyro") 16 | 17 | #pyro code 18 | obj = Pyro.core.ObjBase() 19 | obj.delegateTo(wiki) 20 | Pyro.core.initServer() 21 | daemon=Pyro.core.Daemon() 22 | uri=daemon.connect(obj,"wiki") 23 | daemon.requestLoop() 24 | 25 | Client code:: 26 | 27 | #pyro code 28 | wiki = Pyro.core.getProxyForURI("PYRO://127.0.0.1:7766/whatever") 29 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # coopy documentation build configuration file, created by 4 | # sphinx-quickstart on Tue May 18 18:06:23 2010. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.append(os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # Add any Sphinx extension module names here, as strings. They can be extensions 24 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 25 | extensions = []#rst2pdf.pdfbuilder] 26 | 27 | # Add any paths that contain templates here, relative to this directory. 28 | templates_path = ['_templates'] 29 | 30 | # The suffix of source filenames. 31 | source_suffix = '.rst' 32 | 33 | # The encoding of source files. 34 | #source_encoding = 'utf-8' 35 | 36 | # The master toctree document. 37 | master_doc = 'index' 38 | 39 | # General information about the project. 40 | project = u'coopy' 41 | copyright = u'2012, Felipe Cruz' 42 | 43 | # The version info for the project you're documenting, acts as replacement for 44 | # |version| and |release|, also used in various other places throughout the 45 | # built documents. 46 | # 47 | # The short X.Y version. 48 | version = '0.4' 49 | # The full version, including alpha/beta/rc tags. 50 | release = '0.4b' 51 | 52 | # The language for content autogenerated by Sphinx. Refer to documentation 53 | # for a list of supported languages. 54 | #language = None 55 | 56 | # There are two options for replacing |today|: either, you set today to some 57 | # non-false value, then it is used: 58 | #today = '' 59 | # Else, today_fmt is used as the format for a strftime call. 60 | #today_fmt = '%B %d, %Y' 61 | 62 | # List of documents that shouldn't be included in the build. 63 | #unused_docs = [] 64 | 65 | # List of directories, relative to source directory, that shouldn't be searched 66 | # for source files. 67 | exclude_trees = [] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'tango' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. Major themes that come with 93 | # Sphinx are currently 'default' and 'sphinxdoc'. 94 | html_theme = 'default' 95 | html_theme_options = { 96 | "rightsidebar" : "true", 97 | "relbarbgcolor" : "white", 98 | "footerbgcolor" : "white", 99 | "footertextcolor" : "black", 100 | "sidebarbgcolor" : "white", 101 | "sidebartextcolor" : "black", 102 | "sidebarlinkcolor" : "red", 103 | "relbarbgcolor" : "white", 104 | "relbartextcolor" : "black", 105 | "relbarlinkcolor" : "red", 106 | "bgcolor" : "white", 107 | "textcolor" : "black", 108 | "linkcolor" : "red", 109 | "headbgcolor" : "white", 110 | "headtextcolor" : "red", 111 | "headlinkcolor" : "red", 112 | "codebgcolor" : "white", 113 | "codetextcolor" : "black", 114 | "bodyfont" : "arial,helvetica,clean,sans-serif", 115 | "headfont" : "arial,helvetica,clean,sans-serif" 116 | } 117 | 118 | # Theme options are theme-specific and customize the look and feel of a theme 119 | # further. For a list of options available for each theme, see the 120 | # documentation. 121 | #html_theme_options = {} 122 | 123 | # Add any paths that contain custom themes here, relative to this directory. 124 | #html_theme_path = [] 125 | 126 | # The name for this set of Sphinx documents. If None, it defaults to 127 | # " v documentation". 128 | #html_title = None 129 | 130 | # A shorter title for the navigation bar. Default is the same as html_title. 131 | #html_short_title = None 132 | 133 | # The name of an image file (relative to this directory) to place at the top 134 | # of the sidebar. 135 | #html_logo = None 136 | 137 | # The name of an image file (within the static path) to use as favicon of the 138 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 139 | # pixels large. 140 | #html_favicon = None 141 | 142 | # Add any paths that contain custom static files (such as style sheets) here, 143 | # relative to this directory. They are copied after the builtin static files, 144 | # so a file named "default.css" will overwrite the builtin "default.css". 145 | html_static_path = ['_static'] 146 | 147 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 148 | # using the given strftime format. 149 | #html_last_updated_fmt = '%b %d, %Y' 150 | 151 | # If true, SmartyPants will be used to convert quotes and dashes to 152 | # typographically correct entities. 153 | #html_use_smartypants = True 154 | 155 | # Custom sidebar templates, maps document names to template names. 156 | #html_sidebars = {} 157 | 158 | # Additional templates that should be rendered to pages, maps page names to 159 | # template names. 160 | #html_additional_pages = {} 161 | 162 | # If false, no module index is generated. 163 | #html_use_modindex = True 164 | 165 | # If false, no index is generated. 166 | html_use_index = False 167 | 168 | # If true, the index is split into individual pages for each letter. 169 | #html_split_index = False 170 | 171 | # If true, links to the reST sources are added to the pages. 172 | #html_show_sourcelink = True 173 | 174 | # If true, an OpenSearch description file will be output, and all pages will 175 | # contain a tag referring to it. The value of this option must be the 176 | # base URL from which the finished HTML is served. 177 | #html_use_opensearch = '' 178 | 179 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). 180 | #html_file_suffix = '' 181 | 182 | # Output file base name for HTML help builder. 183 | htmlhelp_basename = 'coopydoc' 184 | 185 | 186 | # -- Options for LaTeX output -------------------------------------------------- 187 | 188 | # The paper size ('letter' or 'a4'). 189 | #latex_paper_size = 'letter' 190 | 191 | # The font size ('10pt', '11pt' or '12pt'). 192 | #latex_font_size = '10pt' 193 | 194 | # Grouping the document tree into LaTeX files. List of tuples 195 | # (source start file, target name, title, author, documentclass [howto/manual]). 196 | latex_documents = [ 197 | ('index', 'coopy.tex', u'coopy Documentation', 198 | u'Felipe Cruz', 'manual'), 199 | ] 200 | 201 | # The name of an image file (relative to this directory) to place at the top of 202 | # the title page. 203 | #latex_logo = None 204 | 205 | # For "manual" documents, if this is true, then toplevel headings are parts, 206 | # not chapters. 207 | #latex_use_parts = False 208 | 209 | # Additional stuff for the LaTeX preamble. 210 | #latex_preamble = '' 211 | 212 | # Documents to append as an appendix to all manuals. 213 | #latex_appendices = [] 214 | 215 | # If false, no module index is generated. 216 | #latex_use_modindex = True 217 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. coopy documentation master file, created by 2 | sphinx-quickstart on Tue May 18 18:06:23 2010. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | coopy 7 | ------- 8 | 9 | **coopy** is a simple, transparent, non-intrusive persistence library for python language. It's released under `BSD License`_ 10 | 11 | * **Simple** - you don't have to learn an API. You can use it with just one line of code. 12 | * **Transparent** - you don't need to call any API functions, just your Object methods. 13 | * **Non-Intrusive** - no inheritance, no interface.. only pure-python-business code. 14 | 15 | It is based on the techniques of system snapshotting and transaction journalling. In the prevalent model, the object data is kept in memory in native object format, rather than being marshalled to an RDBMS or other data storage system. A snapshot of data is regularly saved to disk, and in addition to this, all changes are serialised and a log of transactions is also stored on disk. 16 | 17 | http://en.wikipedia.org/wiki/Object_prevalence 18 | 19 | Using 20 | ----- 21 | 22 | Simple, transparent and non-intrusive. Note that ``Todo`` could be any class 23 | that you want to persist state across method calls that modifies it's internal 24 | state:: 25 | 26 | from coopy.base import init_persistent_system 27 | 28 | class Todo(object): 29 | def __init__(self): 30 | self.tasks = [] 31 | 32 | def add_task(self, name, description): 33 | task = dict(name=name, description=description) 34 | self.tasks.append(task) 35 | 36 | persistent_todo_list = init_persistent_system(Todo()) 37 | persistent_todo_list.add_task("Some Task Name", "A Task Description") 38 | 39 | Check out how coopy works with this little :doc:`tutorial` and then... 40 | 41 | It's very important to know how coopy works, to use it. Check out :doc:`basics` 42 | 43 | 44 | Restrictions 45 | ------------ 46 | 47 | This should not affect end-user code. To get datetime or date objects you 48 | need to get from an internal clock. Check this page :doc:`use_clock` 49 | 50 | Status 51 | ------ 52 | 53 | coopy is compatible with py2.6, py2.7, py3.2, py3.3 and pypy. 54 | 55 | contribute 56 | ---------- 57 | 58 | **coopy** code is hosted on github at: http://github.com/felipecruz/coopy 59 | 60 | Found a bug? http://github.com/felipecruz/coopy 61 | 62 | contents 63 | -------- 64 | 65 | .. toctree:: 66 | :maxdepth: 2 67 | 68 | installation 69 | tutorial 70 | usage 71 | basics 72 | use_clock 73 | method_decorators 74 | snapshots 75 | util_api 76 | client_server 77 | replication 78 | tests 79 | roadmap 80 | changelog 81 | todo 82 | 83 | .. _BSD License: http://www.opensource.org/licenses/bsd-license.php 84 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. _installation: 2 | 3 | Installation 4 | ============ 5 | :: 6 | 7 | $ pip install coopy 8 | 9 | Development Version 10 | ``````````````````` 11 | 12 | You can always check our bleeding-edge development version:: 13 | 14 | $ git clone http://github.com/felipecruz/coopy.git 15 | 16 | and then:: 17 | 18 | $ python setup.py install 19 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | set SPHINXBUILD=sphinx-build 6 | set BUILDDIR=build 7 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 8 | if NOT "%PAPER%" == "" ( 9 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 10 | ) 11 | 12 | if "%1" == "" goto help 13 | 14 | if "%1" == "help" ( 15 | :help 16 | echo.Please use `make ^` where ^ is one of 17 | echo. html to make standalone HTML files 18 | echo. dirhtml to make HTML files named index.html in directories 19 | echo. pickle to make pickle files 20 | echo. json to make JSON files 21 | echo. htmlhelp to make HTML files and a HTML help project 22 | echo. qthelp to make HTML files and a qthelp project 23 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 24 | echo. changes to make an overview over all changed/added/deprecated items 25 | echo. linkcheck to check all external links for integrity 26 | echo. doctest to run all doctests embedded in the documentation if enabled 27 | goto end 28 | ) 29 | 30 | if "%1" == "clean" ( 31 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 32 | del /q /s %BUILDDIR%\* 33 | goto end 34 | ) 35 | 36 | if "%1" == "html" ( 37 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 38 | echo. 39 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 40 | goto end 41 | ) 42 | 43 | if "%1" == "dirhtml" ( 44 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 45 | echo. 46 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 47 | goto end 48 | ) 49 | 50 | if "%1" == "pickle" ( 51 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 52 | echo. 53 | echo.Build finished; now you can process the pickle files. 54 | goto end 55 | ) 56 | 57 | if "%1" == "json" ( 58 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 59 | echo. 60 | echo.Build finished; now you can process the JSON files. 61 | goto end 62 | ) 63 | 64 | if "%1" == "htmlhelp" ( 65 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 66 | echo. 67 | echo.Build finished; now you can run HTML Help Workshop with the ^ 68 | .hhp project file in %BUILDDIR%/htmlhelp. 69 | goto end 70 | ) 71 | 72 | if "%1" == "qthelp" ( 73 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 74 | echo. 75 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 76 | .qhcp project file in %BUILDDIR%/qthelp, like this: 77 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\coopy.qhcp 78 | echo.To view the help file: 79 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\coopy.ghc 80 | goto end 81 | ) 82 | 83 | if "%1" == "latex" ( 84 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 85 | echo. 86 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 87 | goto end 88 | ) 89 | 90 | if "%1" == "changes" ( 91 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 92 | echo. 93 | echo.The overview file is in %BUILDDIR%/changes. 94 | goto end 95 | ) 96 | 97 | if "%1" == "linkcheck" ( 98 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 99 | echo. 100 | echo.Link check complete; look for any errors in the above output ^ 101 | or in %BUILDDIR%/linkcheck/output.txt. 102 | goto end 103 | ) 104 | 105 | if "%1" == "doctest" ( 106 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 107 | echo. 108 | echo.Testing of doctests in the sources finished, look at the ^ 109 | results in %BUILDDIR%/doctest/output.txt. 110 | goto end 111 | ) 112 | 113 | :end 114 | -------------------------------------------------------------------------------- /docs/method_decorators.rst: -------------------------------------------------------------------------------- 1 | .. _method_decorators: 2 | 3 | Method Decorators 4 | ----------------- 5 | 6 | @readonly 7 | ========= 8 | 9 | This decorator means thar your method will not mofidy business objects. Like a get method from a Wiki class. Therefore, this method will not generate a log entry at coopy actions log:: 10 | 11 | from coopy.decorators import readonly 12 | @readonly 13 | def get_page(self, id): 14 | if id in self.pages: 15 | return self.pages[id] 16 | else: 17 | return None 18 | 19 | @unlocked 20 | ========= 21 | 22 | How coopy assures thread-safety? By synchronizing method invocations using a reetrant lock. 23 | 24 | This decorator provides a means of leaving the thread safety in your hands via the @unlocked decorator. Using this decorator, you should implement concurrency mechanism by yourself. 25 | 26 | @abort_exception 27 | ================ 28 | 29 | Default behaviour is to log on disk, even if your code raises an exception. 30 | 31 | If your 'business' method raises an exception and your method is decoreted by @abort_exception, this execution will not be logged at disk. This means that during restore process, this invocation that raised an exception will not be re-executed:: 32 | 33 | from coopy.decorators import abort_exception 34 | @abort_exception 35 | def create_page(self, wikipage): 36 | page = None 37 | wikipage.last_modify = coopy.clock.now() 38 | if wikipage.id in self.pages: 39 | page = self.pages[wikipage.id] 40 | if not page: 41 | self.pages[wikipage.id] = wikipage 42 | raise Exception('Exemple error') 43 | else: 44 | self.update_page(wikipage.id, wikipage.content) 45 | 46 | 47 | **Restore process will not execute this method because it wasn't logged at disk.** 48 | -------------------------------------------------------------------------------- /docs/replication.rst: -------------------------------------------------------------------------------- 1 | .. _replication: 2 | 3 | Master/Slave replication 4 | ------------------------ 5 | 6 | **coopy** comes with master/slave replication mechanism. 7 | 8 | Basically: 9 | 10 | * Master instance are read/write 11 | * Slaves are read only 12 | * Slaves can only execute @readonly methods. 13 | 14 | Another detail, is that you can set a password on master. This password provides basich auth to slaves connects to a master instance. 15 | 16 | When slaves connects to master and passes authentication process, it will receive all data to synchronize with master state. Commands executed further on master will be replicated to slave node. 17 | 18 | Slaves are useful to take snapshots without needing master to be locked as well to provide load balancing for reading. 19 | 20 | Snipets to run master and slave instances 21 | 22 | Master instance:: 23 | 24 | init_system(Wiki, master=True) 25 | 26 | Slave instance, default host and default port:: 27 | 28 | init_system(Wiki, replication=True) 29 | -------------------------------------------------------------------------------- /docs/roadmap.rst: -------------------------------------------------------------------------------- 1 | .. _roadmap: 2 | 3 | Roadmap 4 | ======= 5 | 6 | * First stable release 7 | -------------------------------------------------------------------------------- /docs/snapshots.rst: -------------------------------------------------------------------------------- 1 | .. _snapshots: 2 | 3 | Snapshots 4 | ========= 5 | 6 | Motivation 7 | ---------- 8 | 9 | If your domain is really active and generates tons of logs, we suggest you to take snapshots from your domain periodically. A snapshot allows you to delete your logs older then it's timestamp and make the restore process faster. 10 | Today, while taking a snapshot the domain is *locked*. It's fairly common setup a local slave just for taking snapshots. 11 | 12 | Example 13 | ------- 14 | 15 | Example:: 16 | 17 | from coopy.base import init_persistent_system 18 | 19 | persistent_todo_list = init_persistent_system(Todo()) 20 | persistent_todo_list.add_task("Some Task Name", "A Task Description") 21 | 22 | # Take snapshot 23 | persistent_todo_list.take_snapshot() 24 | 25 | 26 | API 27 | --- 28 | 29 | For domain instances 30 | 31 | .. function:: domain.take_snapshot() 32 | Takes the domain snapshot. 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /docs/tests.rst: -------------------------------------------------------------------------------- 1 | .. _tests: 2 | 3 | Tests 4 | ----- 5 | 6 | First time:: 7 | 8 | pip install -r requirements.txt 9 | 10 | To actually run the tests:: 11 | 12 | make test 13 | 14 | Coverage Report 15 | --------------- 16 | 17 | First time:: 18 | 19 | pip install -r requirements.txt 20 | 21 | And then:: 22 | 23 | make coverage 24 | 25 | Coverage report:: 26 | 27 | $ py.test --cov coopy 28 | 29 | Name Stmts Miss Cover 30 | -------------------------------------------------- 31 | coopy/__init__ 0 0 100% 32 | coopy/base 135 17 87% 33 | coopy/decorators 9 0 100% 34 | coopy/error 3 0 100% 35 | coopy/fileutils 125 5 96% 36 | coopy/foundation 71 8 89% 37 | coopy/journal 30 2 93% 38 | coopy/network/__init__ 1 0 100% 39 | coopy/network/default_select 192 56 71% 40 | coopy/network/linux_epoll 0 0 100% 41 | coopy/network/network 42 10 76% 42 | coopy/network/osx_kqueue 0 0 100% 43 | coopy/restore 42 6 86% 44 | coopy/snapshot 45 3 93% 45 | coopy/utils 9 0 100% 46 | coopy/validation 45 1 98% 47 | -------------------------------------------------- 48 | TOTAL 749 108 86% 49 | -------------------------------------------------------------------------------- /docs/todo.rst: -------------------------------------------------------------------------------- 1 | .. _todo: 2 | 3 | TODO 4 | ==== 5 | 6 | * Finish Documentation 7 | -------------------------------------------------------------------------------- /docs/tutorial.rst: -------------------------------------------------------------------------------- 1 | 1 minute coopy tutorial 2 | ========================= 3 | 4 | **coopy** enforces you to implement code in the object-oriented way. Imagine a wiki system:: 5 | 6 | class WikiPage(): 7 | def __init__(self, id, content): 8 | self.id = id 9 | self.content = content 10 | self.history = [] 11 | self.last_modify = datetime.datetime.now() 12 | 13 | class Wiki(): 14 | def __init__(self): 15 | self.pages = {} 16 | def create_page(self, page_id, content): 17 | page = None 18 | if page_id in self.pages: 19 | page = self.pages[page_id] 20 | if not page: 21 | page = WikiPage(page_id, content) 22 | self.pages[page_id] = page 23 | return page 24 | 25 | It's very easy to implement a wiki system thinking only on it's objects. Let's move forward:: 26 | 27 | from coopy import init_system 28 | wiki = init_system(Wiki(), "/path/to/somedir") 29 | wiki.create_page('My First Page', 'My First Page Content') 30 | 31 | That's all you need to use coopy. If you stop your program and run again:: 32 | 33 | from coopy import init_system 34 | wiki = init_system(Wiki(), "/path/to/somedir") 35 | page = wiki.pages['My First Page'] 36 | 37 | If you want to know how coopy works, check out :doc:`basics` 38 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | .. _usage: 2 | 3 | Using coopy 4 | ============= 5 | 6 | There are many different ways to use **coopy**. Let me show you some:: 7 | 8 | class WikiPage(): 9 | def __init__(self, id, content): 10 | self.id = id 11 | self.content = content 12 | self.history = [] 13 | self.last_modify = datetime.datetime.now() 14 | 15 | class Wiki(): 16 | def __init__(self): 17 | self.pages = {} 18 | def create_page(self, page_id, content): 19 | page = None 20 | if page_id in self.pages: 21 | page = self.pages[page_id] 22 | if not page: 23 | page = WikiPage(page_id, content) 24 | self.pages[page_id] = page 25 | return page 26 | 27 | wiki = init_system(Wiki) 28 | 29 | or:: 30 | 31 | wiki = init_system(Wiki()) 32 | 33 | or:: 34 | 35 | wiki = init_system(Wiki(),'/path/to/log/files') 36 | 37 | or setup a Master node:: 38 | 39 | init_system(Wiki, master=True) 40 | 41 | or setup a Slave node:: 42 | 43 | init_system(Wiki, replication=True) 44 | 45 | or check all arguments:: 46 | 47 | def init_system(obj, basedir=None, snapshot_time=0, master=False, replication=False, port=5466, host='127.0.0.1', password='copynet'): 48 | -------------------------------------------------------------------------------- /docs/use_clock.rst: -------------------------------------------------------------------------------- 1 | .. _use_clock: 2 | 3 | How to Use Clock 4 | ================ 5 | 6 | Date problem 7 | ```````````` 8 | 9 | **coopy** is based on re-execute actions performed in the past. When you call datetime.now() inside an 'business' method, when your actions are executed in restore process, datetime.now() will be executed again. This behaviour will produce unexpected results. 10 | 11 | Why use Clock? 12 | `````````````` 13 | 14 | Clock uses **coopy** timestamp. When you execute a 'business' method, coopy takes the current timestamp and persist inside action object. Clock object has his timestamp updated with action timestamp so in a restore process, Clock will have the original timestamp, and not the timestamp from the re-execution process. 15 | 16 | 17 | Wrong code:: 18 | 19 | def create_page(self, wikipage): 20 | page = None 21 | wikipage.last_modify = datetime.now() 22 | .... 23 | 24 | Right code:: 25 | 26 | from coopy import clock 27 | def create_page(self, wikipage): 28 | page = None 29 | wikipage.last_modify = self._clock.now() 30 | .... 31 | 32 | 33 | Clock API 34 | ````````` 35 | 36 | Clock-aware code validation 37 | --------------------------- 38 | 39 | coopy has a validation mechanism that will not accept obvious code errors such as 40 | calling ``datetime.now()`` inside a system method. 41 | 42 | API 43 | --- 44 | 45 | Take note that a ``_clock`` attribute is injected on your system instance and the API is 46 | always called via ``self._clock``. 47 | 48 | For Clock instances 49 | 50 | .. function:: clock.now() 51 | Return datetime.now() 52 | 53 | :rtype: datetime 54 | 55 | 56 | .. function:: clock.utcnow() 57 | Return datetime.now() 58 | 59 | :rtype: datetime 60 | 61 | 62 | .. function:: clock.today() 63 | Return date.today() 64 | 65 | :rtype: date 66 | -------------------------------------------------------------------------------- /docs/util_api.rst: -------------------------------------------------------------------------------- 1 | .. _usage: 2 | 3 | Utilitary API 4 | ============= 5 | 6 | There are some utilitary methods to help you. 7 | 8 | Given:: 9 | 10 | wiki = init_system(Wiki) 11 | 12 | 13 | .. function:: basedir_abspath() 14 | 15 | Return a list with all basedirs absolute paths 16 | 17 | 18 | Tests utils 19 | ----------- 20 | 21 | If your domain uses the :doc:`use_clock` feature, you'll likely to face errors while 22 | testing your pure domain since the `_clock` is injected by coopy. 23 | 24 | There are 2 ways of handle this: Enable a regular clock on your domain, for testing 25 | or mock your clock to return the same date. 26 | 27 | .. function:: TestSystemMixin.mock_clock(domain, mocked_date) 28 | 29 | This method will inject a clock that always return `mocked_date` 30 | 31 | .. function:: TestSystemMixin.enable_clock(domain) 32 | 33 | This method will inject a regular coopy clock on your domain instance 34 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | six 3 | pytest-cov 4 | mock 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages 2 | from distutils.core import setup 3 | 4 | setup(name='coopy', 5 | version='0.4.6b', 6 | description='coopy - plain objects persistence/prevalence tool', 7 | author='Felipe Cruz', 8 | author_email='felipecruz@loogica.net', 9 | url='http://coopy.readthedocs.org', 10 | packages=find_packages(), 11 | install_requires=['six']) 12 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felipecruz/coopy/507221f2dead6145e884dea99999d603ee175178/tests/__init__.py -------------------------------------------------------------------------------- /tests/domain.py: -------------------------------------------------------------------------------- 1 | from coopy.decorators import readonly, abort_exception, unlocked 2 | 3 | class WikiPage(object): 4 | def __init__(self, id, content, last_modified, parent=None): 5 | self.id = id 6 | self.content = content 7 | self.history = [] 8 | self.last_modified = last_modified 9 | self.parent = parent 10 | 11 | class Wiki(object): 12 | def __init__(self): 13 | self.pages = {} 14 | 15 | def create_page(self, id, content, parent): 16 | page = WikiPage(id, content, self._clock.now(), parent) 17 | 18 | if not id in self.pages: 19 | self.pages[id] = page 20 | else: 21 | self.update_page(id, content, parent) 22 | 23 | return page 24 | 25 | @readonly 26 | def get_page(self, id): 27 | if id in self.pages: 28 | return self.pages[id] 29 | else: 30 | return None 31 | 32 | def update_page(self, page_id, content, parent): 33 | old_page = self.pages[page_id] 34 | page = WikiPage(id, content, self._clock.now(), parent) 35 | page = self.add_history(old_page, page) 36 | self.pages[page_id] = page 37 | 38 | 39 | def add_history(self, old_page, page): 40 | page.history = old_page.history 41 | page.history.append(old_page) 42 | return page 43 | 44 | def multiple_call_now(self): 45 | self.dt1 = self._clock.now() 46 | import time 47 | time.sleep(0.1) 48 | self.dt2 = self._clock.now() 49 | 50 | @abort_exception 51 | def check_abort_exception(self): 52 | raise Exception("Abort Exception") 53 | 54 | @unlocked 55 | def unlocked_method(self): 56 | pass 57 | -------------------------------------------------------------------------------- /tests/functional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felipecruz/coopy/507221f2dead6145e884dea99999d603ee175178/tests/functional/__init__.py -------------------------------------------------------------------------------- /tests/functional/test_coopy.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import unittest 4 | 5 | TEST_DIR = 'coopy_test/' 6 | TEST_DIR_AUX = 'coopynet_test/' 7 | 8 | 9 | class TestCoopy(unittest.TestCase): 10 | def setUp(self): 11 | os.mkdir(TEST_DIR) 12 | 13 | def tearDown(self): 14 | shutil.rmtree(TEST_DIR) 15 | 16 | def test_coopy(self): 17 | from ..domain import Wiki 18 | import coopy.base 19 | 20 | wiki = coopy.base.init_persistent_system(Wiki, basedir=TEST_DIR) 21 | 22 | page_a = wiki.create_page('Home','Welcome', None) 23 | date_a = page_a.last_modified 24 | 25 | page_b = wiki.create_page('coopy', 26 | 'http://bitbucket.org/loogica/coopy', 27 | page_a) 28 | date_b = page_b.last_modified 29 | 30 | page_c = wiki.create_page('Test data','0.2', page_a) 31 | date_c = page_c.last_modified 32 | wiki.close() 33 | 34 | new_wiki = coopy.base.init_persistent_system(Wiki(), basedir=TEST_DIR) 35 | 36 | self.assertEqual('Home', new_wiki.get_page('Home').id) 37 | self.assertEqual('Welcome', new_wiki.get_page('Home').content) 38 | 39 | self.assertEqual('coopy', new_wiki.get_page('coopy').id) 40 | self.assertEqual('http://bitbucket.org/loogica/coopy', 41 | new_wiki.get_page('coopy').content) 42 | 43 | self.assertEqual('Test data', new_wiki.get_page('Test data').id) 44 | self.assertEqual('0.2', new_wiki.get_page('Test data').content) 45 | 46 | self.assertEqual(date_a, new_wiki.get_page('Home').last_modified) 47 | self.assertEqual(date_b, new_wiki.get_page('coopy').last_modified) 48 | self.assertEqual(date_c, new_wiki.get_page('Test data').last_modified) 49 | new_wiki.close() 50 | 51 | def test_coopy_multiple_calls_now(self): 52 | from ..domain import Wiki 53 | import coopy.base 54 | 55 | wiki = coopy.base.init_persistent_system(Wiki, basedir=TEST_DIR) 56 | 57 | wiki.multiple_call_now() 58 | dt1 = wiki.dt1 59 | dt2 = wiki.dt2 60 | 61 | wiki.close() 62 | new_wiki = coopy.base.init_persistent_system(Wiki(), basedir=TEST_DIR) 63 | 64 | self.assertEqual(dt1, new_wiki.dt1) 65 | self.assertEqual(dt2, new_wiki.dt2) 66 | 67 | new_wiki.close() 68 | 69 | def test_bad_system(self): 70 | ''' 71 | Because calls to datetime.{now(),utcnow()} or to date.today() 72 | aren't allowed. Use the clock: 73 | http://coopy.readthedocs.org/en/latest/use_clock.html 74 | ''' 75 | from ..domain import Wiki 76 | import coopy.base 77 | from coopy.error import PrevalentError 78 | 79 | class BadWiki(Wiki): 80 | def bad_method(self): 81 | from datetime import date 82 | dt = date.today() 83 | 84 | self.assertRaises(PrevalentError, 85 | coopy.base.init_persistent_system, 86 | *[BadWiki], **dict(basedir=TEST_DIR)) 87 | 88 | def test_enable_clock(self): 89 | from coopy.tests.utils import TestSystemMixin 90 | class DummyWiky(): 91 | def __init__(self): 92 | pass 93 | 94 | dummy_wiki = DummyWiky() 95 | assert not hasattr(dummy_wiki, '_clock') 96 | TestSystemMixin().enable_clock(dummy_wiki) 97 | assert hasattr(dummy_wiki, '_clock') 98 | 99 | 100 | def test_mock_clock(self): 101 | import datetime 102 | from coopy.tests.utils import TestSystemMixin 103 | class DummyWiky(): 104 | def __init__(self): 105 | pass 106 | 107 | dt = datetime.datetime.now() 108 | dummy_wiki = DummyWiky() 109 | assert not hasattr(dummy_wiki, '_clock') 110 | TestSystemMixin().mock_clock(dummy_wiki, dt) 111 | assert hasattr(dummy_wiki, '_clock') 112 | assert dt == dummy_wiki._clock.now() 113 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felipecruz/coopy/507221f2dead6145e884dea99999d603ee175178/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/test_base.py: -------------------------------------------------------------------------------- 1 | import os 2 | import six 3 | import unittest 4 | 5 | import pytest 6 | 7 | from ..domain import Wiki 8 | 9 | if six.PY3: 10 | from unittest import mock 11 | else: 12 | import mock 13 | 14 | class TestBase(unittest.TestCase): 15 | def tearDown(self): 16 | import shutil 17 | try: 18 | shutil.rmtree('wiki/') 19 | except: 20 | "nothing to do" 21 | pass 22 | 23 | def test_base(self): 24 | from coopy.base import init_system, CoopyProxy 25 | 26 | dummy_subscribers = ['a'] 27 | 28 | dummy = init_system(Wiki, dummy_subscribers) 29 | 30 | self.assertEquals(dummy.__class__, CoopyProxy) 31 | self.assertEquals(dummy.publisher.subscribers, dummy_subscribers) 32 | 33 | 34 | 35 | def test_base_persistent(self): 36 | import os 37 | from coopy.base import init_persistent_system, CoopyProxy 38 | 39 | dummy = init_persistent_system(Wiki, basedir='wiki/') 40 | 41 | files = os.listdir('wiki/') 42 | self.assertTrue('snapshot_000000000000002.dat' in files) 43 | self.assertEquals(dummy.__class__, CoopyProxy) 44 | self.assertEquals(len(dummy.publisher.subscribers), 1) 45 | 46 | dummy.close() 47 | 48 | def test_base_basedir_abspath(self): 49 | import os 50 | import tempfile 51 | from coopy.base import init_system, init_persistent_system, CoopyProxy 52 | from coopy.journal import DiskJournal 53 | 54 | dummy = init_persistent_system(Wiki, basedir='wiki/') 55 | 56 | current_dir = os.path.abspath(os.getcwd()) 57 | self.assertEquals([current_dir + "/wiki/"], 58 | dummy.basedir_abspath()) 59 | 60 | dummy.close() 61 | 62 | dir1 = tempfile.mkdtemp() 63 | dir2 = tempfile.mkdtemp() 64 | 65 | j1 = DiskJournal(dir1, os.getcwd()) 66 | j2 = DiskJournal(dir2, os.getcwd()) 67 | 68 | subscribers = [j1, j2] 69 | 70 | dummy2 = init_system(Wiki, subscribers) 71 | self.assertTrue(dir1 in dummy2.basedir_abspath()) 72 | self.assertTrue(dir2 in dummy2.basedir_abspath()) 73 | 74 | 75 | class TestCoopyProxy(unittest.TestCase): 76 | def tearDown(self): 77 | import shutil 78 | try: 79 | shutil.rmtree('wiki/') 80 | except: 81 | "nothing to do" 82 | pass 83 | 84 | def test_coopyproxy_init(self): 85 | from coopy.base import CoopyProxy 86 | 87 | import os 88 | os.mkdir('wiki') 89 | 90 | class PassPublisher(object): 91 | def close(self): 92 | pass 93 | def receive(self): 94 | pass 95 | 96 | proxy = CoopyProxy(Wiki(), [PassPublisher()]) 97 | 98 | self.assertTrue(hasattr(proxy, 'obj')) 99 | self.assertTrue(hasattr(proxy, 'publisher')) 100 | self.assertTrue(hasattr(proxy, 'lock')) 101 | self.assertTrue(hasattr(proxy, 'take_snapshot')) 102 | self.assertTrue(hasattr(proxy, 'close')) 103 | self.assertTrue(hasattr(proxy, 'shutdown')) 104 | 105 | proxy.close() 106 | 107 | def test_coopyproxy_start_snapshot_manager(self): 108 | from coopy.base import CoopyProxy 109 | import os 110 | os.mkdir('wiki') 111 | 112 | class PassPublisher(object): 113 | def close(self): 114 | pass 115 | def receive(self): 116 | pass 117 | 118 | proxy = CoopyProxy(Wiki(), [PassPublisher()]) 119 | proxy.start_snapshot_manager(0) 120 | self.assertTrue(hasattr(proxy, 'snapshot_timer')) 121 | 122 | proxy.shutdown() 123 | proxy.close() 124 | 125 | def test_coopyproxy_start_master(self): 126 | from coopy.base import CoopyProxy 127 | import os 128 | os.mkdir('wiki') 129 | 130 | class PassPublisher(object): 131 | def close(self): 132 | pass 133 | def receive(self): 134 | pass 135 | 136 | proxy = CoopyProxy(Wiki(), [PassPublisher()]) 137 | proxy.start_master() 138 | 139 | self.assertTrue(hasattr(proxy, 'server')) 140 | self.assertTrue(proxy.server in proxy.publisher.subscribers) 141 | proxy.shutdown() 142 | proxy.close() 143 | 144 | def test_coopyproxy_start_slave(self): 145 | from coopy.base import CoopyProxy 146 | 147 | import os 148 | os.mkdir('wiki') 149 | 150 | class PassPublisher(object): 151 | def close(self): 152 | pass 153 | def receive(self): 154 | pass 155 | 156 | proxy = CoopyProxy(Wiki(), [PassPublisher()]) 157 | 158 | args = ('localhost', 8012) 159 | 160 | self.assertRaises(Exception, 161 | proxy.start_slave, 162 | *args) 163 | proxy.close() 164 | 165 | def test_coopyproxy__getattr__(self): 166 | from coopy.base import CoopyProxy 167 | 168 | import os 169 | os.mkdir('wiki') 170 | 171 | class PassPublisher(object): 172 | def close(self): 173 | pass 174 | def receive(self): 175 | pass 176 | 177 | wiki = Wiki() 178 | 179 | wiki.__dict__['_private'] = "private" 180 | wiki.__dict__['some_callable'] = lambda x: x 181 | 182 | proxy = CoopyProxy(wiki, [PassPublisher()]) 183 | 184 | self.assertTrue(proxy._private == "private") 185 | self.assertTrue(callable(proxy.some_callable)) 186 | 187 | proxy.close() 188 | 189 | def test_coopyproxy_abort_exception(self): 190 | from coopy.base import CoopyProxy 191 | 192 | import os 193 | os.mkdir('wiki') 194 | 195 | class PassPublisher(object): 196 | def __init__(self): 197 | self.messages = [] 198 | def close(self): 199 | pass 200 | def receive(self, message): 201 | self.messages.append(message) 202 | def receive_before(self, message): 203 | self.messages.append(message) 204 | def receive_exception(self, message): 205 | self.messages.append(message) 206 | 207 | 208 | publisher = PassPublisher() 209 | proxy = CoopyProxy(Wiki(), [publisher]) 210 | 211 | with pytest.raises(Exception): 212 | proxy.check_abort_exception() 213 | 214 | self.assertEquals(1, len(publisher.messages)) 215 | 216 | proxy.close() 217 | 218 | def test_coopyproxy_unlocked(self): 219 | from coopy.base import CoopyProxy 220 | 221 | import os 222 | os.mkdir('wiki') 223 | 224 | class PassPublisher(object): 225 | def close(self): 226 | pass 227 | def receive(self, message): 228 | pass 229 | def receive_before(self, message): 230 | pass 231 | def receive_exception(self, message): 232 | pass 233 | 234 | proxy = CoopyProxy(Wiki(), [PassPublisher()]) 235 | proxy.create_page('id', 'content', None) 236 | 237 | # we're checking that system remains unlocked after a method execution 238 | # thus raising a RuntimeError on a release() 239 | with pytest.raises(RuntimeError): 240 | proxy.lock.release() 241 | 242 | 243 | # mock testing 244 | proxy.lock = mock.MagicMock() 245 | proxy.create_page('id', 'content', None) 246 | proxy.lock.acquire.assert_called_with(1) 247 | proxy.lock.release.assert_called() 248 | 249 | proxy.unlocked_method() 250 | proxy.lock.acquire.assert_not_called() 251 | proxy.close() 252 | 253 | def test_coopyproxy_take_snapshot(self): 254 | from coopy.base import CoopyProxy 255 | 256 | import os 257 | os.mkdir('wiki') 258 | 259 | class PassPublisher(object): 260 | def close(self): 261 | pass 262 | def receive(self, message): 263 | pass 264 | def receive_before(self, message): 265 | pass 266 | def receive_exception(self, message): 267 | pass 268 | 269 | proxy = CoopyProxy(Wiki(), [PassPublisher()]) 270 | 271 | # mock testing 272 | proxy.lock = mock.MagicMock() 273 | proxy.create_page('id', 'content', None) 274 | 275 | proxy.take_snapshot() 276 | 277 | proxy.lock.acquire.assert_called() 278 | proxy.lock.release.assert_called() 279 | proxy.close() 280 | 281 | if __name__ == "__main__": 282 | unittest.main() 283 | -------------------------------------------------------------------------------- /tests/unit/test_fileutils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | import shutil 4 | import six 5 | 6 | import coopy 7 | import coopy.fileutils as fu 8 | from ..domain import Wiki 9 | 10 | TEST_DIR = 'fileutils_test/' 11 | 12 | class TestFileUtils(unittest.TestCase): 13 | def setUp(self): 14 | try: 15 | os.mkdir(TEST_DIR) 16 | except: 17 | pass 18 | 19 | def tearDown(self): 20 | try: 21 | shutil.rmtree(TEST_DIR) 22 | except: 23 | pass 24 | 25 | def test_number_as_string(self): 26 | self.assertEqual(fu.number_as_string(0),'000000000000000') 27 | self.assertEqual(fu.number_as_string(1),'000000000000001') 28 | self.assertEqual(fu.number_as_string(10),'000000000000010') 29 | self.assertEqual(fu.number_as_string(11),'000000000000011') 30 | 31 | def test_list_snapshot_files(self): 32 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 33 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 34 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 35 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 36 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 37 | result = fu.list_snapshot_files(TEST_DIR) 38 | self.assertEqual(1, len(result)) 39 | 40 | def test_list_log_files(self): 41 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 42 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 43 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 44 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 45 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 46 | result = fu.list_log_files(TEST_DIR) 47 | self.assertEqual(4, len(result)) 48 | 49 | def test_last_snapshot_file(self): 50 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 51 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 52 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 53 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 54 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 55 | result = fu.last_snapshot_file(TEST_DIR) 56 | self.assertEqual('snapshot_000000000000004.dat',result) 57 | 58 | self.tearDown() 59 | self.setUp() 60 | 61 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 62 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 63 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 64 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 65 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 66 | open(TEST_DIR + 'snapshot_000000000000006.dat', 'w') 67 | result = fu.last_snapshot_file(TEST_DIR) 68 | self.assertEqual('snapshot_000000000000006.dat',result) 69 | 70 | 71 | def test_last_log_file(self): 72 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 73 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 74 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 75 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 76 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 77 | result = fu.last_log_file(TEST_DIR) 78 | self.assertEqual('transaction_000000000000005.log',result) 79 | 80 | self.tearDown() 81 | self.setUp() 82 | 83 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 84 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 85 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 86 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 87 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 88 | open(TEST_DIR + 'snapshot_000000000000006.dat', 'w') 89 | result = fu.last_log_file(TEST_DIR) 90 | self.assertEqual('transaction_000000000000005.log',result) 91 | 92 | def test_lastest_snapshot_number(self): 93 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 94 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 95 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 96 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 97 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 98 | result = fu.lastest_snapshot_number(TEST_DIR) 99 | self.assertEqual(4,result) 100 | 101 | self.tearDown() 102 | self.setUp() 103 | 104 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 105 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 106 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 107 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 108 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 109 | open(TEST_DIR + 'snapshot_000000000000006.dat', 'w') 110 | result = fu.lastest_snapshot_number(TEST_DIR) 111 | self.assertEqual(6,result) 112 | 113 | 114 | def test_lastest_log_number(self): 115 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 116 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 117 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 118 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 119 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 120 | result = fu.lastest_log_number(TEST_DIR) 121 | self.assertEqual(5,result) 122 | 123 | self.tearDown() 124 | self.setUp() 125 | 126 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 127 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 128 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 129 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 130 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 131 | open(TEST_DIR + 'snapshot_000000000000006.dat', 'w') 132 | result = fu.lastest_log_number(TEST_DIR) 133 | self.assertEqual(5,result) 134 | 135 | def test_to_log_file(self): 136 | result = fu.to_log_file(TEST_DIR, 1) 137 | self.assertEqual(TEST_DIR + 'transaction_000000000000001.log', result) 138 | 139 | def test_to_snapshot_file(self): 140 | result = fu.to_snapshot_file(TEST_DIR, 1) 141 | self.assertEqual(TEST_DIR + 'snapshot_000000000000001.dat' , result) 142 | 143 | def test_get_number(self): 144 | result = fu.get_number('transaction_000000000000001.log') 145 | self.assertEqual(1,result) 146 | result = fu.get_number('transaction_000000000000010.log') 147 | self.assertEqual(10,result) 148 | result = fu.get_number('snapshot_000000000000001.dat') 149 | self.assertEqual(1,result) 150 | result = fu.get_number('snapshot_000000000000010.dat') 151 | self.assertEqual(10,result) 152 | 153 | def test_next_number(self): 154 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 155 | result = fu.next_number(TEST_DIR) 156 | self.assertEqual(2, result) 157 | 158 | self.tearDown() 159 | self.setUp() 160 | 161 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 162 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 163 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 164 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 165 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 166 | result = fu.next_number(TEST_DIR) 167 | self.assertEqual(6, result) 168 | 169 | self.tearDown() 170 | self.setUp() 171 | 172 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 173 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 174 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 175 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 176 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 177 | open(TEST_DIR + 'snapshot_000000000000006.dat', 'w') 178 | result = fu.next_number(TEST_DIR) 179 | self.assertEqual(7, result) 180 | 181 | def test_next_snapshot_file(self): 182 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 183 | result = fu.next_snapshot_file(TEST_DIR) 184 | self.assertEqual(TEST_DIR + 'snapshot_000000000000002.dat', result) 185 | 186 | self.tearDown() 187 | self.setUp() 188 | 189 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 190 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 191 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 192 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 193 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 194 | result = fu.next_snapshot_file(TEST_DIR) 195 | self.assertEqual(TEST_DIR + 'snapshot_000000000000006.dat', result) 196 | 197 | self.tearDown() 198 | self.setUp() 199 | 200 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 201 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 202 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 203 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 204 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 205 | open(TEST_DIR + 'snapshot_000000000000006.dat', 'w') 206 | result = fu.next_snapshot_file(TEST_DIR) 207 | self.assertEqual(TEST_DIR + 'snapshot_000000000000007.dat', result) 208 | 209 | def test_next_log_file(self): 210 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 211 | result = fu.next_log_file(TEST_DIR) 212 | self.assertEqual(TEST_DIR + 'transaction_000000000000002.log', result) 213 | 214 | self.tearDown() 215 | self.setUp() 216 | 217 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 218 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 219 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 220 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 221 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 222 | result = fu.next_log_file(TEST_DIR) 223 | self.assertEqual(TEST_DIR + 'transaction_000000000000006.log', result) 224 | 225 | self.tearDown() 226 | self.setUp() 227 | 228 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 229 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 230 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 231 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 232 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 233 | open(TEST_DIR + 'snapshot_000000000000006.dat', 'w') 234 | result = fu.next_log_file(TEST_DIR) 235 | self.assertEqual(TEST_DIR + 'transaction_000000000000007.log', result) 236 | 237 | def test_last_log_files(self): 238 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 239 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 240 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 241 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 242 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 243 | result = fu.last_log_files(TEST_DIR) 244 | self.assertEqual(1, len(result)) 245 | self.assertEqual(TEST_DIR + 'transaction_000000000000005.log', result[0]) 246 | 247 | self.tearDown() 248 | self.setUp() 249 | 250 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 251 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 252 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 253 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 254 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 255 | open(TEST_DIR + 'snapshot_000000000000006.dat', 'w') 256 | result = fu.last_log_files(TEST_DIR) 257 | self.assertEqual(0, len(result)) 258 | 259 | self.tearDown() 260 | self.setUp() 261 | 262 | open(TEST_DIR + 'transaction_000000000000001.log', 'w') 263 | open(TEST_DIR + 'transaction_000000000000002.log', 'w') 264 | open(TEST_DIR + 'transaction_000000000000003.log', 'w') 265 | open(TEST_DIR + 'snapshot_000000000000004.dat', 'w') 266 | open(TEST_DIR + 'transaction_000000000000005.log', 'w') 267 | open(TEST_DIR + 'snapshot_000000000000006.dat', 'w') 268 | open(TEST_DIR + 'transaction_000000000000007.log', 'w') 269 | open(TEST_DIR + 'transaction_000000000000008.log', 'w') 270 | result = fu.last_log_files(TEST_DIR) 271 | self.assertEqual(2, len(result)) 272 | self.assertEqual(TEST_DIR + 'transaction_000000000000007.log', result[0]) 273 | self.assertEqual(TEST_DIR + 'transaction_000000000000008.log', result[1]) 274 | 275 | def test_snapshot_init(self): 276 | manager = fu.name_to_dir('') 277 | self.assertEqual(manager, './') 278 | 279 | manager = fu.name_to_dir('.') 280 | self.assertEqual(manager, './') 281 | 282 | manager = fu.name_to_dir('app') 283 | self.assertEqual(manager, 'app/') 284 | 285 | manager = fu.name_to_dir('app/') 286 | self.assertEqual(manager, 'app/') 287 | 288 | manager = fu.name_to_dir('app\\') 289 | self.assertEqual(manager, 'app/') 290 | 291 | manager = fu.name_to_dir('app\\app\\') 292 | self.assertEqual(manager, 'app/app/') 293 | 294 | manager = fu.name_to_dir('app\\app') 295 | self.assertEqual(manager, 'app/app/') 296 | 297 | manager = fu.name_to_dir('app/app') 298 | self.assertEqual(manager, 'app/app/') 299 | 300 | manager = fu.name_to_dir('app/app/') 301 | self.assertEqual(manager, 'app/app/') 302 | 303 | def test_obj_to_dir_name(self): 304 | result = fu.obj_to_dir_name(Wiki) 305 | self.assertEqual('wiki',result) 306 | result = fu.obj_to_dir_name(Wiki()) 307 | self.assertEqual('wiki',result) 308 | result = fu.obj_to_dir_name("string") 309 | self.assertEqual('str',result) 310 | 311 | class RotateFileWrapperTest(unittest.TestCase): 312 | def setUp(self): 313 | try: 314 | os.mkdir(TEST_DIR) 315 | except: 316 | pass 317 | 318 | def tearDown(self): 319 | try: 320 | shutil.rmtree(TEST_DIR) 321 | except: 322 | pass 323 | 324 | def test_rotate_file_write_close(self): 325 | _file = fu.RotateFileWrapper(open(TEST_DIR + 'file.log', 'wb'), 326 | TEST_DIR, 327 | os.getcwd()) 328 | data = b"a byte sequence!" 329 | _file.write(data) 330 | _file.close() 331 | 332 | self.assertTrue(_file.closed) 333 | 334 | raw_file = open(TEST_DIR + 'file.log', 'rb') 335 | self.assertEqual(raw_file.read(), data) 336 | 337 | def test_rotate_file_rotate(self): 338 | _file = fu.RotateFileWrapper(open(TEST_DIR + 'file.log', 'wb'), 339 | TEST_DIR, 340 | os.getcwd(), 341 | max_size=1024) 342 | class PicklerMock(object): 343 | def clear_memo(self): 344 | pass 345 | pickler_mock = PicklerMock() 346 | _file.pickler = pickler_mock 347 | 348 | self.assertEqual(len(os.listdir(TEST_DIR)), 1) 349 | 350 | if six.PY3: 351 | data = b"a" * 1025 352 | else: 353 | data = "a" * 1025 354 | _file.write(data) 355 | 356 | self.assertEqual(len(os.listdir(TEST_DIR)), 2) 357 | 358 | if six.PY3: 359 | second_data = b"b" * 1000 360 | else: 361 | second_data = "b" * 1000 362 | 363 | _file.write(second_data) 364 | files = os.listdir(TEST_DIR) 365 | self.assertEqual(len(files), 2) 366 | 367 | _file.close() 368 | 369 | self.assertTrue(_file.closed) 370 | raw_file_a = open(TEST_DIR + "file.log", 'rb').read() 371 | raw_file_b = open(TEST_DIR + "transaction_000000000000002.log", 'rb').read() 372 | self.assertEqual(raw_file_a, data) 373 | self.assertEqual(raw_file_b, second_data) 374 | 375 | def test_guarantee_cwd(self): 376 | _file = fu.RotateFileWrapper(open(TEST_DIR + 'file.log', 'wb'), 377 | TEST_DIR, 378 | os.getcwd()) 379 | 380 | current_dir = os.getcwd() 381 | os.chdir('/tmp') 382 | 383 | data = b"a byte sequence!" 384 | _file.write(data) 385 | _file.close() 386 | 387 | self.assertTrue(_file.closed) 388 | 389 | os.chdir(current_dir) 390 | raw_file = open(TEST_DIR + 'file.log', 'rb') 391 | self.assertEqual(raw_file.read(), data) 392 | -------------------------------------------------------------------------------- /tests/unit/test_foundation.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | class TestAction(unittest.TestCase): 4 | def test_execute_action(self): 5 | from coopy.foundation import Action, RecordClock 6 | from datetime import datetime 7 | from coopy.utils import inject 8 | 9 | class Dummy(object): 10 | def __init__(self): 11 | self.exec_count = 0 12 | 13 | def business_method_noargs(self): 14 | self.exec_count += 1 15 | 16 | def business_method_args(self, arg): 17 | self.exec_count += 2 18 | 19 | def business_method_kwargs(self, keyword_arg="test"): 20 | self.exec_count += 3 21 | 22 | 23 | dummy = Dummy() 24 | # force clock into dummy 25 | inject(dummy, '_clock', RecordClock()) 26 | 27 | action = Action('caller_id', 28 | 'business_method_noargs', 29 | datetime.now(), 30 | (), 31 | {}) 32 | 33 | action.execute_action(dummy) 34 | 35 | self.assertEquals(1, dummy.exec_count) 36 | 37 | action = Action('caller_id', 38 | 'business_method_args', 39 | datetime.now(), 40 | ([1]), 41 | {}) 42 | 43 | action.execute_action(dummy) 44 | 45 | self.assertEquals(3, dummy.exec_count) 46 | 47 | action = Action('caller_id', 48 | 'business_method_kwargs', 49 | datetime.now(), 50 | (), 51 | {'keyword_arg' : 'test'}) 52 | 53 | action.execute_action(dummy) 54 | 55 | self.assertEquals(6, dummy.exec_count) 56 | 57 | class TestRecordClock(unittest.TestCase): 58 | def test_record_clock(self): 59 | from coopy.foundation import RecordClock 60 | 61 | clock = RecordClock() 62 | self.assertTrue(len(clock.results) == 0) 63 | 64 | dt1 = clock.now() 65 | self.assertEquals(dt1, clock.results[0]) 66 | 67 | dt2 = clock.now() 68 | self.assertEquals(dt2, clock.results[1]) 69 | 70 | dt = clock.today() 71 | self.assertEquals(dt, clock.results[2]) 72 | 73 | utcnow = clock.utcnow() 74 | self.assertEquals(utcnow, clock.results[3]) 75 | 76 | class TestRestoreClock(unittest.TestCase): 77 | def test_restore_clock(self): 78 | from coopy.foundation import RestoreClock 79 | from datetime import datetime, date 80 | 81 | dt1 = datetime.now() 82 | dt2 = date.today() 83 | dt3 = datetime.utcnow() 84 | 85 | clock = RestoreClock([dt1, dt2, dt3]) 86 | 87 | self.assertEquals(dt1, clock.now()) 88 | self.assertEquals(dt2, clock.today()) 89 | self.assertEquals(dt3, clock.utcnow()) 90 | 91 | if __name__ == "__main__": 92 | unittest.main() 93 | -------------------------------------------------------------------------------- /tests/unit/test_journal.py: -------------------------------------------------------------------------------- 1 | import six 2 | import unittest 3 | 4 | import os 5 | import shutil 6 | 7 | from coopy.journal import DiskJournal 8 | 9 | JOURNAL_DIR = 'journal_test/' 10 | CURRENT_DIR = os.getcwd() 11 | 12 | class TestJournal(unittest.TestCase): 13 | def setUp(self): 14 | os.mkdir(JOURNAL_DIR) 15 | 16 | def tearDown(self): 17 | shutil.rmtree(JOURNAL_DIR) 18 | 19 | def test_current_journal_file(self): 20 | journal = DiskJournal(JOURNAL_DIR, CURRENT_DIR) 21 | expected_file_name = '%s%s' % (JOURNAL_DIR, 22 | 'transaction_000000000000002.log') 23 | 24 | self.assertEquals(expected_file_name, 25 | journal.current_journal_file(JOURNAL_DIR).name) 26 | 27 | # test hack! - create next file 28 | new_file_name = expected_file_name.replace('2','3') 29 | open(new_file_name, 'wt').close() 30 | 31 | self.assertEquals(new_file_name, 32 | journal.current_journal_file(JOURNAL_DIR).name) 33 | 34 | def test_receive(self): 35 | import pickle 36 | class Message(object): 37 | def __init__(self, value): 38 | self.value = value 39 | def __getstate__(self): 40 | raise pickle.PicklingError() 41 | 42 | message = Message('test message') 43 | journal = DiskJournal(JOURNAL_DIR, CURRENT_DIR) 44 | journal.setup() 45 | 46 | self.assertRaises( 47 | pickle.PicklingError, 48 | journal.receive, 49 | (message) 50 | ) 51 | 52 | def test_close(self): 53 | journal = DiskJournal(JOURNAL_DIR, CURRENT_DIR) 54 | self.assertTrue(not journal.file) 55 | 56 | journal.setup() 57 | self.assertTrue(not journal.file.closed) 58 | 59 | journal.close() 60 | self.assertTrue(journal.file.closed) 61 | 62 | def test_setup(self): 63 | journal = DiskJournal(JOURNAL_DIR, CURRENT_DIR) 64 | self.assertEquals(JOURNAL_DIR, journal.basedir) 65 | 66 | journal.setup() 67 | expected_file_name = '%s%s' % (JOURNAL_DIR, 68 | 'transaction_000000000000002.log') 69 | self.assertEquals(expected_file_name, 70 | journal.file.name) 71 | 72 | if six.PY3: 73 | import pickle 74 | else: 75 | import cPickle as pickle 76 | # test hack 77 | pickle_class = pickle.Pickler(open(expected_file_name, 'rb'))\ 78 | .__class__ 79 | self.assertTrue(isinstance(journal.pickler, pickle_class)) 80 | 81 | -------------------------------------------------------------------------------- /tests/unit/test_network.py: -------------------------------------------------------------------------------- 1 | from tests.domain import Wiki 2 | 3 | def test_prepare_data(): 4 | from coopy.network.network import prepare_data 5 | from coopy.foundation import RecordClock 6 | from coopy.utils import inject 7 | import six 8 | if six.PY3: 9 | import pickle 10 | else: 11 | import cPickle as pickle 12 | import zlib 13 | 14 | wiki = Wiki() 15 | inject(wiki, '_clock', RecordClock()) 16 | wiki.create_page('test', 'test content', None) 17 | 18 | (header, compressed_data) = prepare_data(wiki) 19 | 20 | copy_wiki = pickle.loads(zlib.decompress(compressed_data)) 21 | 22 | assert copy_wiki.get_page('test').id == 'test' 23 | assert copy_wiki.get_page('test').content == 'test content' 24 | 25 | def test_prepare_action(): 26 | from coopy.foundation import Action 27 | from coopy.network.network import prepare_data 28 | import six 29 | if six.PY3: 30 | import pickle 31 | else: 32 | import cPickle as pickle 33 | import zlib 34 | import datetime 35 | 36 | args = [] 37 | kwargs = {} 38 | 39 | action = Action('caller_id', 40 | 'test', 41 | datetime.datetime.now(), 42 | args, 43 | kwargs) 44 | 45 | (header, compressed_data) = prepare_data(action) 46 | 47 | copy_action = pickle.loads(zlib.decompress(compressed_data)) 48 | 49 | assert action.caller_id == copy_action.caller_id 50 | assert action.action == copy_action.action 51 | assert action.args == copy_action.args 52 | assert action.kwargs == copy_action.kwargs 53 | assert action.results == copy_action.results 54 | -------------------------------------------------------------------------------- /tests/unit/test_network_select.py: -------------------------------------------------------------------------------- 1 | import six 2 | import sys 3 | import pytest 4 | import socket 5 | 6 | from coopy.network.network import COPYNET_HEADER 7 | from coopy.network.default_select import CopyNet, CopyNetSlave, _HEADER_SIZE 8 | 9 | _str_to_bytes = lambda x: x.encode('utf-8') if type(x) != bytes else x 10 | 11 | if six.PY3 or 'PyPy' in sys.version: 12 | socket_select_error = ValueError 13 | else: 14 | socket_select_error = socket.error 15 | 16 | def tcp_actor(address, port, _type): 17 | if _type == "inet": 18 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 19 | s.connect((address, port)) 20 | else: 21 | s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 22 | s.connect(address) 23 | s.setblocking(0) 24 | return s 25 | 26 | def tcp_server(address, port, _type, max_clients=5): 27 | if _type == "inet": 28 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 29 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 30 | s.bind((address, port)) 31 | else: 32 | s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 33 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 34 | s.bind(address) 35 | s.listen(max_clients) 36 | s.setblocking(0) 37 | return s 38 | 39 | def test_network_select_init(): 40 | system = "a string represented system state" 41 | 42 | copynet = CopyNet(system) 43 | 44 | assert isinstance(copynet.server, socket.socket) 45 | assert len(copynet.clientmap) == 0 46 | assert copynet.queues == {} 47 | assert copynet.obj == system 48 | 49 | copynet.close() 50 | 51 | def test_network_select_init_close(): 52 | import select 53 | system = "a string represented system state" 54 | 55 | copynet = CopyNet(system) 56 | 57 | #no error because socket is open 58 | select.select([copynet.server], [], [], 0) 59 | 60 | copynet.close() 61 | 62 | assert copynet.running == False 63 | 64 | #must raise error 65 | with pytest.raises(socket_select_error): 66 | select.select([copynet.server], [], [], 0) 67 | 68 | #@pytest.skip 69 | def test_network_select_receive(): 70 | from coopy.base import logging_config 71 | 72 | logging_config(basedir="./") 73 | 74 | system = "a string represented system state" 75 | 76 | copynet = CopyNet(system, host="127.0.0.1", port=7777) 77 | copynet.start() 78 | 79 | actor = tcp_actor("127.0.0.1", 7777, "inet") 80 | actor.send(b'copynet') 81 | 82 | #guarantee that the client is already connected 83 | import time 84 | time.sleep(0.2) 85 | 86 | copynet.receive(b"message") 87 | 88 | #if no error, socket is open 89 | import select 90 | select.select([], [], [actor], 0) 91 | 92 | import struct, zlib 93 | if six.PY3: 94 | import pickle 95 | else: 96 | import cPickle as pickle 97 | 98 | size = struct.calcsize(COPYNET_HEADER) 99 | header = actor.recv(size) 100 | (psize, stype) = struct.unpack(COPYNET_HEADER, header) 101 | data = pickle.loads(zlib.decompress(actor.recv(psize))) 102 | 103 | assert stype == b's' 104 | assert data == b"message" 105 | 106 | copynet.close() 107 | actor.close() 108 | 109 | def test_network_select_disconnect_senders(): 110 | from coopy.base import logging_config 111 | 112 | logging_config(basedir="./") 113 | 114 | system = "a string represented system state" 115 | 116 | copynet = CopyNet(system, host="127.0.0.1", port=7777) 117 | copynet.start() 118 | 119 | actor = tcp_actor("127.0.0.1", 7777, "inet") 120 | actor.send(_str_to_bytes('copynet')) 121 | 122 | #guarantee that the client is already connected 123 | import time 124 | time.sleep(0.2) 125 | 126 | actor.send(_str_to_bytes('actor should be disconnected')) 127 | time.sleep(0.2) 128 | 129 | assert 0 == len(copynet.clientmap) 130 | 131 | copynet.close() 132 | actor.close() 133 | 134 | def test_network_select_broadcast(): 135 | received_msgs = [] 136 | from coopy.base import logging_config 137 | 138 | logging_config(basedir="./") 139 | 140 | system = "a string represented system state" 141 | 142 | copynet = CopyNet(system, host="127.0.0.1", port=7777) 143 | copynet.start() 144 | 145 | actor1 = tcp_actor("127.0.0.1", 7777, "inet") 146 | actor1.send(_str_to_bytes('copynet')) 147 | 148 | actor2 = tcp_actor("127.0.0.1", 7777, "inet") 149 | actor2.send(_str_to_bytes('copynet')) 150 | 151 | clients = [actor1, actor2] 152 | 153 | #guarantee that the client is already connected 154 | import time 155 | time.sleep(0.2) 156 | 157 | copynet.receive(b"message") 158 | if six.PY3: 159 | import pickle 160 | else: 161 | import cPickle as pickle 162 | 163 | #both clients should receive the same message 164 | for cli in clients: 165 | import struct, zlib, pickle 166 | size = struct.calcsize(COPYNET_HEADER) 167 | header = cli.recv(size) 168 | (psize, stype) = struct.unpack(COPYNET_HEADER, header) 169 | data = pickle.loads(zlib.decompress(cli.recv(psize))) 170 | received_msgs.append(data) 171 | 172 | assert stype == b's' 173 | assert data == b"message" 174 | 175 | assert len(received_msgs) == 2 176 | assert received_msgs == [b"message", b"message"] 177 | 178 | copynet.close() 179 | actor1.close() 180 | actor2.close() 181 | 182 | def test_network_select_send_direct(): 183 | received_msgs = [] 184 | from coopy.base import logging_config 185 | 186 | logging_config(basedir="./") 187 | 188 | system = "a string represented system state" 189 | 190 | copynet = CopyNet(system, host="127.0.0.1", port=7777) 191 | copynet.start() 192 | 193 | actor1 = tcp_actor("127.0.0.1", 7777, "inet") 194 | actor1.send(_str_to_bytes('copynet')) 195 | 196 | actor2 = tcp_actor("127.0.0.1", 7777, "inet") 197 | actor2.send(_str_to_bytes('copynet')) 198 | 199 | actors = [actor1, actor2] 200 | 201 | #guarantee that the client is already connected 202 | import time 203 | time.sleep(0.2) 204 | 205 | copynet_client1 = list(copynet.clientmap.values())[0] 206 | copynet.send_direct(copynet_client1.client, _str_to_bytes("message")) 207 | 208 | time.sleep(0.2) 209 | 210 | if six.PY3: 211 | import pickle 212 | else: 213 | import cPickle as pickle 214 | import struct, zlib, pickle 215 | size = struct.calcsize(COPYNET_HEADER) 216 | 217 | #one of the 2 reads will raise an error and the other will work 218 | error_count = 0 219 | 220 | for actor in actors: 221 | try: 222 | header = actor.recv(size) 223 | (psize, stype) = struct.unpack(COPYNET_HEADER, header) 224 | data = pickle.loads(zlib.decompress(actor.recv(psize))) 225 | received_msgs.append(data) 226 | except Exception: 227 | error_count += 1 228 | 229 | assert len(received_msgs) == 1 230 | assert error_count == 1 231 | 232 | copynet.close() 233 | actor1.close() 234 | actor2.close() 235 | 236 | def test_network_select_check_if_authorized_client(): 237 | from coopy.base import logging_config 238 | 239 | logging_config(basedir="./") 240 | 241 | system = "a string represented system state" 242 | 243 | copynet = CopyNet(system, host="127.0.0.1", port=7777) 244 | copynet.start() 245 | 246 | actor1 = tcp_actor("127.0.0.1", 7777, "inet") 247 | actor1.send(_str_to_bytes('copynet')) 248 | 249 | #guarantee that the client is already connected 250 | import time 251 | time.sleep(0.2) 252 | 253 | copynet_client1 = list(copynet.clientmap.values())[0] 254 | actor1.send(_str_to_bytes('copynet')) 255 | assert True == copynet.check_if_authorized_client(copynet_client1.client) 256 | 257 | actor1.close() 258 | 259 | actor2 = tcp_actor("127.0.0.1", 7777, "inet") 260 | actor2.send(_str_to_bytes('copynet')) 261 | 262 | time.sleep(0.2) 263 | 264 | copynet_client2 = list(copynet.clientmap.values())[0] 265 | actor2.send(_str_to_bytes('_copynet')) 266 | assert False == copynet.check_if_authorized_client(copynet_client2.client) 267 | 268 | actor2.close() 269 | copynet.close() 270 | 271 | def test_copynetslave_init(): 272 | class mock(object): 273 | def __init__(self): 274 | self.value = 0 275 | def inc(self): 276 | self.value += 1 277 | return self.value 278 | 279 | server = tcp_server('127.0.0.1', 5466, "inet") 280 | 281 | system = mock() 282 | slave = CopyNetSlave(system) 283 | 284 | slave.close() 285 | assert slave.running == False 286 | 287 | server.close() 288 | 289 | def test_copynetslave_disconnect_on_empty_data(): 290 | class mock(object): 291 | def __init__(self): 292 | self.value = 0 293 | def inc(self): 294 | self.value += 1 295 | return self.value 296 | 297 | from coopy.base import logging_config 298 | 299 | logging_config() 300 | 301 | server = tcp_server('127.0.0.1', 5466, "inet") 302 | 303 | system = mock() 304 | slave = CopyNetSlave(system, host='127.0.0.1') 305 | slave.start() 306 | 307 | import time 308 | time.sleep(0.2) 309 | 310 | cli, address = server.accept() 311 | 312 | time.sleep(0.2) 313 | cli.sendall(_str_to_bytes('1')) 314 | 315 | time.sleep(0.2) 316 | assert slave.running == False 317 | 318 | import select 319 | with pytest.raises(socket_select_error): 320 | select.select([slave.sock], [], [], 0) 321 | 322 | server.close() 323 | -------------------------------------------------------------------------------- /tests/unit/test_snapshot.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | import shutil 4 | from coopy.snapshot import * 5 | 6 | TEST_DIR = 'snapshot_test' 7 | 8 | class TestSnapshot(unittest.TestCase): 9 | 10 | def setUp(self): 11 | os.mkdir(TEST_DIR) 12 | 13 | def tearDown(self): 14 | shutil.rmtree(TEST_DIR) 15 | 16 | def test_take_snapshot(self): 17 | mock = 'test object' 18 | manager = SnapshotManager(TEST_DIR) 19 | manager.take_snapshot(mock) 20 | self.assertEqual(len(os.listdir(TEST_DIR)), 1) 21 | self.assertTrue(os.listdir(TEST_DIR)[0].endswith('dat')) 22 | 23 | def test_recover_snapshot(self): 24 | mock = 'test object' 25 | manager = SnapshotManager(TEST_DIR) 26 | manager.take_snapshot(mock) 27 | result = manager.recover_snapshot() 28 | self.assertEqual(result, mock) 29 | 30 | 31 | if __name__ == '__main__': 32 | unittest.main() 33 | -------------------------------------------------------------------------------- /tests/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from coopy import utils 3 | 4 | class UtilsTest(unittest.TestCase): 5 | 6 | def test_attribute_access(self): 7 | class Dummy(object): 8 | def __init__(self, some_property): 9 | self.some_property = some_property 10 | 11 | @property 12 | def check_property(self): 13 | return self.some_property 14 | 15 | def business_method(self, args): 16 | return 1 17 | 18 | dummy = Dummy('property') 19 | 20 | self.assertTrue(not 21 | utils.method_or_none(dummy,'some_property')) 22 | 23 | self.assertTrue(not 24 | utils.method_or_none(dummy,'check_property')) 25 | 26 | self.assertTrue(utils.method_or_none(dummy,'business_method')) 27 | 28 | def test_action_check(self): 29 | from coopy.decorators import readonly, unlocked, abort_exception 30 | 31 | def no_decorator(): 32 | pass 33 | 34 | @readonly 35 | def read_method(): 36 | pass 37 | 38 | @unlocked 39 | def not_locked(): 40 | pass 41 | 42 | @abort_exception 43 | def abort_on_exception(): 44 | pass 45 | 46 | @readonly 47 | @unlocked 48 | def readonly_unlocked(): 49 | pass 50 | 51 | self.assertEquals( 52 | (False, False, False), 53 | utils.action_check(no_decorator) 54 | ) 55 | 56 | self.assertEquals( 57 | (True, False, False), 58 | utils.action_check(read_method) 59 | ) 60 | 61 | self.assertEquals( 62 | (False, True, False), 63 | utils.action_check(not_locked) 64 | ) 65 | 66 | self.assertEquals( 67 | (False, False, True), 68 | utils.action_check(abort_on_exception) 69 | ) 70 | 71 | self.assertEquals( 72 | (True, True, False), 73 | utils.action_check(readonly_unlocked) 74 | ) 75 | -------------------------------------------------------------------------------- /tests/unit/test_validation.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | 4 | from coopy.validation import (validate_date_datetime_calls, validate_system, 5 | unindent_source, FORBIDDEN_OBJECTS, 6 | FORBIDDEN_FUNCS) 7 | from coopy.error import PrevalentError 8 | 9 | @pytest.mark.skipif("sys.version_info <= (2,7)") 10 | def test_check_forbidden_calls(): 11 | import itertools 12 | 13 | bad_code_templ = ''' 14 | def func(): 15 | a = [] 16 | b = %s.%s() 17 | ''' 18 | good_code_templ = ''' 19 | def func(): 20 | a = [] 21 | b = clock.%s() 22 | ''' 23 | 24 | for pair in itertools.product(FORBIDDEN_OBJECTS, FORBIDDEN_FUNCS): 25 | bad_code = bad_code_templ % (pair[0], pair[1]) 26 | code = good_code_templ % (pair[1]) 27 | 28 | assert not validate_date_datetime_calls(bad_code) 29 | assert validate_date_datetime_calls(code) 30 | 31 | def test_unindent_source(): 32 | 33 | code_templ = \ 34 | '''def func(self): 35 | self.func''' 36 | 37 | code_lines = [ 38 | ' def func(self):', 39 | ' self.func' 40 | ] 41 | 42 | assert unindent_source(code_lines) == code_templ 43 | 44 | def test_validate_system(): 45 | from datetime import datetime 46 | 47 | class BadSystem(object): 48 | def __init__(self): 49 | self.data = [] 50 | 51 | def bad_method(self): 52 | now = datetime.now() 53 | self.data.append(now) 54 | 55 | with pytest.raises(PrevalentError) as error: 56 | validate_system(BadSystem()) 57 | 58 | def test_validate_system_method_with_subscript(): 59 | class GoodSystem(object): 60 | def __init__(self): 61 | self.data = [] 62 | 63 | def ok_method(self): 64 | temp = [] 65 | temp_el = temp[0] 66 | element = self.data[0] 67 | 68 | def ok2_method(self): 69 | os.urandom(16).encode('hex') 70 | return ''.join([]) 71 | 72 | assert validate_system(GoodSystem()) 73 | --------------------------------------------------------------------------------