├── __init__.py ├── slothtest ├── sloth_log.py ├── sloth_config.py ├── __init__.py ├── sloth_connector.py ├── sloth_watcher.py └── sloth_xml_converter.py ├── setup.py ├── LICENSE ├── slothtest_tests └── test.py └── README.md /__init__.py: -------------------------------------------------------------------------------- 1 | name = "slothtest" 2 | -------------------------------------------------------------------------------- /slothtest/sloth_log.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | import os 4 | 5 | 6 | def setup_sloth_logger(name=""): 7 | 8 | formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s', 9 | datefmt='%Y-%m-%d %H:%M:%S') 10 | handler = logging.FileHandler('sloth_log.txt', mode='w') 11 | 12 | handler.setFormatter(formatter) 13 | screen_handler = logging.StreamHandler(stream=sys.stdout) 14 | screen_handler.setFormatter(formatter) 15 | 16 | logger = logging.getLogger(name) 17 | logger.setLevel(logging.INFO) 18 | logger.addHandler(handler) 19 | logger.addHandler(screen_handler) 20 | 21 | return logger 22 | 23 | 24 | sloth_log = setup_sloth_logger(os.environ.get('SLOTH_INSTANCE', "")) 25 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="slothtest", 8 | version="0.0.4", 9 | author="Paul Kovtun", 10 | author_email="trademet@gmail.com", 11 | description="Sloth Test: An Automatic Unit Test Generator", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/elegantwist/slothtest", 15 | packages=setuptools.find_packages(), 16 | classifiers=[ 17 | "Programming Language :: Python :: 3", 18 | "License :: OSI Approved :: MIT License", 19 | "Operating System :: OS Independent", 20 | ], 21 | install_requires=[ 22 | "joblib", 23 | ], 24 | ) -------------------------------------------------------------------------------- /slothtest/sloth_config.py: -------------------------------------------------------------------------------- 1 | class SlothConfig: 2 | 3 | # an iteration amount after which the dump will happen in watchme decorator 4 | DUMP_ITER_COUNT = 100 5 | 6 | # a dictionary that defines the equality operator between two values of the particular type 7 | # used in pytest creation 8 | objects_eq = { 9 | "": "equals", 10 | "": "equals" 11 | } 12 | 13 | class SlothState: 14 | IDLE = "0" 15 | WATCHING = "1" 16 | TESTING = "2" 17 | 18 | class SlothValueState: 19 | RESULT = "0" 20 | INCOME = "1" 21 | TEST = "2" 22 | 23 | class SlothResultState: 24 | NoErrors = "0" 25 | Warning = "1" 26 | Errors = "2" 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Paul Kovtun 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /slothtest/__init__.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | import os 3 | import asyncio 4 | import datetime 5 | from copy import deepcopy 6 | from .sloth_log import sloth_log 7 | from .sloth_connector import SlothConnector 8 | from .sloth_config import SlothConfig 9 | from .sloth_watcher import SlothWatcher 10 | from functools import wraps 11 | 12 | # the name of the particular instance 13 | os.environ['SLOTH_INSTANCE'] = 'SLOTH_WATCHER' 14 | 15 | # an instance of sloth watcher starts with the initiation of the package 16 | slothwatcher = SlothWatcher() 17 | 18 | 19 | def watchme(): 20 | """ 21 | The main decorator for the method you need to watch at 22 | 1. copying the income arguments 23 | 2. intercepting the outcome result of the method 24 | 3. regularly dumping the in-and-out of the method to dump-file 25 | 26 | """ 27 | 28 | def subst_function(fn): 29 | 30 | @wraps(fn) 31 | def save_vars(*args, **kwargs): 32 | 33 | if slothwatcher.sloth_state != SlothConfig.SlothState.WATCHING: 34 | return fn(*args, **kwargs) 35 | 36 | if slothwatcher.dump_counter >= SlothConfig.DUMP_ITER_COUNT: 37 | slothwatcher.dump() 38 | 39 | in_args = deepcopy(args) 40 | in_kwargs = deepcopy(kwargs) 41 | 42 | start_time = datetime.datetime.now() 43 | 44 | try: 45 | res = fn(*args, **kwargs) 46 | additional_info = "" 47 | except Exception as e: 48 | res = e 49 | additional_info = traceback.format_exc() 50 | 51 | stop_time = datetime.datetime.now() 52 | 53 | asyncio.run(slothwatcher.watch(fn, in_args, in_kwargs, res, additional_info, start_time, stop_time)) 54 | 55 | return res 56 | 57 | return save_vars 58 | 59 | return subst_function 60 | -------------------------------------------------------------------------------- /slothtest_tests/test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | import pandas as pd 4 | from slothtest import watchme 5 | from slothtest import slothwatcher 6 | 7 | 8 | class ClassForTesting: 9 | debugging = 99 10 | 11 | def __init__(self, dd=0): 12 | self.debugging = dd 13 | 14 | @watchme() 15 | def im_a_function_for_testing(self, d_table=None, vv=1): 16 | for i, row in d_table.iterrows(): 17 | d_table['value'][i] = row['value'] * 10 18 | 19 | return d_table, vv 20 | 21 | 22 | @watchme() 23 | def im_another_function_for_testing(d_table=None, vv=1): 24 | for i, row in d_table.iterrows(): 25 | d_table['value'][i] = row['value'] * 10 26 | 27 | return d_table, vv 28 | 29 | 30 | def test_classmethod(): 31 | dirname = os.path.dirname(__file__) 32 | 33 | d_data = [{'column': 1, 'value': 1}, 34 | {'column': 2, 'value': 2}, 35 | {'column': 3, 'value': 4}] 36 | 37 | d_table = pd.DataFrame(d_data) 38 | 39 | slothwatcher.start() 40 | 41 | fn = ClassForTesting(12).im_a_function_for_testing(d_table, 2) 42 | 43 | assert slothwatcher.dump_counter == 1 44 | 45 | assert len(slothwatcher.data_watch_dump) == 1 46 | assert len(slothwatcher.data_watch_dump[0]['function']) > 0 47 | assert len(slothwatcher.data_watch_dump[0]['arguments']) == 3 48 | assert len(slothwatcher.data_watch_dump[0]['results']) == 2 49 | 50 | assert slothwatcher.data_watch_dump[0]['function']['class_name'] == 'ClassForTesting' 51 | assert slothwatcher.data_watch_dump[0]['function']['function_name'] == 'im_a_function_for_testing' 52 | assert slothwatcher.data_watch_dump[0]['function']['scope_name'] == 'test' 53 | 54 | slothwatcher.stop() 55 | 56 | assert os.path.isfile(os.path.join(dirname, slothwatcher.session_id + '.zip')) 57 | 58 | 59 | def test_function(): 60 | dirname = os.path.dirname(__file__) 61 | 62 | d_data = [{'column': 1, 'value': 1}, 63 | {'column': 2, 'value': 2}, 64 | {'column': 3, 'value': 4}] 65 | 66 | d_table = pd.DataFrame(d_data) 67 | 68 | slothwatcher.start() 69 | 70 | fn = im_another_function_for_testing(d_table, 2) 71 | 72 | assert slothwatcher.dump_counter == 1 73 | 74 | assert len(slothwatcher.data_watch_dump) == 1 75 | assert len(slothwatcher.data_watch_dump[0]['function']) > 0 76 | assert len(slothwatcher.data_watch_dump[0]['arguments']) == 2 77 | assert len(slothwatcher.data_watch_dump[0]['results']) == 2 78 | 79 | assert slothwatcher.data_watch_dump[0]['function']['class_name'] == '' 80 | assert slothwatcher.data_watch_dump[0]['function']['function_name'] == 'im_another_function_for_testing' 81 | assert slothwatcher.data_watch_dump[0]['function']['scope_name'] == 'test' 82 | 83 | slothwatcher.stop() 84 | 85 | assert os.path.isfile(os.path.join(dirname, slothwatcher.session_id + '.zip')) 86 | -------------------------------------------------------------------------------- /slothtest/sloth_connector.py: -------------------------------------------------------------------------------- 1 | import os 2 | import xml.etree.ElementTree as xml 3 | import zipfile 4 | from typing import List, Dict 5 | from . import sloth_log 6 | 7 | 8 | class SlothConnector: 9 | sloth_service = None 10 | instance_id = "" 11 | snapshot_id = "" 12 | session_id = "" 13 | scope_path = "" 14 | engage_counter = 0 15 | xml_filename = "" 16 | xml_data = None 17 | to_dir = None 18 | 19 | def __init__(self, session_id: str = "", snapshot_id: str = "", to_dir: str = None): 20 | 21 | self.to_dir = to_dir 22 | 23 | self.session_id = session_id 24 | if snapshot_id != "": 25 | self.snapshot_id = snapshot_id 26 | 27 | self.init_xml() 28 | 29 | def init_xml(self): 30 | 31 | if self.to_dir is None: 32 | self.to_dir = os.getcwd() 33 | 34 | self.xml_filename = os.path.join(self.to_dir, self.snapshot_id + ".xml") 35 | 36 | self.xml_data = xml.Element("SlothWatch") 37 | 38 | instance_name = xml.SubElement(self.xml_data, "instance_name") 39 | instance_name.text = self.instance_id 40 | 41 | snapshot_name = xml.SubElement(self.xml_data, "snapshot_name") 42 | snapshot_name.text = self.snapshot_id 43 | 44 | session_name = xml.SubElement(self.xml_data, "session_id") 45 | session_name.text = self.session_id 46 | 47 | def dump_data(self, data_watch_dump: List = None) -> str: 48 | 49 | if data_watch_dump is None: 50 | sloth_log.error("couldn't dump the data. watch data was not provided!") 51 | return "" 52 | 53 | functions_xml = xml.Element("functions_list") 54 | 55 | n = 0 56 | for func_watch in data_watch_dump: 57 | n += 1 58 | self.dump_function(functions_xml, func_watch['function'], func_watch['arguments'], func_watch['results'], n) 59 | 60 | self.xml_data.append(functions_xml) 61 | 62 | tree = xml.ElementTree(self.xml_data) 63 | 64 | with open(self.xml_filename, "wb") as fh: 65 | tree.write(fh) 66 | 67 | zip_fn = self.xml_filename[:-4] + '.zip' 68 | with zipfile.ZipFile(zip_fn, 'w', compression=zipfile.ZIP_DEFLATED) as myzip: 69 | myzip.write(self.xml_filename, arcname=os.path.basename(self.xml_filename)) 70 | 71 | sloth_log.info('zip pack created: ' + zip_fn) 72 | 73 | os.remove(self.xml_filename) 74 | 75 | return zip_fn 76 | 77 | def dump_function(self, functions_element=None, 78 | function_dict: Dict = None, args_dicts: List = None, res_dicts: List = None, n: int = 0): 79 | 80 | function_element = xml.SubElement(functions_element, "function") 81 | 82 | run_id = xml.SubElement(function_element, "run_id") 83 | run_id.text = str(n) 84 | 85 | scope_name = xml.SubElement(function_element, "scope_name") 86 | scope_name.text = function_dict['scope_name'] 87 | 88 | classname = xml.SubElement(function_element, "class_name") 89 | classname.text = function_dict['class_name'] 90 | 91 | classdump = xml.SubElement(function_element, "class_dump") 92 | classdump.text = function_dict['class_dump'] 93 | 94 | function_name = xml.SubElement(function_element, "function_name") 95 | function_name.text = function_dict['function_name'] 96 | 97 | run_time = xml.SubElement(function_element, "run_time") 98 | run_time.text = function_dict['run_time'] 99 | 100 | call_stack = xml.SubElement(function_element, "call_stack") 101 | call_stack.text = function_dict['call_stack'] 102 | 103 | args_xml = xml.SubElement(function_element, "arguments_list") 104 | for arg_dict in args_dicts: 105 | arg_xml = xml.SubElement(args_xml, "argument") 106 | 107 | par_type = xml.SubElement(arg_xml, "par_type") 108 | par_type.text = arg_dict['par_type'] 109 | 110 | par_name = xml.SubElement(arg_xml, "par_name") 111 | par_name.text = arg_dict['par_name'] 112 | 113 | par_value = xml.SubElement(arg_xml, "par_value") 114 | par_value.text = arg_dict['par_value'] 115 | 116 | par_state = xml.SubElement(arg_xml, "par_state") 117 | par_state.text = arg_dict['par_state'] 118 | 119 | par_simple = xml.SubElement(arg_xml, "par_simple") 120 | par_simple.text = arg_dict['par_simple'] 121 | 122 | additional_info = xml.SubElement(arg_xml, "additional_info") 123 | additional_info.text = arg_dict['additional_info'] 124 | 125 | reslts_xml = xml.SubElement(function_element, "results_list") 126 | for res_dict in res_dicts: 127 | reslt_xml = xml.SubElement(reslts_xml, "result") 128 | 129 | par_type = xml.SubElement(reslt_xml, "par_type") 130 | par_type.text = res_dict['par_type'] 131 | 132 | par_name = xml.SubElement(reslt_xml, "par_name") 133 | par_name.text = res_dict['par_name'] 134 | 135 | par_value = xml.SubElement(reslt_xml, "par_value") 136 | par_value.text = res_dict['par_value'] 137 | 138 | par_state = xml.SubElement(reslt_xml, "par_state") 139 | par_state.text = res_dict['par_state'] 140 | 141 | par_simple = xml.SubElement(reslt_xml, "par_simple") 142 | par_simple.text = res_dict['par_simple'] 143 | 144 | additional_info = xml.SubElement(reslt_xml, "additional_info") 145 | additional_info.text = res_dict['additional_info'] 146 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/elegantwist/slothtest.svg?branch=master)](https://travis-ci.org/elegantwist/slothtest) [![PyPI version](https://badge.fury.io/py/slothtest.svg)](https://badge.fury.io/py/slothtest) 2 | 3 | # Description 4 | 5 | Sloth Test is a Python library that automatically creates unit tests based on previous real-life cases to prevent regression bugs. 6 | 1. You will connect the Sloth Test library to your project and run the project for execution of the typical routine. 7 | 2. The Sloth collect the internal states of the classes, methods and functions you use in your project and you pointed the Sloth to watch at. It will record all possible incomes and outcomes of each method for each run 8 | 3. After it collects enough data, the library dumps the collected data to a file 9 | 4. For each recorded run in this file, Sloth Test will automatically create a particular unit test, with the particular state of the class, the particular recorded serialized incomes and an assertion of outcomes for this method. 10 | The result is a collection of typical pytest unit tests that can be executed as a part of the testing routine. 11 | 5. For each modification of this method you can run these created test cases to check if the method doesn’t get new bugs and implements the business logic is supposed to have. 12 | ------------------------------------------------------------------ 13 | 14 | # Installing 15 | 16 | You can use pip to install slothtest: 17 | 18 | ```pip install slothtest``` 19 | 20 | from any directory 21 | 22 | # Usage 23 | 24 | 25 | Suppose that we have a critical and sophisticated method that is a part of our ETL process (pd_table is a pandas table) : 26 | 27 | ```python 28 | def do_useful_stuff(pd_table=None, a=0, b=0): 29 | 30 | for i, row in pd_table.iterrows(): 31 | pd_table['value'][i] = row['value'] * a + b 32 | 33 | return pd_table 34 | ``` 35 | 36 | Let’s show some run examples that we will implement via another method as the part of our ETL process: 37 | 38 | ```python 39 | def run(): 40 | 41 | tables = { 42 | 'table1': pd.DataFrame([{'column': 1, 'value': 1}, 43 | {'column': 2, 'value': 2}, 44 | {'column': 3, 'value': 4}]), 45 | 46 | 'table2': pd.DataFrame([{'column': 1, 'value': 1}, 47 | {'column': 2, 'value': 2}, 48 | {'column': 3, 'value': 4}]), 49 | 50 | 'table3': pd.DataFrame([{'value': 1}, 51 | {'value': 2}, 52 | {'value': 4}]), 53 | 54 | 'table4': pd.DataFrame([{'value': 1000}, 55 | {'value': 10}]), 56 | } 57 | 58 | for t_name, pd_table in tables.items(): 59 | print("Table {name}: \n {table} \n". 60 | format(name=t_name, table=str(do_useful_stuff(pd_table=pd_table, a=2, b=3)))) 61 | 62 | if __name__ == '__main__': 63 | run() 64 | ``` 65 | 66 | the results are: 67 | 68 | ``` 69 | Table table1: 70 | column value 71 | 0 1 5 72 | 1 2 7 73 | 2 3 11 74 | 75 | Table table2: 76 | column value 77 | 0 1 5 78 | 1 2 7 79 | 2 3 11 80 | 81 | Table table3: 82 | value 83 | 0 5 84 | 1 7 85 | 2 11 86 | 87 | Table table4: 88 | value 89 | 0 2003 90 | 1 23 91 | ``` 92 | 93 | Ok. Next, we need to be sure that this method will implement the business logic is supposed to implement. To do that, we need to write manually a bunch of pytests for this method for various incomes and outcomes (perhaps 100+ tests for different variants of tables). Or use a Sloth Test library to do it for us automatically. 94 | 95 | 1. The first step - we need to import a @watchme() decorator from a slothtest library. This decorator should be used on the target method need the Sloth to watch. Let’s add it to our function: 96 | 97 | ```python 98 | from slothtest import watchme 99 | 100 | @watchme() 101 | def do_useful_stuff(pd_table=None, a=0, b=0): 102 | 103 | for i, row in pd_table.iterrows(): 104 | pd_table['value'][i] = row['value'] * a + b 105 | 106 | ``` 107 | 108 | 2. We need to point a sloth watcher where it should start its watching process and where it should stop to watch. It can be an entry and exits points of an application, or logic start and stop track inside our app. For our tiny app it’s a run method, so our code will look like: 109 | 110 | ```python 111 | if __name__ == '__main__': 112 | slothwatcher.start() 113 | run() 114 | slothwatcher.stop() 115 | 116 | ``` 117 | 118 | .. and that’s all! 119 | 120 | 3. Now, let’s run our app as usual, and let the Sloth watch our process run. After a run, in a folder with our example, a new zip-file appears with a filename in digits (it’s a timestamp) and a dump of our runs inside this zip file 121 | The zip-dump creates after a sloth is stopped, or it recorded a certain amount of runs for all the methods it watched. A number of runs we can set via SlothConfig class 122 | 123 | ```python 124 | from slothtest import SlothConfig 125 | SlothConfig.DUMP_ITER_COUNT = 200 126 | 127 | ``` 128 | 129 | 4. At this point, we have a dump file. Now, for further development purpose we need to get a typical pytest unit tests. We can create that from our dump file, using a sloth translator: 130 | 131 | ```python -m slothtest.sloth_xml_converter -p o:\work\slothexample -d o:\work\slothexample 1549134821.zip``` 132 | 133 | where -p is the key to a directory where we will put a path to our project, and -d is the key to a directory where the result pytest files will be created 134 | 135 | 5. The result of the conversion are two files: 136 | 1) test_sloth_1549134821.py and 2) sloth_test_parval_1549134821.py 137 | The first one is a basic pytest collection for each run of our watched function: 138 | 139 | 140 | ```python 141 | import sloth_test_parval_1549134821 as sl 142 | 143 | def test_do_useful_stuff_1(): 144 | from themethod import do_useful_stuff 145 | 146 | try: 147 | run_result = do_useful_stuff(pd_table=sl.val_do_useful_stuff_1_pd_table, a=sl.val_do_useful_stuff_1_a, b=sl.val_do_useful_stuff_1_b, ) 148 | except Exception as e: 149 | run_result = e 150 | 151 | test_result = sl.res_do_useful_stuff_1_ret_0 152 | assert(type(run_result) == type(test_result)) 153 | assert(run_result.equals(test_result)) 154 | 155 | 156 | def test_do_useful_stuff_2(): 157 | from themethod import do_useful_stuff 158 | 159 | try: 160 | run_result = do_useful_stuff(pd_table=sl.val_do_useful_stuff_2_pd_table, a=sl.val_do_useful_stuff_2_a, b=sl.val_do_useful_stuff_2_b, ) 161 | except Exception as e: 162 | run_result = e 163 | 164 | test_result = sl.res_do_useful_stuff_2_ret_0 165 | assert(type(run_result) == type(test_result)) 166 | assert(run_result.equals(test_result)) 167 | … 168 | 169 | 170 | ``` 171 | 172 | And the second one is the serialized (or raw values if they are a primitive type) income and outcome values for each run of the method (4 cases): 173 | 174 | ```python 175 | import codecs 176 | import io 177 | import joblib 178 | 179 | 180 | # ===== 1: do_useful_stuff@themethod 181 | 182 | var_stream = io.BytesIO() 183 | var_stream_str = codecs.decode('gANdWIu…'.encode(),"base64") 184 | 185 | var_stream.write(var_stream_str) 186 | var_stream.seek(0) 187 | val_do_useful_stuff_1_pd_table = joblib.load(var_stream) 188 | 189 | val_do_useful_stuff_1_a = 2 190 | 191 | val_do_useful_stuff_1_b = 3 192 | 193 | res_stream = io.BytesIO() 194 | res_stream_str = codecs.decode('gANdWIu…\n'.encode(),"base64") 195 | res_stream.write(res_stream_str) 196 | res_stream.seek(0) 197 | res_do_useful_stuff_1_ret_0 = joblib.load(res_stream) 198 | … 199 | 200 | ``` 201 | 202 | 6. Now we can run our testing routine with pytest as usual: 203 | 204 | 205 | ``` 206 | python -m pytest test_sloth_1549134821.py 207 | 208 | ================================================= test session starts ================================================= 209 | platform win32 -- Python 3.7.0, pytest-4.1.1, py-1.7.0, pluggy-0.8.1 210 | rootdir: o:\work\slothexample, inifile: 211 | plugins: remotedata-0.3.1, openfiles-0.3.2, doctestplus-0.2.0, arraydiff-0.3 212 | collected 4 items 213 | 214 | test_sloth_1549134821.py .... [100%] 215 | 216 | ======================================== 4 passed, 2 warnings in 0.34 seconds ========================================= 217 | 218 | ``` 219 | 220 | And that’s all. Easy! 221 | 222 | This approach to generating unit tests automatically can be extrapolated for as many cases as you need if your methods and classes are serializable and if you have enough space for data dumps 223 | -------------------------------------------------------------------------------- /slothtest/sloth_watcher.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datetime 3 | import inspect 4 | import pickle 5 | import codecs 6 | import joblib 7 | import io 8 | import asyncio 9 | from typing import List, Dict 10 | from . import SlothConnector, sloth_log 11 | from . import SlothConfig 12 | 13 | 14 | class SlothWatcher: 15 | sloth_state = None 16 | 17 | instance_id = "" 18 | snapshot_id = "" 19 | session_id = "" 20 | 21 | service_online = False 22 | sloth_connector = None 23 | 24 | data_watch_dump = [] 25 | 26 | dump_counter = 0 27 | 28 | to_dir = None 29 | 30 | def __init__(self): 31 | 32 | self.instance_id = str(os.environ.get('SLOTH_INSTANCE_ID', "")) 33 | 34 | def start(self, to_dir: str = None): 35 | 36 | self.to_dir = to_dir 37 | 38 | self.session_id = str(datetime.datetime.now().replace(microsecond=0).timestamp())[:-2] 39 | 40 | snap_id = str(os.environ.get('SLOTH_SNAPSHOT_ID', "")) 41 | if snap_id == "": 42 | self.snapshot_id = self.session_id 43 | 44 | self.sloth_connector = SlothConnector(self.session_id, self.snapshot_id, self.to_dir) 45 | 46 | self.sloth_state = SlothConfig.SlothState.WATCHING 47 | os.environ['SLOTH_STATE'] = SlothConfig.SlothState.WATCHING 48 | os.environ['SLOTH_SNAPSHOT_ID'] = str(self.snapshot_id) 49 | 50 | sloth_log.info("Started id: " + self.session_id) 51 | 52 | def stop(self): 53 | 54 | sloth_log.info("Stopped id: " + self.session_id) 55 | 56 | self.sloth_state = SlothConfig.SlothState.IDLE 57 | os.environ['SLOTH_STATE'] = str(SlothConfig.SlothState.IDLE) 58 | self.snapshot_id = "" 59 | os.environ['SLOTH_SNAPSHOT_ID'] = "" 60 | 61 | zip_fn = self.sloth_connector.dump_data(self.data_watch_dump) 62 | 63 | sloth_log.info("Snapshot dumped to: " + zip_fn) 64 | 65 | def dump(self): 66 | 67 | self.stop() 68 | self.start() 69 | 70 | self.dump_counter = 0 71 | 72 | async def watch(self, fn, in_args: List = None, in_kwargs: Dict = None, res=None, additional_info: str = "", 73 | start_time: datetime = None, stop_time: datetime = None): 74 | 75 | sloth_log.debug("Start watching: " + str(fn)) 76 | 77 | if start_time and stop_time: 78 | run_time = (stop_time - start_time).microseconds 79 | else: 80 | run_time = 0 81 | 82 | try: 83 | 84 | func_dict = await self.watch_function(fn, in_args, run_time) 85 | 86 | args_dict = await self.watch_function_args(fn, in_args, in_kwargs) 87 | 88 | res_dict = await self.watch_function_result(res, additional_info) 89 | 90 | sloth_log.debug("End watching: " + str(fn)) 91 | 92 | self.data_watch_dump.append({ 93 | 'function': func_dict, 94 | 'arguments': args_dict, 95 | 'results': res_dict 96 | }) 97 | 98 | sloth_log.debug("Data dumped for: " + str(fn)) 99 | 100 | self.dump_counter += 1 101 | 102 | except Exception as e: 103 | 104 | sloth_log.error("Data was not dumped. Error: " + str(e)) 105 | 106 | async def watch_function(self, fn, in_args: List = None, run_time: int = 0) -> Dict: 107 | 108 | def get_full_scope(fn): 109 | # build a full path to the method 110 | 111 | def unique_path(shorter_dir: List = None, longer_dir: List = None) -> List: 112 | 113 | sep = 0 114 | for i in range(len(longer_dir)): 115 | if i >= len(shorter_dir): 116 | sep = i 117 | break 118 | if shorter_dir[i] != longer_dir[i]: 119 | sep = i 120 | break 121 | 122 | return longer_dir[-(len(longer_dir) - sep):] 123 | 124 | this_dir = os.getcwd() 125 | remote_dir = os.path.normpath(inspect.getfile(fn)[:-3]) 126 | 127 | remote_dir_list = remote_dir.split(os.path.sep) 128 | this_dir_list = this_dir.split(os.path.sep) 129 | 130 | diff_dir = unique_path(this_dir_list, remote_dir_list) 131 | 132 | return '.'.join(diff_dir) 133 | 134 | def get_callers_stack(fn) -> str: 135 | # get a human-readable stack of callers for the method 136 | 137 | stack = inspect.stack() 138 | 139 | stack_size = len(stack) 140 | 141 | modules = [(index, inspect.getmodule(stack[index][0])) 142 | for index in reversed(range(1, stack_size))] 143 | 144 | s = '{name}@{module} ' 145 | callers = [] 146 | 147 | for index, module in modules: 148 | if module.__name__.find("sloth") == -1: 149 | callers.append(s.format(module=module.__name__, name=stack[index][3])) 150 | 151 | callers.append(s.format(module=fn.__module__, name=fn.__name__)) 152 | 153 | callers.append('') 154 | callers.reverse() 155 | 156 | return ' <- '.join(callers) 157 | 158 | # check if the function is a class member 159 | # if it's a class member, we need to dump a class instance 160 | 161 | fn_name = fn.__name__ 162 | if fn.__qualname__.find(".") == -1: 163 | classname = "" 164 | class_dump = "" 165 | else: 166 | classname = fn.__qualname__.split(".")[0] 167 | class_dump = self.dump_class_with_joblib(in_args[0]) 168 | 169 | dict_comm = { 170 | 'instance_name': self.instance_id, 171 | 'snapshot_name': self.snapshot_id, 172 | 'scope_name': get_full_scope(fn), 173 | 'class_name': classname, 174 | 'class_dump': class_dump, 175 | 'function_name': fn_name, 176 | 'run_time': str(run_time), 177 | 'call_stack': get_callers_stack(fn) 178 | } 179 | 180 | return dict_comm 181 | 182 | async def watch_function_args(self, fn=None, in_args: List = None, in_kwargs: Dict = None) -> List: 183 | # bound income real arguments with the all possible arguments of the method 184 | # and serialize this kwargs list 185 | 186 | bound_args = inspect.signature(fn).bind(*in_args, **in_kwargs) 187 | bound_args.apply_defaults() 188 | target_args = dict(bound_args.arguments) 189 | 190 | var_pack = [] 191 | 192 | for key, value in target_args.items(): 193 | 194 | p_type_tmp = type(value) 195 | 196 | if p_type_tmp is int or p_type_tmp is float or p_type_tmp is bool: 197 | d_simple = True 198 | d_val = value 199 | else: 200 | d_simple = False 201 | d_val = self.dump_class_with_joblib(value) 202 | 203 | d_type = codecs.encode(pickle.dumps(p_type_tmp), "base64").decode() 204 | 205 | var_pack.append(self.var_d_pack(par_type=d_type, par_name=key, par_value=d_val, 206 | par_state=str(SlothConfig.SlothValueState.INCOME), 207 | par_simple=d_simple, )) 208 | 209 | await asyncio.sleep(1) 210 | 211 | return var_pack 212 | 213 | async def watch_function_result(self, res=None, additional_info: str = "") -> List: 214 | # watch and save the result of the method 215 | 216 | var_pack = [] 217 | if type(res) == tuple: 218 | 219 | i = 0 220 | for ret_val in res: 221 | 222 | p_type_tmp = type(ret_val) 223 | 224 | if p_type_tmp is int or p_type_tmp is float or p_type_tmp is bool: 225 | d_simple = True 226 | d_res = ret_val 227 | else: 228 | d_simple = False 229 | d_res = self.dump_class_with_joblib(ret_val) 230 | 231 | d_type = codecs.encode(pickle.dumps(p_type_tmp), "base64").decode() 232 | 233 | var_pack.append(self.var_d_pack(par_type=d_type, par_name='ret_' + str(i), par_value=d_res, 234 | par_state=str(SlothConfig.SlothValueState.RESULT), 235 | par_simple=d_simple, )) 236 | i = i + 1 237 | 238 | await asyncio.sleep(1) 239 | else: 240 | 241 | p_type_tmp = type(res) 242 | 243 | if p_type_tmp is int or p_type_tmp is float or p_type_tmp is bool: 244 | d_simple = True 245 | d_res = res 246 | else: 247 | d_simple = False 248 | d_res = self.dump_class_with_joblib(res) 249 | 250 | d_type = codecs.encode(pickle.dumps(p_type_tmp), "base64").decode() 251 | 252 | var_pack.append(self.var_d_pack(par_type=d_type, par_name='ret_0', par_value=d_res, 253 | par_state=str(SlothConfig.SlothValueState.RESULT), 254 | par_simple=d_simple, 255 | additional_info=additional_info)) 256 | 257 | return var_pack 258 | 259 | def dump_class_with_joblib(self, value) -> str: 260 | 261 | outputStream = io.BytesIO() 262 | joblib.dump(value, outputStream) 263 | outputStream.seek(0) 264 | d_val = codecs.encode(outputStream.read(), "base64").decode() 265 | 266 | return d_val 267 | 268 | def var_d_pack(self, **kwargs): 269 | 270 | return { 271 | 'par_type': kwargs.get('par_type', ""), 272 | 'par_name': kwargs.get('par_name', ""), 273 | 'par_value': str(kwargs.get('par_value', "")), 274 | 'par_state': kwargs.get('par_state', ""), 275 | 'par_simple': str(kwargs.get('par_simple', "")), 276 | 'additional_info': kwargs.get('additional_info', ""), 277 | } 278 | -------------------------------------------------------------------------------- /slothtest/sloth_xml_converter.py: -------------------------------------------------------------------------------- 1 | import xml.etree.ElementTree as ET 2 | import pickle 3 | import codecs 4 | import argparse 5 | import os 6 | import zipfile 7 | from typing import Dict, List 8 | 9 | # it can be either called via the CLI as a standalone, or as a class 10 | try: 11 | from .sloth_config import SlothConfig 12 | from .sloth_log import sloth_log 13 | except: 14 | from sloth_config import SlothConfig 15 | from sloth_log import sloth_log 16 | 17 | RUN_TIME_INCREASE_RATE = 3 18 | 19 | class SlothTestConverter: 20 | 21 | xml_list_tag = '__list__' 22 | xml_content_tag = '__content__' 23 | 24 | objects_eq_function = SlothConfig.objects_eq 25 | 26 | def _parseXMLtodict(self, parent): 27 | """ 28 | Converting incoming xml file to dict for better usability 29 | 30 | :param parent: root element, on initial run 31 | :return: dict 32 | """ 33 | 34 | ret = {} 35 | if parent.items(): 36 | ret.update(dict(parent.items())) 37 | 38 | if parent.text: 39 | ret[self.xml_content_tag] = parent.text 40 | 41 | if 's_list' in parent.tag: 42 | ret[self.xml_list_tag] = [] 43 | for element in parent: 44 | ret[self.xml_list_tag].append(self._parseXMLtodict(element)) 45 | else: 46 | for element in parent: 47 | ret[element.tag] = self._parseXMLtodict(element) 48 | 49 | return ret 50 | 51 | def create_text_of_test_module(self, func_data_dict: Dict = None) -> (str, str): 52 | """ 53 | Creating python code for test module, based on incoming dictionary with the description of the function and params 54 | 55 | Results are separated by two modules: test and variables. 56 | Test module consists of the python unit test for a particular function 57 | Variables module consists of the income and outcome variables, used in created unit test module 58 | 59 | :param func_data_dict: description of the function and income/outcome params 60 | :return: text of the test module, text of the variables module 61 | 62 | == Test module text ex: 63 | 64 | ... 65 | def test_cleanup_1(): 66 | from main import cleanup 67 | 68 | try: 69 | run_result = cleanup(dt=sl.val_cleanup_1_dt, column_name=sl.val_cleanup_1_column_name, ) 70 | except Exception as e: 71 | run_result = e 72 | 73 | test_result = sl.res_cleanup_1_ret_0 74 | assert(run_result.equals(test_result)) 75 | ... 76 | 77 | == Variables module text ex: 78 | 79 | ... 80 | val_cleanup_1_dt = pickle.loads(codecs.decode('gANjcGFu.......'.encode(),"base64")) 81 | val_cleanup_1_column_name = 'Advertisers list' 82 | res_cleanup_1_ret_0 = pickle.loads(codecs.decode('gANjcGFuZGF.........'.encode(),"base64")) 83 | ... 84 | 85 | """ 86 | 87 | func_text = "" 88 | var_text = "" 89 | 90 | if not func_data_dict: 91 | sloth_log.error("Couldn't convert the pack. Data dict was not provided!") 92 | return func_text, var_text 93 | 94 | run_id = func_data_dict.get('run_id', "") 95 | scope = func_data_dict.get('scope', "") 96 | classname = func_data_dict.get('class_name', "") 97 | class_dump = func_data_dict.get('class_dump', "") 98 | fnname = func_data_dict.get('func_name', "") 99 | run_time = int(func_data_dict.get('run_time', 0)) 100 | target_values_raw = func_data_dict.get('in', []) 101 | target_result_raw = func_data_dict.get('out', []) 102 | 103 | target_values = self.get_dumped_parameters(target_values_raw) 104 | target_result = self.get_dumped_parameters(target_result_raw) 105 | 106 | t_func_name = fnname + "_" + str(run_id) 107 | 108 | func_text += 'def test_'+t_func_name+"(): \n" 109 | 110 | if classname == "": 111 | func_text += ' from ' + scope + ' import ' + fnname + '\n' 112 | 113 | par_str = "" 114 | for v_val in target_values: 115 | 116 | parname = str(v_val['par_name']) 117 | 118 | if parname == 'self': 119 | continue 120 | 121 | v_parname = 'val_' + t_func_name + '_' + parname 122 | 123 | if v_val['par_simple']: 124 | 125 | parval = v_val['par_value'] 126 | var_text += v_parname + ' = ' + str(eval(parval)) + '\n\n' 127 | 128 | else: 129 | 130 | parval = "%r" % v_val['par_value'] 131 | var_text += 'var_stream = io.BytesIO()\n' 132 | var_text += 'var_stream_str = codecs.decode(' + parval + '.encode(),"base64")\n\n' 133 | var_text += 'var_stream.write(var_stream_str)\n' 134 | var_text += 'var_stream.seek(0)\n' 135 | var_text += v_parname + ' = joblib.load(var_stream)\n\n' 136 | 137 | par_str += parname + '=sl.' + v_parname + ', ' 138 | 139 | func_text += "\n try:\n" 140 | 141 | if classname == "": 142 | 143 | if run_time > 0: 144 | func_text += ' start_time = datetime.datetime.now()\n' 145 | 146 | func_text += ' run_result = ' + fnname + "(" + par_str + ") \n" 147 | 148 | if run_time > 0: 149 | func_text += ' stop_time = datetime.datetime.now()\n' 150 | func_text += ' run_time = (stop_time - start_time).microseconds\n' 151 | 152 | else: 153 | parval = "%r" % class_dump 154 | var_text += 'class_stream = io.BytesIO()\n' 155 | var_text += 'class_stream_str = codecs.decode(' + parval + '.encode(),"base64")\n' 156 | var_text += 'class_stream.write(class_stream_str)\n' 157 | var_text += 'class_stream.seek(0)\n' 158 | v_classname = 'cls_' + t_func_name + '_' + classname 159 | var_text += v_classname + ' = joblib.load(class_stream)\n\n' 160 | 161 | if run_time > 0: 162 | func_text += ' start_time = datetime.datetime.now()\n' 163 | 164 | func_text += ' run_result = sl.' + v_classname + "." + fnname + "(" + par_str + ") \n" 165 | 166 | if run_time > 0: 167 | func_text += ' stop_time = datetime.datetime.now()\n' 168 | func_text += ' run_time = (stop_time - start_time).microseconds\n' 169 | 170 | func_text += ' except Exception as e:\n' 171 | func_text += ' run_time = 0\n' 172 | func_text += ' run_result = e\n\n' 173 | 174 | # result 175 | 176 | res = [] 177 | for v_res in target_result: 178 | 179 | parname = str(v_res['par_name']) 180 | v_parname = 'res_' + t_func_name + '_' + parname 181 | 182 | if v_res['par_simple']: 183 | 184 | parval = v_res['par_value'] 185 | var_text += v_parname + ' = ' + str(eval(parval)) + '\n\n' 186 | 187 | else: 188 | 189 | parval = "%r" % v_res['par_value'] 190 | var_text += 'res_stream = io.BytesIO()\n' 191 | var_text += 'res_stream_str = codecs.decode(' + parval + '.encode(),"base64")\n' 192 | var_text += 'res_stream.write(res_stream_str)\n' 193 | var_text += 'res_stream.seek(0)\n' 194 | var_text += v_parname + ' = joblib.load(res_stream)\n\n' 195 | 196 | res.append('sl.' + v_parname) 197 | 198 | t_res = ', '.join(res) 199 | 200 | if len(target_result) == 0: 201 | 202 | func_text += " assert(run_result is None)\n" 203 | 204 | elif len(target_result) == 1: 205 | 206 | func_text += f" test_result = {t_res} \n" 207 | 208 | var_type = target_result[0]['par_type'] 209 | 210 | eq_expr = self.objects_eq_function.get(str(var_type), None) 211 | if eq_expr is None: 212 | eq_expr = "__eq__" 213 | 214 | func_text += " assert(type(run_result) == type(test_result))\n" 215 | func_text += " assert(run_result." + eq_expr + "(test_result))\n" 216 | 217 | else: 218 | 219 | func_text += f" test_result = ({t_res}) \n" 220 | 221 | for i in range(len(target_result)): 222 | 223 | var_type = target_result[i]['par_type'] 224 | 225 | eq_expr = self.objects_eq_function.get(str(var_type), None) 226 | if eq_expr is None: 227 | eq_expr = "__eq__" 228 | 229 | func_text += " assert(type(run_result["+str(i)+"]) == type(test_result["+str(i)+"]))\n" 230 | func_text += " assert(run_result["+str(i)+"]." + eq_expr + "(test_result["+str(i)+"]))\n" 231 | 232 | if run_time > 0: 233 | max__run_time = run_time * RUN_TIME_INCREASE_RATE 234 | func_text += f" assert(run_time <= {max__run_time})\n" 235 | 236 | return func_text, var_text 237 | 238 | def get_dumped_parameters(self, par_val_arr: List = None) -> List: 239 | """ 240 | 241 | Collecting and parsing an array of income/outcome parameters to dict 242 | An additional logic: decision if the value is simple or not (can you push a value itself to a var, 243 | or you can serialize only) 244 | 245 | :param par_val_arr: an array of parameters 246 | :return: an array of dictionaries 247 | """ 248 | 249 | params = [] 250 | 251 | if par_val_arr is not None: 252 | 253 | for rec in par_val_arr: 254 | 255 | p_type_tmp = pickle.loads(codecs.decode(rec['par_type'].encode(), "base64")) 256 | 257 | t_d = { 258 | 'par_name': rec['par_name'], 259 | 'par_value': rec['par_value'], 260 | 'par_type': p_type_tmp, 261 | 'par_simple': rec['par_simple'] 262 | } 263 | params.append(t_d) 264 | 265 | return params 266 | 267 | def parse_file_create_tests(self, filename: str = None, to_dir: str = None): 268 | """ 269 | The Main function. Get an XML file (zip) of dumped functions and converts it to python unit-test code 270 | 271 | The result is two files: 272 | test_sloth.py - python code for unit testing all dumped function 273 | sloth_test_parval.py - python code for dumped variables that used in unit tests 274 | 275 | :param filename: the XML file with dumps you need to convert 276 | :param to_dir: the directory where to put the result files (current dir by default) 277 | :return: None 278 | """ 279 | 280 | if filename is None: 281 | sloth_log.error('Filename was not defined') 282 | raise Exception('Filename was not defined') 283 | 284 | packname = os.path.basename(filename)[:-4] 285 | with zipfile.ZipFile(filename) as myzip: 286 | xml_filename = packname+'.xml' 287 | with myzip.open(xml_filename) as myfile: 288 | with open(os.path.join(os.path.dirname(filename),xml_filename), "wb") as fh: 289 | fh.write(myfile.read()) 290 | 291 | sloth_log.info("Converting: " + packname) 292 | 293 | if to_dir is None: 294 | to_dir = os.path.dirname(os.path.abspath(__file__)) 295 | 296 | try: 297 | tree = ET.parse(xml_filename) 298 | root = tree.getroot() 299 | except Exception as e: 300 | sloth_log.error('Error while parsing the XML file: ' + str(e)) 301 | raise Exception('Error while parsing the XML file: ' + str(e)) 302 | 303 | func_dict = self._parseXMLtodict(root) 304 | 305 | target_test_file = 'import sloth_test_parval_'+packname+' as sl \n\n' 306 | target_test_file += "import datetime\n\n" 307 | 308 | target_variable_file = "import codecs\n" 309 | target_variable_file += "import io\n" 310 | target_variable_file += "import joblib\n" 311 | 312 | for func_element in func_dict['functions_list'][self.xml_list_tag]: 313 | 314 | f_run_id = func_element['run_id'].get(self.xml_content_tag, "") 315 | f_scope = func_element['scope_name'].get(self.xml_content_tag, "") 316 | f_class_name = func_element['class_name'].get(self.xml_content_tag, "") 317 | f_class_dump = func_element['class_dump'].get(self.xml_content_tag, "") 318 | f_name = func_element['function_name'].get(self.xml_content_tag, "") 319 | run_time = func_element['run_time'].get(self.xml_content_tag, "") 320 | 321 | f_in = [] 322 | for arg_element in func_element['arguments_list'][self.xml_list_tag]: 323 | in_d = { 324 | 'par_type': arg_element['par_type'].get(self.xml_content_tag, ""), 325 | 'par_name': arg_element['par_name'].get(self.xml_content_tag, ""), 326 | 'par_value': arg_element['par_value'].get(self.xml_content_tag, ""), 327 | 'par_simple': eval(arg_element['par_simple'].get(self.xml_content_tag, 'True')), 328 | } 329 | 330 | f_in.append(in_d) 331 | 332 | f_out = [] 333 | for arg_element in func_element['results_list'][self.xml_list_tag]: 334 | out_d = { 335 | 'par_type': arg_element['par_type'].get(self.xml_content_tag, ""), 336 | 'par_name': arg_element['par_name'].get(self.xml_content_tag, ""), 337 | 'par_value': arg_element['par_value'].get(self.xml_content_tag, ""), 338 | 'par_simple': eval(arg_element['par_simple'].get(self.xml_content_tag, 'True')), 339 | } 340 | 341 | f_out.append(out_d) 342 | 343 | func_dict = { 344 | 'scope': f_scope, 345 | 'class_name': f_class_name, 346 | 'class_dump': f_class_dump, 347 | 'func_name': f_name, 348 | 'run_time': run_time, 349 | 'run_id': f_run_id, 350 | 'in': f_in, 351 | 'out': f_out 352 | } 353 | 354 | t_f_t, v_f_t = self.create_text_of_test_module(func_dict) 355 | 356 | target_variable_file += "\n# ===== "+f_run_id+": "+f_name+"@"+f_scope+"\n\n" 357 | 358 | target_test_file += t_f_t 359 | target_variable_file += v_f_t 360 | 361 | target_variable_file += "\n\n" 362 | target_test_file += "\n\n" 363 | 364 | ttf = os.path.join(to_dir, "test_sloth_" + packname + ".py") 365 | with open(ttf, 'w') as f: 366 | f.write(target_test_file) 367 | 368 | ttv = os.path.join(to_dir, "sloth_test_parval_"+packname+".py") 369 | with open(ttv, 'w') as f: 370 | f.write(target_variable_file) 371 | 372 | os.remove(xml_filename) 373 | 374 | sloth_log.info("Convertion finished. Files " + ttf + " and "+ttv+" created!") 375 | 376 | 377 | if __name__ == '__main__': 378 | 379 | import sys 380 | 381 | parser = argparse.ArgumentParser(description='Sloth Watcher Dump to pytest converter') 382 | parser.add_argument("filename", help="a Sloth's xml dump file (zip archive)") 383 | parser.add_argument('-p', "--project_dir", help="The directory where the target project lives", 384 | default=os.getcwd()) 385 | parser.add_argument('-d', "--to_dir", help="The directory for result files", 386 | default=os.getcwd()) 387 | 388 | args = parser.parse_args() 389 | 390 | sys.path.append(os.path.abspath(args.project_dir)) 391 | 392 | sltc = SlothTestConverter() 393 | sltc.parse_file_create_tests(args.filename, args.to_dir) 394 | 395 | --------------------------------------------------------------------------------