├── .gitignore ├── README.md ├── README.rst ├── deploy.sh ├── losswise ├── __init__.py └── libs.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist/ 3 | *.html 4 | zenpix.egg-info 5 | *venv -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # losswise-python 2 | 3 | To install this package do: 4 | 5 | ```shell 6 | pip install losswise 7 | ``` 8 | 9 | Please consult for usage instructions. 10 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | losswise-python 2 | ============================== 3 | 4 | This is the official Losswise Python library. This library allows for 5 | server-side integration of Losswise. 6 | 7 | 8 | Installation 9 | ------------ 10 | 11 | The library can be installed using pip:: 12 | 13 | pip install losswise 14 | 15 | 16 | Getting Started 17 | --------------- 18 | 19 | First create an account on the Losswise website (https://losswise.com). This will automatically generate a unique API key. 20 | 21 | Typical usage usually looks like this:: 22 | 23 | import random 24 | import losswise 25 | 26 | # replace with your own api key 27 | losswise.set_api_key('your_api_key') 28 | 29 | # replace with a string that identifies your model 30 | session = losswise.Session(tag='my_dilated_convnet', max_iter=10, data={'num_params': 10000000}) 31 | 32 | # create empty graph for loss, keep track of minima here hence kind='min' 33 | graph = session.graph(title='loss', kind='min') 34 | 35 | # track artificial loss over time 36 | for x in xrange(10): 37 | train_loss = 1. / (0.1 + x + 0.1 * random.random()) 38 | test_loss = 1.5 / (0.1 + x + 0.2 * random.random()) 39 | graph.append(x, {'train_loss': train_loss, 'test_loss': test_loss}) 40 | 41 | # mark session as complete 42 | session.done() 43 | 44 | 45 | You can then view the visualization results on your dashboard. 46 | 47 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | rm dist/* 2 | python setup.py sdist 3 | python setup.py bdist_wheel --universal 4 | twine upload dist/* 5 | -------------------------------------------------------------------------------- /losswise/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import threading 4 | import json 5 | import time 6 | import requests 7 | import random 8 | import math 9 | from six.moves import queue 10 | from six import iteritems 11 | import subprocess 12 | from threading import Thread 13 | import re 14 | import base64 15 | import io 16 | 17 | 18 | API_KEY = os.environ.get('LW_API_KEY', None) 19 | BASE_URL = os.environ.get('LW_BASE_URL', 'https://api.losswise.com') 20 | 21 | 22 | def set_api_key(api_key): 23 | global API_KEY 24 | API_KEY = api_key 25 | 26 | 27 | def set_base_url(base_url): 28 | global BASE_URL 29 | BASE_URL = base_url 30 | 31 | 32 | def get_git_info(): 33 | git_info = {'diff' : '', 'branch': '', 'url': ''} 34 | try: 35 | git_info['diff'] = subprocess.check_output(['git', 'diff']).decode('utf8') 36 | size_mb = sys.getsizeof(git_info['diff']) / 1000000. 37 | if size_mb > 0.2: 38 | git_info['diff'] = "git diff too large to show here" 39 | print("Losswise warning: git diff too large to track.") 40 | git_info['branch'] = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf8').strip() 41 | git_remote = subprocess.check_output(['git', 'remote', '-v']).decode('utf8') 42 | git_info['url'] = re.findall('\S*\.git', git_remote)[0] 43 | except Exception as e: 44 | print(e) 45 | return git_info 46 | 47 | 48 | work_queue = queue.Queue() 49 | def worker(): 50 | while True: 51 | point_list = [] 52 | stats_map = {} 53 | while not work_queue.empty() or len(point_list) == 0: 54 | (x, y, stats, t, graph_id, session_id) = work_queue.get() 55 | json_data = { 56 | 'x': x, 57 | 'y': y, 58 | 'time': t, 59 | 'graph_id': graph_id, 60 | 'session_id': session_id, 61 | } 62 | if any(stats): 63 | stats_map[graph_id] = stats 64 | point_list.append(json_data) 65 | json_message = json.dumps({'point_list': point_list, 'stats_map': stats_map}) 66 | url = BASE_URL + '/api/v1/point-list' 67 | headers = {"Authorization": API_KEY, "Content-type": "application/json"} 68 | try: 69 | r = requests.post(url, data=json_message, headers=headers) 70 | json_resp = r.json() 71 | if json_resp.get('error', None): 72 | print(json_resp['error']) 73 | print(point_list) 74 | print(stats_map) 75 | except requests.exceptions.ConnectionError: 76 | print("Losswise warning: request failed.") 77 | except Exception as e: 78 | print(e) 79 | for _ in range(len(point_list)): 80 | work_queue.task_done() 81 | 82 | 83 | event_thread = Thread(target=worker) 84 | event_thread.daemon = True 85 | event_thread.start() 86 | 87 | 88 | class Graph(object): 89 | def __init__(self, tracker, xlabel, ylabel, title, kind=None, max_iter=None, display_interval=None): 90 | self.tracker = tracker 91 | self.kind = kind 92 | self.max_iter = max_iter 93 | if display_interval is not None: 94 | self.display_interval = display_interval 95 | else: 96 | if max_iter is None: 97 | print("Losswise warning: please set max_iter or display_interval for optimal user experience.") 98 | print("Losswise will track all points without smoothing.") 99 | self.display_interval = 1 100 | else: 101 | self.display_interval = max(1, max_iter // 200) 102 | print("Losswise: choosing optimal display_interval = %d for \"%s\" graph." % (self.display_interval, title)) 103 | print("You may override this default behavior by manually setting display_interval yourself.") 104 | json_data = { 105 | 'session_id': self.tracker.session_id, 106 | 'xlabel': xlabel, 107 | 'ylabel': ylabel, 108 | 'title': title, 109 | 'kind': kind, 110 | 'display_interval': display_interval 111 | } 112 | json_message = json.dumps(json_data) 113 | r = requests.post(BASE_URL + '/api/v1/graphs', 114 | data=json_message, 115 | headers={"Authorization": API_KEY, "Content-type": "application/json"}) 116 | json_resp = r.json() 117 | if json_resp['success'] is True: 118 | self.graph_id = r.json()['graph_id'] 119 | else: 120 | error = json_resp['error'] 121 | error_msg = 'Unable to create graph: %s' % error 122 | raise RuntimeError(error_msg) 123 | self.tracked_value_map = {} 124 | self.stats = {} 125 | if kind not in ['min', 'max', None]: 126 | raise ValueError("'kind' variable must be 'min', 'max', or empty!") 127 | self.kind = kind 128 | self.x = 0 129 | 130 | def now(self): 131 | return time.time() 132 | 133 | def append(self, *args): 134 | if len(args) == 1: 135 | x = int(self.x) 136 | y_raw = args[0] 137 | elif len(args) == 2: 138 | x = int(args[0]) 139 | y_raw = args[1] 140 | else: 141 | raise ValueError("Append method only accepts one or two arguments.") 142 | stats_update = {} 143 | y = {} 144 | for key, val in iteritems(y_raw): 145 | if math.isnan(val): 146 | print("Losswise warning: skipping '%s' due to NaN value." % key) 147 | continue 148 | if math.isinf(val): 149 | print("Losswise warning: skipping '%s' due to Infinite value." % key) 150 | continue 151 | if key in self.tracked_value_map: 152 | tracked_value_list = self.tracked_value_map[key] 153 | tracked_value_list.append((x, val)) 154 | if self.max_iter is not None and x < self.max_iter - 1 and x % self.display_interval != 0: 155 | return 156 | tracked_value_len = len(tracked_value_list) 157 | diff = tracked_value_list[tracked_value_len - 1][0] - tracked_value_list[tracked_value_len - 2][0] 158 | if diff == 1 and tracked_value_len >= self.display_interval > 1: 159 | xy_tuple_values = tracked_value_list[-self.display_interval:] 160 | y_values = [xy_tuple[1] for xy_tuple in xy_tuple_values] 161 | y_smooth = sum(y_values) / len(y_values) 162 | y[key] = float(y_smooth) 163 | else: 164 | y[key] = float(y_raw[key]) 165 | self.tracked_value_map[key] = [(x, val)] 166 | if len(tracked_value_list) > 3 * self.display_interval: 167 | del tracked_value_list[:self.display_interval] 168 | else: 169 | y[key] = float(y_raw[key]) 170 | self.tracked_value_map[key] = [(x, val)] 171 | data_new = y.copy() 172 | data_new['x'] = x 173 | if self.max_iter is not None: 174 | data_new['xper'] = min(1., (x + 1.) / self.max_iter) 175 | for key, val in iteritems(data_new): 176 | if key in ['x', 'xper']: 177 | kind = 'max' 178 | else: 179 | kind = self.kind 180 | if kind is None: 181 | continue 182 | val_old = self.stats.get(key, {}).get(kind, None) 183 | if val_old is None: 184 | val_new = val 185 | elif kind == 'max': 186 | val_new = max(val, val_old) 187 | elif kind == 'min': 188 | val_new = min(val, val_old) 189 | if val_new != val_old: 190 | stats_update[key] = {kind: val_new} 191 | self.stats.update(stats_update) 192 | if any(stats_update): 193 | stats = self.stats 194 | else: 195 | stats = {} 196 | work_queue.put((x, y, stats, int(self.now()), self.graph_id, self.tracker.session_id)) 197 | self.x = self.x + 1 198 | 199 | 200 | class ImageSequence(object): 201 | def __init__(self, session_id, x, name): 202 | json_data = {"session_id": session_id, "name": name, "x": x, "type": "image"} 203 | json_message = json.dumps(json_data) 204 | self.prediction_sequence_id = None 205 | try: 206 | r = requests.post(BASE_URL + '/api/v1/prediction-sequences', 207 | data=json_message, 208 | headers={"Authorization": API_KEY, "Content-type": "application/json"}) 209 | json_resp = r.json() 210 | except Exception as e: 211 | print(e) 212 | return 213 | if json_resp.get('success', False) is True: 214 | self.prediction_sequence_id = json_resp["prediction_sequence_id"] 215 | else: 216 | error = json_resp.get('error', '') 217 | error_msg = 'Unable to create image sequence: %s' % error 218 | print(error_msg) 219 | 220 | def append(self, image_pil, image_id='', outputs={}, metrics={}): 221 | if not isinstance(image_id, str): 222 | raise TypeError("\"image_id\" argument must be a valid python string") 223 | if not isinstance(outputs, dict): 224 | raise TypeError("\"outputs\" argument must be a valid python dictionary") 225 | if not isinstance(metrics, dict): 226 | raise TypeError("\"metrics\" argument must be a valid python dictionary") 227 | if self.prediction_sequence_id is None: 228 | print("Skipping append due to failed create image sequence API call.") 229 | return 230 | image_buffer = io.BytesIO() 231 | try: 232 | image_pil.save(image_buffer, format='PNG') 233 | except AttributeError as e: 234 | print("Unable to save image as PNG! Make sure you're using a PIL image.") 235 | return 236 | contents = image_buffer.getvalue() 237 | image_buffer.close() 238 | image_data = base64.b64encode(contents).decode('utf-8') 239 | json_data = {"prediction_sequence_id": self.prediction_sequence_id, 240 | "image": image_data, 241 | "metrics": metrics, 242 | "outputs": outputs, 243 | "image_id": image_id} 244 | json_message = json.dumps(json_data) 245 | try: 246 | r = requests.post(BASE_URL + '/api/v1/image-prediction', 247 | data=json_message, 248 | headers={"Authorization": API_KEY, "Content-type": "application/json"}) 249 | json_resp = r.json() 250 | err = json_resp.get("error", None) 251 | if json_resp['success'] is False and err: 252 | print ("Request failed! " + err) 253 | except requests.exceptions.ConnectionError: 254 | print("Losswise warning: request failed due to connection error.") 255 | except Exception as e: 256 | print("Losswise warning: POST request failure.") 257 | print(e) 258 | 259 | 260 | class Session(object): 261 | def __init__(self, tag=None, max_iter=None, params={}, track_git=True, info={}): 262 | self.graph_list = [] 263 | self.max_iter = max_iter 264 | self.api_key = API_KEY 265 | git_info = get_git_info() 266 | self.tag = 'default' 267 | if tag is None: 268 | if 'BUILDKITE_BRANCH' in os.environ: 269 | self.tag = os.environ['BUILDKITE_BRANCH'] 270 | elif git_info.get('branch', None) is not None: 271 | if len(git_info['branch'].replace(" ", "")) > 0: 272 | self.tag = git_info['branch'] 273 | else: 274 | self.tag = tag 275 | json_data = { 276 | 'tag': self.tag, 277 | 'params': params, 278 | 'max_iter': max_iter, 279 | 'env': {}, 280 | 'lbr_build_uuid': os.environ.get("LBR_BUILD_UUID", "") 281 | } 282 | if track_git: 283 | json_data['git'] = git_info 284 | if info: 285 | try: 286 | assert type(info) == dict 287 | for key, val in info.items(): 288 | assert type(key) == type(val) == str 289 | json_data['info'] = info 290 | except Exception as e: 291 | print(e) 292 | raise RuntimeError("Error: info must be dictionary with string keys and values") 293 | for env_var in ['BUILDKITE_BUILD_URL', 'BUILDKITE_REPO', 294 | 'BUILDKITE_PIPELINE_PROVIDER', 'BUILDKITE_BRANCH', 295 | 'LBR_BUILD_UUID', 'LBR_AGENT_ID']: 296 | if env_var in os.environ: 297 | json_data['env'][env_var] = os.environ[env_var] 298 | json_message = json.dumps(json_data) 299 | try: 300 | r = requests.post(BASE_URL + '/api/v1/sessions', 301 | data=json_message, 302 | headers={"Authorization": API_KEY, "Content-type": "application/json"}) 303 | except requests.exceptions.ConnectionError: 304 | raise RuntimeError("Error: losswise connection tracker failed. Please contact support@losswise.com") 305 | json_resp = r.json() 306 | if json_resp['success'] is True: 307 | self.session_id = json_resp['session_id'] 308 | else: 309 | error = json_resp['error'] 310 | error_msg = 'Unable to create session: %s. Please contact support@losswise.com' % error 311 | raise RuntimeError(error_msg) 312 | # start monitoring thread 313 | self.status = 'active' 314 | self.stop_event = threading.Event() 315 | def keepalive(stop_event): 316 | while not stop_event.is_set(): 317 | json_message = json.dumps({'attributes' : {'status': self.status}}) 318 | try: 319 | r = requests.patch(BASE_URL + '/api/v1/sessions/' + self.session_id, 320 | data=json_message, 321 | headers={"Authorization": API_KEY, "Content-type": "application/json"}) 322 | except requests.exceptions.ConnectionError: 323 | print("Losswise warning: request failed.") 324 | except Exception as e: 325 | print(e) 326 | time.sleep(10) 327 | self.thread = Thread(target=keepalive, args=(self.stop_event,)) 328 | self.thread.daemon = True 329 | self.thread.start() 330 | 331 | def done(self): 332 | self.status = 'complete' 333 | self.stop_event.set() 334 | work_queue.join() 335 | json_message = json.dumps({'attributes' : {'status': self.status}}) 336 | try: 337 | r = requests.patch(BASE_URL + '/api/v1/sessions/' + self.session_id, 338 | data=json_message, 339 | headers={"Authorization": API_KEY, "Content-type": "application/json"}) 340 | except requests.exceptions.ConnectionError: 341 | print("Losswise warning: request failed.") 342 | except Exception as e: 343 | print(e) 344 | 345 | def image_sequence(self, x, name=''): 346 | seq = ImageSequence(self.session_id, x, name) 347 | return seq 348 | 349 | def graph(self, title='', xlabel='', ylabel='', kind=None, display_interval=None): 350 | assert kind in [None, 'min', 'max'] 351 | graph = Graph(self, title=title, xlabel=xlabel, ylabel=ylabel, 352 | kind=kind, max_iter=self.max_iter, display_interval=display_interval) 353 | self.graph_list.append(graph) 354 | return graph 355 | 356 | def Graph(self, title='', xlabel='', ylabel='', kind=None, display_interval=None): 357 | assert kind in [None, 'min', 'max'] 358 | graph = Graph(self, title=title, xlabel=xlabel, ylabel=ylabel, 359 | kind=kind, max_iter=self.max_iter, display_interval=self.display_interval) 360 | self.graph_list.append(graph) 361 | return graph 362 | 363 | def set_values(self, values): 364 | if not isinstance(values, dict): 365 | print("set_values must take a python dictionary as input! skipping") 366 | return 367 | json_message = json.dumps({'session_id': self.session_id, 'values': values}) 368 | url = BASE_URL + '/api/v1/values' 369 | headers = {"Authorization": API_KEY, "Content-type": "application/json"} 370 | try: 371 | r = requests.post(url, data=json_message, headers=headers) 372 | json_resp = r.json() 373 | if json_resp.get('error', None): 374 | print(json_resp['error']) 375 | print(point_list) 376 | print(stats_map) 377 | except requests.exceptions.ConnectionError: 378 | print("Losswise warning: request failed.") 379 | except Exception as e: 380 | print(e) 381 | -------------------------------------------------------------------------------- /losswise/libs.py: -------------------------------------------------------------------------------- 1 | from losswise import Session 2 | from keras.callbacks import Callback 3 | 4 | 5 | class LosswiseKerasCallback(Callback): 6 | def __init__(self, tag=None, params={}, track_git=True, display_interval=None, max_iter=None): 7 | # model hyper parameters, json serializable Python object 8 | self.tag = tag 9 | if not isinstance(params, dict): 10 | raise TypeError("\"params\" argument must be a valid python dictionary") 11 | if tag is not None and not isinstance(tag, str): 12 | raise TypeError("\"tag\" argument must be a valid python string") 13 | self.params_data = params 14 | self.track_git = track_git 15 | self.graph_map = {} 16 | self.display_interval = display_interval 17 | self.max_iter = max_iter 18 | super(LosswiseKerasCallback, self).__init__() 19 | def on_train_begin(self, logs={}): 20 | if self.max_iter is None: 21 | if 'epochs' in self.params and 'samples' in self.params and self.params['samples'] is not None and 'batch_size' in self.params and self.params['batch_size'] is not None: 22 | self.max_iter = int(self.params['epochs'] * self.params['samples'] / self.params['batch_size']) 23 | elif 'steps_per_epoch' in self.params and 'epochs' in self.params: 24 | self.max_iter = self.params['steps_per_epoch'] * self.params['epochs'] 25 | elif 'samples_per_epoch' in self.params and 'epochs' in self.params: 26 | self.max_iter = self.params['samples_per_epoch'] * self.params['epochs'] 27 | elif 'steps' in self.params and 'epochs' in self.params: 28 | self.max_iter = self.params['steps'] * self.params['epochs'] 29 | else: 30 | print("Warning: Please specify max_iter!") 31 | print("You have not set max_iter, for example do LosswiseKerasCallback(..., max_iter=10000)") 32 | self.session = Session(tag=self.tag, max_iter=self.max_iter, params=self.params_data, 33 | track_git=self.track_git) 34 | self.metric_list = [] 35 | for metric in self.params['metrics']: 36 | if not metric.startswith('val_'): 37 | if metric not in self.metric_list: 38 | self.metric_list.append(metric) 39 | for metric in self.metric_list: 40 | if 'acc' in metric: 41 | kind = 'max' 42 | else: 43 | kind = 'min' 44 | self.graph_map[metric] = self.session.graph(metric, kind=kind, display_interval=self.display_interval) 45 | self.x = 0 46 | def on_epoch_end(self, epoch, logs={}): 47 | for metric in self.metric_list: 48 | metric_val = "val_" + metric 49 | if metric_val in logs: 50 | data = {metric_val: logs[metric_val]} 51 | self.graph_map[metric].append(self.x, data) 52 | def on_batch_end(self, batch, logs={}): 53 | for metric in self.metric_list: 54 | data = {metric: logs.get(metric)} 55 | self.graph_map[metric].append(self.x, data) 56 | self.x += 1 57 | def on_train_end(self, logs={}): 58 | self.session.done() 59 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='losswise', 5 | version='4.0', 6 | description='Official Losswise library for Python', 7 | long_description=open('README.rst').read(), 8 | url='https://losswise.com', 9 | author='Losswise, Inc.', 10 | author_email='nicodjimenez@gmail.com', 11 | license='Apache', 12 | install_requires=['requests >= 2.9.1', 'six >= 1.9.0'], 13 | classifiers=[ 14 | 'Operating System :: OS Independent', 15 | 'Programming Language :: Python :: 2', 16 | 'Programming Language :: Python :: 3', 17 | ], 18 | 19 | keywords='losswise analytics dashboard deep machine learning tensorflow pytorch optimization ML deep learning', 20 | packages=['losswise'], 21 | ) 22 | 23 | --------------------------------------------------------------------------------