├── .gitignore ├── bfx.py ├── bitmex-wallet-parser.py ├── config.sample.json ├── deribit.py ├── readme.md └── requirements.pip /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.pyo 3 | *.pyc 4 | 20* 5 | Wallet* 6 | config.* 7 | -------------------------------------------------------------------------------- /bfx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.6 2 | 3 | # Logging module 4 | import coloredlogs, logging 5 | logger = logging.getLogger(__name__) 6 | coloredlogs.install(level='DEBUG', logger=logger) 7 | 8 | # Datetime 9 | from datetime import datetime, timedelta as datetime_timedelta 10 | import time 11 | from dateutil.relativedelta import relativedelta 12 | 13 | # For api requests 14 | from requests import get as requests_get 15 | 16 | # # Loading/unloading json (for api requests) 17 | from json import dumps as json_dumps, loads as json_loads 18 | 19 | 20 | 21 | class Timeperiods: 22 | 23 | 24 | timeframes_list = ['1T','5T','15T','30T','1H','2H','3H','4H','6H','8H','12H','1D','W-MON','MS'] 25 | timeframes = { 26 | '1T' : { 27 | 'seconds': 60, 28 | }, 29 | '5T' : { 30 | 'upcycle': '1T', 31 | 'seconds':5*60, 32 | 'group_intervals': [0,5,10,15,20,25,30,35,40,45,50,55] 33 | }, 34 | '15T' : { 35 | 'upcycle': '5T', 36 | 'seconds':15*60, 37 | 'group_intervals': [0,15,30,45] 38 | }, 39 | '30T' : { 40 | 'upcycle': '15T', 41 | 'seconds':30*60, 42 | 'group_intervals': [0,30] 43 | }, 44 | '1H' : { 45 | 'upcycle': '30T', 46 | 'seconds':60*60, 47 | 'group_intervals': [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23] 48 | }, 49 | '2H' : { 50 | 'upcycle': '1H', 51 | 'seconds':2*(60*60), 52 | 'group_intervals': [0,2,4,6,8,10,12,14,16,18,20,22] 53 | }, 54 | '3H' : { 55 | 'upcycle': '1H', 56 | 'seconds':3*(60*60), 57 | 'group_intervals': [0,3,6,9,12,18,21] 58 | }, 59 | '4H' : { 60 | 'upcycle': '2H', 61 | 'seconds':4*(60*60), 62 | 'group_intervals': [0,4,8,12,16,20] 63 | }, 64 | '6H' : { 65 | 'upcycle': '3H', 66 | 'seconds':6*(60*60), 67 | 'group_intervals': [0,6,12,18] 68 | }, 69 | '8H' : { 70 | 'upcycle': '4H', 71 | 'seconds':8*(60*60), 72 | 'group_intervals': [0,8,16] 73 | }, 74 | '12H' : { 75 | 'upcycle': '6H', 76 | 'seconds':12*(60*60), 77 | 'group_intervals': [0,12] 78 | }, 79 | '1D' : { 80 | 'upcycle': '12H', 81 | 'seconds':24*(60*60) 82 | }, 83 | 'W-MON': { 84 | 'upcycle': '1D', 85 | 'seconds':7*(24*(60*60)) 86 | }, 87 | 'MS' : { 88 | 'upcycle': '1D', 89 | } 90 | } 91 | 92 | def __init__(self): 93 | pass 94 | 95 | def increment_timeperiods(self, orig_timestamp, num=1, resolution=None): 96 | 97 | if resolution == None: 98 | resolution = '1T' 99 | 100 | if resolution == 'MS': 101 | output = orig_timestamp + relativedelta(months=num) 102 | return output 103 | else: 104 | output = orig_timestamp + relativedelta(seconds=(self.timeframes[resolution]['seconds']*num)) 105 | return output 106 | 107 | 108 | def now(self): 109 | return DateUtils.now_utc().replace(second=0, microsecond=0) 110 | 111 | def get_timeperiods(self, now=None, resolution=None): 112 | 113 | periods = {} 114 | 115 | if now == None: 116 | now = self.now() 117 | 118 | year = now.year 119 | month = now.month 120 | day = now.day 121 | hour = now.hour 122 | minute = now.minute 123 | 124 | 125 | periods['1T'] = now 126 | 127 | # 5T, 15T, 30T 128 | bins = ['5T','15T','30T'] 129 | for timeperiod in bins: 130 | 131 | p = str(year)+"-"+str(month).zfill(2)+"-"+str(day).zfill(2)+" "+str(hour).zfill(2)+":00" 132 | 133 | for i in Timeperiods.timeframes[ timeperiod ]['group_intervals']: 134 | if i <= minute: 135 | p = str(year)+"-"+str(month).zfill(2)+"-"+str(day).zfill(2)+" "+str(hour).zfill(2)+":"+str(i).zfill(2) 136 | 137 | periods[timeperiod] = datetime.strptime(p, '%Y-%m-%d %H:%M') 138 | 139 | 140 | # 1H 141 | p = str(year)+"-"+str(month).zfill(2)+"-"+str(day).zfill(2)+" "+str(hour).zfill(2)+":00" 142 | p = datetime.strptime(p, '%Y-%m-%d %H:%M') 143 | periods['1H'] = p 144 | 145 | # 2H, 3H, 4H, 6H, 8H, 12H 146 | bins = ['2H','3H','4H','6H','8H','12H'] 147 | for timeperiod in bins: 148 | 149 | p = str(year)+"-"+str(month).zfill(2)+"-"+str(day).zfill(2)+" 00:00" 150 | for i in Timeperiods.timeframes[ timeperiod ]['group_intervals']: 151 | if i <= hour: 152 | p = str(year)+"-"+str(month).zfill(2)+"-"+str(day).zfill(2)+" "+str(i).zfill(2)+":00" 153 | 154 | periods[timeperiod] = datetime.strptime(p, '%Y-%m-%d %H:%M') 155 | 156 | 157 | # 1D 158 | p = str(year)+"-"+str(month).zfill(2)+"-"+str(day).zfill(2) 159 | p = datetime.strptime(p, '%Y-%m-%d').replace(hour=0, minute=0) 160 | periods['1D'] = p 161 | 162 | # W-MON 163 | to_beggining_of_week = datetime_timedelta(days=now.weekday()) 164 | p = (now - to_beggining_of_week).replace(hour=0, minute=0) 165 | periods['W-MON'] = p 166 | 167 | # M 168 | p = datetime.strptime(str(year)+"-"+str(month).zfill(2)+"-01", '%Y-%m-%d') 169 | #next_month = p.replace(day=28) + datetime_timedelta(days=4) 170 | #p = next_month - datetime_timedelta(days=next_month.day) 171 | periods['MS'] = p 172 | 173 | 174 | if resolution != None: 175 | return periods[resolution] 176 | 177 | else: 178 | return periods 179 | 180 | 181 | 182 | ############################################################ 183 | class BFX( Timeperiods ): 184 | 185 | api_limit_seconds = 5 186 | resolution_to_api = { 187 | '1T': '1m', 188 | '5T': '5m', 189 | '15T': '15m', 190 | '30T': '30m', 191 | '1H': '1h', 192 | '2H': '1h', 193 | '3H': '3h', 194 | '4H': '1h', 195 | '6H': '6h', 196 | '12H': '12h', 197 | '1D': '1D', 198 | '2D': '1D', 199 | 'W-MON': '1D' 200 | } 201 | api_timeframe_resample_mapping = { 202 | '2H' : '1H', 203 | '4H' : '1H', 204 | '2D' : '1D', 205 | 'W-MON': '1D' 206 | } 207 | 208 | def datetime_to_miliseconds(self, inputdate=None): 209 | if inputdate == None: 210 | inputdate = datetime.utcnow() 211 | return (inputdate - datetime.utcfromtimestamp(0)).total_seconds() * 1000 212 | 213 | 214 | 215 | def api_request_candles(self, resolution, ticker, start_date=None, end_date=None, limit=200): 216 | 217 | 218 | resolution_api = resolution 219 | resolution_increment = resolution 220 | 221 | 222 | if resolution in self.api_timeframe_resample_mapping: 223 | 224 | resolution_increment = self.api_timeframe_resample_mapping[ resolution ] 225 | 226 | logger.info('Getting optimal subperiod, '+str(resolution_increment)+' end date.') 227 | 228 | next_timeperiod = self.increment_timeperiods( end_date, 1 ) 229 | optimal_subperiod = self.increment_timeperiods( end_date, 1, resolution_increment ) 230 | while optimal_subperiod < next_timeperiod: 231 | if optimal_subperiod < self.now(): 232 | end_date = optimal_subperiod 233 | 234 | optimal_subperiod = self.increment_timeperiods( optimal_subperiod, 1, resolution_increment ) 235 | 236 | if start_date == None: 237 | start_date = self.get_timeperiods(resolution=resolution) 238 | start_date = self.increment_timeperiods( start_date, -limit ) 239 | limit = 200 240 | 241 | if end_date == None: 242 | end_date = self.get_timeperiods(resolution=resolution) 243 | 244 | start_date_ms = self.datetime_to_miliseconds(start_date) 245 | end_date_ms = self.datetime_to_miliseconds(end_date) 246 | 247 | logger.info( 'Api request candles from '+str(start_date)+' to '+str(end_date) ) 248 | 249 | # Base URL to be getting the candlestick data from 250 | base_url = 'https://api.bitfinex.com/v2/candles/trade:'+resolution_api+':t'+ticker+'/hist?limit='+str(limit) 251 | 252 | 253 | if start_date != None: 254 | api_query_url = base_url+'&start='+str(start_date_ms)+'&sort=1' 255 | 256 | candles = self.api_request( api_query_url, skip_slow=True ) 257 | cut_off = len(candles) 258 | 259 | for i,c in enumerate(candles): 260 | 261 | if c[0] > end_date_ms: 262 | cut_off = i 263 | break 264 | 265 | candles = candles[0:cut_off] 266 | 267 | logger.info( 'First candle returned: '+str( datetime.utcfromtimestamp(candles[0][0]/1000.0) )+', '+str(candles[0]) ) 268 | logger.info( 'Last candle returned: '+str( datetime.utcfromtimestamp(candles[len(candles)-1][0]/1000.0) )+', '+str(candles[len(candles)-1]) ) 269 | 270 | last_date_ms = candles[len(candles)-1][0] 271 | 272 | while last_date_ms < end_date_ms: 273 | 274 | api_query_url = base_url+'&start='+str(last_date_ms)+'&sort=1' 275 | api_candles = self.api_request( api_query_url ) 276 | cut_off = len(api_candles)-1 277 | 278 | for i, c in enumerate(api_candles): 279 | if c[0] > end_date_ms: 280 | cut_off = i 281 | break 282 | 283 | candles = candles[0:len(candles)-1] + api_candles[0:cut_off+1] 284 | 285 | logger.info('First candle returned: '+str( datetime.utcfromtimestamp(api_candles[0][0]/1000.0) )+', '+str(api_candles[0]) ) 286 | logger.info( 'Last candle used: '+str( datetime.utcfromtimestamp(api_candles[cut_off][0]/1000.0) )+', '+str(api_candles[cut_off]) ) 287 | 288 | last_date_ms = candles[len(candles)-1][0] 289 | 290 | 291 | output = [] 292 | 293 | candle_len = len(candles) 294 | 295 | for i,candle in enumerate(candles): 296 | 297 | c = self.candles_map_api_to_list( candle ) 298 | output.append(c) 299 | 300 | expected_nxt_ms = int( self.datetime_to_miliseconds( self.increment_timeperiods( c['timestamp'], 1, resolution_increment ) ) ) 301 | 302 | if i < candle_len-1: 303 | 304 | if candles[i+1][0] != expected_nxt_ms: 305 | 306 | logger.warning('Api returned missing candles, at '+str(candle[0])+', expected '+str(expected_nxt_ms)+', got '+str(candles[i+1][0]) ) 307 | logger.warning('at: '+str( datetime.utcfromtimestamp(candle[0]/1000.0) )) 308 | logger.warning('expected_nxt_ms: '+str( datetime.utcfromtimestamp(expected_nxt_ms/1000.0) )) 309 | logger.warning('got: '+str( datetime.utcfromtimestamp(candles[i+1][0]/1000.0) )) 310 | 311 | done = False 312 | while done == False: 313 | 314 | fake_candle = [ expected_nxt_ms, candle[2], candle[2], candle[2], candle[2], 0 ] 315 | logger.warning('Caution: Adding fake candle '+str( datetime.utcfromtimestamp(expected_nxt_ms/1000.0) )+' '+str(fake_candle)) 316 | c = self.candles_map_api_to_list( fake_candle ) 317 | output.append(c) 318 | 319 | expected_nxt_ms = expected_nxt_ms + (self.timeframes[resolution_increment]['seconds']*1000) 320 | if expected_nxt_ms == candles[i+1][0]: 321 | done = True 322 | elif expected_nxt_ms < candles[i+1][0]: 323 | continue 324 | elif expected_nxt_ms > candles[i+1][0]: 325 | error('The expected next timestamp: '+str(expected_nxt_ms)+' is greater than the next timestamp: '+str(candles[i+1][0])) 326 | exit() 327 | 328 | 329 | if resolution in self.api_timeframe_resample_mapping: 330 | df = self.list_to_df(output).resample(resolution, closed='left', label='left').agg(self.resample_aggregation) 331 | 332 | output = [] 333 | for index, row in df.iterrows(): 334 | output.append({ 335 | 'timestamp': index.to_pydatetime(), 336 | 'open': row['open'], 337 | 'high': row['high'], 338 | 'low': row['low'], 339 | 'close': row['close'], 340 | 'volume': row['volume'] 341 | }) 342 | 343 | return output 344 | 345 | 346 | def api_request(self, url, skip_slow=False): 347 | 348 | if (self.api_limit_seconds > 0) and (skip_slow == False): 349 | 350 | logger.info('Slow api mode, sleeping for '+str(self.api_limit_seconds)+' seconds') 351 | time.sleep(self.api_limit_seconds) 352 | 353 | logger.info( 'Requesting: '+url ) 354 | 355 | response = requests_get(url).text 356 | data = json_loads(response) 357 | 358 | # Check we actually got the data back 359 | # Not just an api error 360 | completed = 0 361 | while completed == 0: 362 | 363 | if isinstance(data, list): 364 | 365 | if str(data[0]) == 'error': 366 | 367 | # Log the error 368 | lmsg = 'BFX API Error: '+str(data) 369 | error(lmsg) 370 | 371 | # give it a bit of time 372 | time.sleep(15) 373 | 374 | # Re-request the data 375 | response = requests_get(url).text 376 | 377 | # Load the response 378 | data = json_loads(response) 379 | 380 | ## Re-run 381 | 382 | else: 383 | # The response json does not contain error 384 | # Therefore we have the response we wanted 385 | # So api request actuall completed successfully 386 | completed = 1 387 | 388 | elif isinstance(data, dict): 389 | 390 | if 'error' in data: 391 | 392 | # Log the error 393 | lmsg = 'BFX API Error: '+str(data) 394 | error(lmsg) 395 | 396 | # give it a bit of time 397 | error('Sleeping for '+str(config.bfx_api_rate_limit_delay)+' seconds') 398 | time.sleep(config.bfx_api_rate_limit_delay) 399 | 400 | # Re-request the data 401 | response = requests_get(url).text 402 | 403 | # Load the response 404 | data = json_loads(response) 405 | 406 | 407 | else: 408 | # WTF has the api returned then ? 409 | lmsg = 'API error' 410 | error(data) 411 | error(lmsg) 412 | exit() 413 | 414 | 415 | 416 | return data 417 | 418 | 419 | def candles_map_api_to_list(self, row, offset=True): 420 | 421 | ts = row[0] 422 | 423 | return { 424 | 'timestamp': datetime.utcfromtimestamp(ts/1000.0), 425 | 'open': row[1], 426 | 'high': row[3], 427 | 'low': row[4], 428 | 'close': row[2], 429 | 'volume': row[5] 430 | } -------------------------------------------------------------------------------- /bitmex-wallet-parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.6 2 | 3 | # logger module 4 | import coloredlogs, logging 5 | logger = logging.getLogger(__name__) 6 | coloredlogs.install(level='DEBUG', logger=logger) 7 | 8 | # Finding files 9 | import glob 10 | 11 | # Exit 12 | from sys import exit 13 | 14 | # Datetime 15 | from datetime import datetime, timedelta as datetime_timedelta 16 | import dateutil.parser 17 | 18 | 19 | # dataframe 20 | import pandas as pd 21 | 22 | # Pretty output 23 | from pprint import pprint 24 | 25 | # Matplotlib for charts 26 | import matplotlib 27 | matplotlib.use('Agg') 28 | import matplotlib.pyplot as plt 29 | import matplotlib.dates as mdates 30 | from matplotlib.patches import Circle 31 | import matplotlib.patches as mpatches 32 | from matplotlib.ticker import ScalarFormatter 33 | 34 | # # Loading/unloading json (for api requests) 35 | from json import dumps as json_dumps, loads as json_loads 36 | 37 | # Import basic bfx library 38 | from bfx import BFX 39 | 40 | # Argparser 41 | import argparse 42 | 43 | 44 | ############################################################ 45 | 46 | 47 | 48 | # Args 49 | parser = argparse.ArgumentParser() 50 | parser.add_argument('-s', '--start_datetime', help='Start date', default=None, type=(lambda s: datetime.strptime(s, '%Y-%m-%d %H:%M')) ) 51 | parser.add_argument('-e', '--end_datetime', help='End date', default=None, type=(lambda s: datetime.strptime(s, '%Y-%m-%d %H:%M')) ) 52 | parser.add_argument('--show-wallet', action='store_true', help='hide wallet / transaction date') 53 | parser.add_argument('--show-affiliate', action='store_true', help='hide affiliate data') 54 | parser.add_argument('--hide-trading', action='store_true', help='hide trading data') 55 | parser.add_argument('--showmoney', action='store_true', help='hide numerical values') 56 | parser.add_argument('--with-api', action='store_true', help='Fetch historical trading data via bitmex api') 57 | 58 | args = parser.parse_args() 59 | start_datetime = args.start_datetime 60 | end_datetime = args.end_datetime 61 | 62 | total_plots = 1 63 | 64 | if args.show_wallet == True: 65 | total_plots += 1 66 | if args.show_affiliate == True: 67 | total_plots += 1 68 | if args.hide_trading == True: 69 | total_plots -= 1 70 | 71 | if total_plots == 0: 72 | logger.warning('All available plots set to hidden. nothing to show') 73 | exit() 74 | 75 | 76 | ############################################################ 77 | 78 | 79 | if args.with_api == True: 80 | 81 | # pip install bitmex - https://github.com/BitMEX/api-connectors/tree/master/official-http/python-swaggerpy 82 | import bitmex 83 | import csv 84 | from pathlib import Path 85 | 86 | bitmex_api_key = None 87 | bitmex_api_secret = None 88 | 89 | 90 | confg_file = Path("config.json") 91 | if confg_file.is_file(): 92 | with open(confg_file, 'r') as file: 93 | data = file.read() 94 | obj = json_loads(data) 95 | 96 | if 'bitmex_api_key' not in obj: 97 | logger.critical('config.json missing bitmex_api_key') 98 | exit() 99 | 100 | elif 'bitmex_api_secret' not in obj: 101 | logger.critical('config.json missing bitmex_api_secret') 102 | exit() 103 | 104 | else: 105 | bitmex_api_key = obj['bitmex_api_key'] 106 | bitmex_api_secret = obj['bitmex_api_secret'] 107 | 108 | 109 | if (bitmex_api_key == None) or (bitmex_api_secret == None): 110 | 111 | print('\033[95m',"-------------------------------", '\033[0m') 112 | print('\033[92m','--use-api is set to True \n','\033[0m') 113 | print('You do not have your bitmex api keys stored in config.json. You can enter them here.') 114 | print('If you do not have an api key one can be obtained from: https://www.bitmex.com/app/apiKeys \n') 115 | print('These keys will not be saved to config.json, you must do that manually as saving api keys in plain text is an important security consideration.\n') 116 | 117 | bitmex_api_key = input("Enter you Bitmex API Key: ") 118 | bitmex_api_secret = input("Enter you Bitmex API Secret: ") 119 | 120 | now = datetime.now().replace(second=0, microsecond=0) 121 | wallet_file = "Wallet History "+now.strftime('%Y-%m-%d %H-%M')+".csv" 122 | 123 | client = bitmex.bitmex(test=False, api_key=bitmex_api_key,api_secret=bitmex_api_secret) 124 | data = client.User.User_getWalletHistory(count=5000000).result() 125 | 126 | fieldnames = [ 127 | 'transactTime', 128 | 'transactType', 129 | 'amount', 130 | 'fee', 131 | 'address', 132 | 'transactStatus', 133 | 'walletBalance' 134 | ] 135 | with open(wallet_file, "w", newline='') as f: 136 | writer = csv.DictWriter(f, fieldnames=fieldnames) 137 | writer.writerow({ 138 | 'transactTime' : 'transactTime', 139 | 'transactType' : 'transactType', 140 | 'amount' : 'amount', 141 | 'fee' : 'fee', 142 | 'address' : 'address', 143 | 'transactStatus': 'transactStatus', 144 | 'walletBalance' : 'walletBalance' 145 | }) 146 | for x in data[0]: 147 | t = {} 148 | 149 | for field in fieldnames: 150 | 151 | if field == 'transactTime': 152 | if x[field] is not None: 153 | 154 | t[field] = x[field].strftime('%Y-%m-%d %H:%M:%S.%f') 155 | else: 156 | t[field] = x[field] 157 | 158 | logger.info('Writing row: '+str(t)) 159 | writer.writerow(t) 160 | 161 | 162 | 163 | # Get the list of availabble bitmex wallet files 164 | files = sorted(glob.glob('Wallet*')) 165 | 166 | if len(files) == 0: 167 | logger.critical('No valid bitmex wallet files found.') 168 | exit() 169 | 170 | # Get the latest most up to date wallet file 171 | wallet_file = files[len(files)-1] 172 | 173 | ############################################################ 174 | 175 | # Parse wallet file to dataframe 176 | df = pd.read_csv(wallet_file, infer_datetime_format=True) 177 | 178 | # Remove cancelled transactions 179 | # Remnove current unrealised P&L 180 | mask = (df['transactStatus'] != 'Canceled') & (df['transactType'] != 'UnrealisedPNL') 181 | df = df[mask] 182 | 183 | # transactiontime to datetime 184 | # df['transactTime'] = df['transactTime'].apply(dateutil.parser.parse) 185 | df['transactTime'] = pd.to_datetime(df['transactTime'], dayfirst=True) 186 | 187 | # Set it to be the index 188 | df.set_index(df['transactTime'], inplace=True) 189 | 190 | # Sort the df by that index 191 | df.sort_index(inplace=True) 192 | 193 | 194 | 195 | # Convert dates to num for matplotlib 196 | df['mpldate'] = df['transactTime'].map(mdates.date2num) 197 | 198 | # Delete the old transactTime column 199 | del df['transactTime'] 200 | 201 | df = df[start_datetime:end_datetime] 202 | 203 | 204 | 205 | ############################################################ 206 | 207 | 208 | 209 | """ 210 | Get some bitcoin price data 211 | """ 212 | finex = BFX() 213 | 214 | if start_datetime == None: 215 | start_datetime = pd.to_datetime(df.index.values[0]).to_pydatetime() 216 | 217 | now = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) 218 | 219 | if (end_datetime == None) or (end_datetime > now): 220 | end_datetime = now 221 | 222 | logger.info('Querying candles between: '+str(start_datetime)+' and '+str(end_datetime)) 223 | 224 | candles = finex.api_request_candles( '1D', 'BTCUSD', start_datetime, end_datetime ) 225 | 226 | 227 | # Format into a nice df 228 | candle_df = pd.DataFrame.from_dict( candles ) 229 | candle_df.set_index(candle_df['timestamp'], inplace=True) 230 | candle_df.drop(['timestamp'], axis=1, inplace=True) 231 | candle_df.sort_index(inplace=True) 232 | candle_df = candle_df[~candle_df.index.duplicated(keep='last')] 233 | candle_df = candle_df[start_datetime:end_datetime] 234 | 235 | 236 | ############################################################ 237 | 238 | 239 | 240 | # Begin plot 241 | fig = plt.figure(facecolor='white', figsize=(15, 11), dpi=100) 242 | fig.suptitle('Bitmex Account History & Performance') 243 | 244 | on_plot = 1 245 | 246 | """ 247 | Wallet / Transaction History 248 | """ 249 | if args.show_wallet == True: 250 | 251 | ax1 = plt.subplot(total_plots,1,on_plot) 252 | on_plot += 1 253 | 254 | ax1.set_title('Transaction History') 255 | 256 | # If flagged private, hide btc values 257 | if args.showmoney == True: 258 | ax1.set_ylabel('BTC') 259 | else: 260 | ax1.get_yaxis().set_ticks([]) 261 | 262 | ax1.plot( df.index.values, df['amount'].values.cumsum()/100000000, color='b') 263 | ax1.fmt_xdata = mdates.DateFormatter('%d/%m/%Y') 264 | 265 | 266 | # Add bitcoin price 267 | ax11 = ax1.twinx() 268 | # ax11.set_yscale('log') 269 | ax11.fill_between(candle_df.index.values, candle_df['low'].min(), candle_df['close'].values, facecolor='blue', alpha=0.2) 270 | ax11.yaxis.set_major_formatter(ScalarFormatter()) 271 | 272 | 273 | """ 274 | Notcable changes to the wallet balance 275 | - Annotations 276 | - ... maybe for later. 277 | """ 278 | # std = df['walletBalance'].std(ddof=1)/2 279 | # mask = abs(df['walletBalance'] - df['walletBalance'].shift(1)) > std 280 | # tmpdf = df[mask] 281 | 282 | # for index, row in tmpdf.iterrows(): 283 | 284 | # y = df.loc[index]['walletBalance']/100000000 285 | # ax1.annotate( s=row['transactType'], xy=(index, y), arrowprops=dict(facecolor='black', shrink=0.05) ) 286 | 287 | 288 | 289 | """ 290 | Affiliate Income 291 | """ 292 | if args.show_affiliate == True: 293 | 294 | ax2 = plt.subplot(total_plots,1,on_plot) 295 | on_plot += 1 296 | 297 | ax2.set_title('Affiliate income') 298 | 299 | # If flagged private, hide btc values 300 | if args.showmoney == True: 301 | ax2.set_ylabel('BTC') 302 | else: 303 | ax2.get_yaxis().set_ticks([]) 304 | 305 | mask = (df['transactType'] == 'AffiliatePayout') 306 | ax2.plot( df[mask].index.values, df[mask]['amount'].values.cumsum()/100000000, color='b') 307 | ax2.fmt_xdata = mdates.DateFormatter('%d/%m/%Y') 308 | 309 | 310 | # Add bitcoin price 311 | ax22 = ax2.twinx() 312 | # ax22.set_yscale('log') 313 | ax22.fill_between(candle_df.index.values, candle_df['low'].min(), candle_df['close'].values, facecolor='blue', alpha=0.2) 314 | ax22.yaxis.set_major_formatter(ScalarFormatter()) 315 | 316 | 317 | """ 318 | Trading Returns 319 | """ 320 | if args.hide_trading != True: 321 | 322 | ax3 = plt.subplot(total_plots,1,on_plot) 323 | on_plot += 1 324 | 325 | ax3.set_title('Trading Returns') 326 | 327 | # If flagged private, hide btc values 328 | if args.showmoney== True: 329 | ax3.set_ylabel('BTC') 330 | else: 331 | ax3.get_yaxis().set_ticks([]) 332 | 333 | mask = (df['transactType'] == ('RealisedPNL' or 'CashRebalance')) 334 | line = ax3.plot( df[mask].index.values, df[mask]['amount'].values.cumsum()/100000000, color='red', label='Performance') 335 | ax3.fmt_xdata = mdates.DateFormatter('%d/%m/%Y') 336 | 337 | ax3.get_xaxis().set_visible(True) 338 | 339 | start = df[mask].index[0] 340 | candle_df = candle_df[start:] 341 | 342 | # Add bitcoin price 343 | ax33 = ax3.twinx() 344 | # ax33.set_yscale('log') 345 | area = ax33.fill_between(candle_df.index.values, candle_df['low'].min(), candle_df['close'].values, facecolor='blue', alpha=0.2, label='BTCUSD Price') 346 | ax33.yaxis.set_major_formatter(ScalarFormatter()) 347 | 348 | red_patch = mpatches.Patch(color='red', label='Trading Returns') 349 | blue_patch = mpatches.Patch(color='blue', label='BTCUSD Price') 350 | ax3.legend(handles=[red_patch, blue_patch]) 351 | 352 | 353 | ############################################################ 354 | 355 | 356 | 357 | # Format dates 358 | fig.autofmt_xdate() 359 | 360 | # Save figure 361 | saved_plot_filename = datetime.today().strftime('%Y-%m-%d-%H-%M-%S')+'.png' 362 | plt.savefig(saved_plot_filename, bbox_inches='tight') 363 | 364 | 365 | 366 | ############################################################ 367 | 368 | 369 | print('\033[95m',"-------------------------------", '\033[0m') 370 | print('\033[92m','Using Wallet : '+wallet_file,'\033[0m') 371 | print('\033[95m', "-------------------------------", '\033[0m') 372 | print('\033[93m', 'Candle df contents', '\033[0m') 373 | print(candle_df.head(4)) 374 | print(candle_df.tail(4)) 375 | print('\033[95m', "-------------------------------", '\033[0m') 376 | print('\033[93m', 'Wallet df contents', '\033[0m') 377 | print(df.head(4)) 378 | print(df.tail(4)) 379 | print('\033[95m', "-------------------------------", '\033[0m') 380 | print('\033[96m','Saved chart to: '+saved_plot_filename+'\n', '\033[0m') 381 | -------------------------------------------------------------------------------- /config.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "bitmex_api_key": "", 3 | "bitmex_api_secret": "" 4 | } -------------------------------------------------------------------------------- /deribit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.6 2 | 3 | # logger module 4 | import matplotlib 5 | import pandas as pd 6 | from sys import exit 7 | import openapi_client as deribit 8 | import csv 9 | from bfx import BFX 10 | from json import dumps as json_dumps, loads as json_loads 11 | from matplotlib.ticker import ScalarFormatter 12 | import matplotlib.patches as mpatches 13 | from matplotlib.patches import Circle 14 | import matplotlib.dates as mdates 15 | import matplotlib.pyplot as plt 16 | import dateutil.parser 17 | from datetime import datetime, timedelta as datetime_timedelta 18 | import glob 19 | import coloredlogs 20 | import logging 21 | logger = logging.getLogger(__name__) 22 | coloredlogs.install(level='DEBUG', logger=logger) 23 | 24 | # Finding files 25 | 26 | # Exit 27 | 28 | # Datetime 29 | 30 | 31 | # dataframe 32 | 33 | # Matplotlib for charts 34 | #matplotlib.use('Agg') 35 | 36 | # # Loading/unloading json (for api requests) 37 | 38 | # Import basic bfx library 39 | 40 | # deribit 41 | # https://github.com/deribit/deribit-api-clients/tree/master/python 42 | 43 | 44 | wallet_file = "Wallet_deribit_all.csv" 45 | MAX_COUNT = 1000 46 | 47 | #subbaccount 48 | KEY = "" 49 | SECRET = "" 50 | 51 | SHOW_MONEY = False 52 | 53 | # keep 54 | start_datetime = None 55 | end_datetime = None 56 | total_plots = 1 57 | 58 | 59 | def createWalletDeribit(): 60 | # Setup configuration instance 61 | conf = deribit.configuration.Configuration() 62 | # Setup unauthenticated client 63 | client = deribit.api_client.ApiClient(conf) 64 | publicApi = deribit.PublicApi(client) 65 | # Authenticate with API credentials 66 | response = publicApi.public_auth_get( 67 | 'client_credentials', '', '', KEY, SECRET, '', '', '', scope='session:test wallet:read') 68 | access_token = response['result']['access_token'] 69 | 70 | conf_authed = deribit.configuration.Configuration() 71 | conf_authed.access_token = access_token 72 | # Use retrieved authentication token to setup private endpoint client 73 | client_authed = deribit.api_client.ApiClient(conf_authed) 74 | privateApi = deribit.PrivateApi(client_authed) 75 | 76 | response = privateApi.private_get_settlement_history_by_currency_get( 77 | "BTC", count=MAX_COUNT) 78 | data = response['result'] 79 | fieldnames = [ 80 | 'transactTime', 81 | 'amount', 82 | 'transactType', 83 | 'transactStatus' 84 | ] 85 | with open(wallet_file, "w", newline='') as f: 86 | writer = csv.DictWriter(f, fieldnames=fieldnames) 87 | writer.writerow({ 88 | 'transactTime': 'transactTime', 89 | 'amount': 'amount', 90 | 'transactType': 'transactType', 91 | 'transactStatus': 'transactStatus' 92 | }) 93 | for x in data['settlements']: 94 | t = {} 95 | t['transactTime'], t['amount'], t['transactType'], t['transactStatus'] = datetime.fromtimestamp(x['timestamp']/1000), x['profit_loss'], 'RealisedPNL', 'DONE' 96 | #print(t) 97 | writer.writerow(t) 98 | 99 | 100 | createWalletDeribit() 101 | 102 | # Get the list of availabble bitmex wallet files 103 | files = sorted(glob.glob('Wallet*')) 104 | 105 | if len(files) == 0: 106 | logger.critical('No valid deribit wallet files found.') 107 | exit() 108 | 109 | # Get the latest most up to date wallet file 110 | wallet_file = files[len(files)-1] 111 | 112 | ############################################################ 113 | 114 | # Parse wallet file to dataframe 115 | df = pd.read_csv(wallet_file, infer_datetime_format=True) 116 | 117 | # Remove cancelled transactions 118 | # Remnove current unrealised P&L 119 | mask = (df['transactStatus'] != 'Canceled') & ( 120 | df['transactType'] != 'UnrealisedPNL') 121 | df = df[mask] 122 | 123 | # transactiontime to datetime 124 | # df['transactTime'] = df['transactTime'].apply(dateutil.parser.parse) 125 | df['transactTime'] = pd.to_datetime(df['transactTime'], dayfirst=True) 126 | 127 | 128 | # Set it to be the index 129 | df.set_index(df['transactTime'], inplace=True) 130 | 131 | # Sort the df by that index 132 | df.sort_index(inplace=True) 133 | 134 | # Convert dates to num for matplotlib 135 | df['mpldate'] = df['transactTime'].map(mdates.date2num) 136 | 137 | # Delete the old transactTime column 138 | del df['transactTime'] 139 | 140 | df = df[start_datetime:end_datetime] 141 | 142 | 143 | ############################################################ 144 | 145 | 146 | """ 147 | Get some bitcoin price data 148 | """ 149 | finex = BFX() 150 | 151 | if start_datetime == None: 152 | start_datetime = pd.to_datetime(df.index.values[0]).to_pydatetime() 153 | 154 | now = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) 155 | 156 | if (end_datetime == None) or (end_datetime > now): 157 | end_datetime = now 158 | 159 | logger.info('Querying candles between: ' + 160 | str(start_datetime)+' and '+str(end_datetime)) 161 | 162 | candles = finex.api_request_candles( 163 | '1D', 'BTCUSD', start_datetime, end_datetime) 164 | 165 | 166 | # Format into a nice df 167 | candle_df = pd.DataFrame.from_dict(candles) 168 | candle_df.set_index(candle_df['timestamp'], inplace=True) 169 | candle_df.drop(['timestamp'], axis=1, inplace=True) 170 | candle_df.sort_index(inplace=True) 171 | candle_df = candle_df[~candle_df.index.duplicated(keep='last')] 172 | candle_df = candle_df[start_datetime:end_datetime] 173 | 174 | 175 | ############################################################ 176 | 177 | 178 | # Begin plot 179 | fig = plt.figure(facecolor='white', figsize=(15, 11), dpi=100) 180 | fig.suptitle('Deribit Account History & Performance') 181 | 182 | on_plot = 1 183 | if 1 == True: 184 | ax3 = plt.subplot(total_plots, 1, on_plot) 185 | on_plot += 1 186 | 187 | ax3.set_title('Trading Returns') 188 | 189 | if SHOW_MONEY: 190 | ax3.set_ylabel('BTC') 191 | else: 192 | ax3.get_yaxis().set_ticks([]) 193 | 194 | mask = (df['transactType'] == ('RealisedPNL' or 'CashRebalance')) 195 | line = ax3.plot(df[mask].index.values, df[mask]['amount'].values.cumsum(), color='red', label='Performance') 196 | ax3.fmt_xdata = mdates.DateFormatter('%d/%m/%Y') 197 | 198 | ax3.get_xaxis().set_visible(True) 199 | 200 | start = df[mask].index[0] 201 | candle_df = candle_df[start:] 202 | 203 | # Add bitcoin price 204 | ax33 = ax3.twinx() 205 | # ax33.set_yscale('log') 206 | area = ax33.fill_between(candle_df.index.values, candle_df['low'].min(), candle_df['close'].values, facecolor='blue', alpha=0.2, label='BTCUSD Price') 207 | ax33.yaxis.set_major_formatter(ScalarFormatter()) 208 | 209 | red_patch = mpatches.Patch(color='red', label='Trading Returns') 210 | blue_patch = mpatches.Patch(color='blue', label='BTCUSD Price') 211 | ax3.legend(handles=[red_patch, blue_patch]) 212 | 213 | ############################################################ 214 | 215 | 216 | 217 | # Format dates 218 | fig.autofmt_xdate() 219 | 220 | # Save figure 221 | saved_plot_filename = datetime.today().strftime('%Y-%m-%d-%H-%M-%S')+'.png' 222 | plt.savefig(saved_plot_filename, bbox_inches='tight') 223 | 224 | 225 | 226 | ############################################################ 227 | 228 | 229 | print('\033[95m',"-------------------------------", '\033[0m') 230 | print('\033[92m','Using Wallet : '+wallet_file,'\033[0m') 231 | print('\033[95m', "-------------------------------", '\033[0m') 232 | print('\033[93m', 'Candle df contents', '\033[0m') 233 | print(candle_df.head(4)) 234 | print(candle_df.tail(4)) 235 | print('\033[95m', "-------------------------------", '\033[0m') 236 | print('\033[93m', 'Wallet df contents', '\033[0m') 237 | print(df.head(4)) 238 | print(df.tail(4)) 239 | print('\033[95m', "-------------------------------", '\033[0m') 240 | print('\033[96m','Saved chart to: '+saved_plot_filename+'\n', '\033[0m') -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Bitmex Wallet Parser by [@whalepoolbtc](https://t.me/whalepoolbtc) - https://whalepool.io 2 | 3 | ## About 4 | A simple script to parse over your bitmex wallet history file and give you an overview of your account. 5 | 6 | ### Support 7 | Please use the [bitmex whalepool affiliate link](http://bitmex.whalepool.io) or the [deribit whalepool affiliate link](http://derbit.whalepool.io) to support [@whalepool](https://t.me/whalepoolbtc) 8 | 9 | # Install / Setup Instructions 10 | - Install `python-devel` package on your machine 11 | - Install pip requirements: 12 | - **Linux** `sudo pip install -r requirements.pip` 13 | - **Windows** `py pip install -r requirements.pip` 14 | 15 | ## Use via downloading your bitmex wallet history as .csv file 16 | - Go to [https://www.bitmex.com/app/wallet](https://www.bitmex.com/app/wallet) and click the 'save as CSV' on the top right. - This will download 'Wallet History - YYYY-MM-DD.csv' file. Put this file in the same location as the python script. 17 | 18 | ## Use via setting up api credentials for your bitmex account 19 | You can save you api credentials to config.json if you want. 20 | Warning, saving api details in an unencrypted format to a file could be a security risk. 21 | You don't have to, there is a prompt for the details if you dont have config.json file setup. 22 | 23 | ```bash 24 | # Setup your api credentials in config.json 25 | # api credentials can be generated here: https://www.bitmex.com/app/apiKeys 26 | cp config.sample.json config.json 27 | 28 | # Use the script with the --use-api flag 29 | python bitmex-wallet-parser.py --with-api 30 | ``` 31 | 32 | 33 | # Run some commands 34 | Script will output a chart for each time you run it. Share and enjoy ! 35 | 36 | ```bash 37 | # Simple trading returns, no balances present 38 | python bitmex-wallet-parser.py 39 | # Show balances 40 | python bitmex-wallet-parser.py --showmoney 41 | 42 | # Show your wallet aswell, so u can see deposit/withdrawals from your account 43 | python bitmex-wallet-parser.py --show-wallet 44 | 45 | # Show your affiliate balances 46 | python bitmex-wallet-parser.py --show-affiliate 47 | 48 | # Hide your trading performance 49 | # Maybe just to show affiliate balances and nothing else ? 50 | python bitmex-wallet-parser.py --hide-trading 51 | python bitmex-wallet-parser.py --hide-trading --show-affiliate 52 | python bitmex-wallet-parser.py --hide-trading --show-affiliate --showmoney 53 | ``` 54 | 55 | 56 | ### Arguments 57 | 58 | | Argument | Description | 59 | | -------- | ----------- | 60 | | `-s` | start date, eg '2019-01-01 00:00' | 61 | | `-e` | end date, eg '2019-02-01 00:00' | 62 | | `--show-wallet` | show full wallet balance history including deposit/withdrawals | 63 | | `--show-affiliate` | show affiliate income | 64 | | `--hide-trading` | hide trading data | 65 | | `--showmoney` | add money values to the axis | 66 | | `--with-api` | get bitmex historical data via api | 67 | 68 | 69 | ### Sample output 70 | 71 | ```bash 72 | python bitmex-wallet-parser.py --hide-trading --show-affiliate -s '2018-01-01 00:00' -e '2019-01-01 00:00' 73 | ``` 74 | ![chart example 1](https://i.imgur.com/RAyqesL.png) 75 | 76 | --- 77 | 78 | 79 | ```bash 80 | python bitmex-wallet-parser.py -s '2019-01-01 00:00' -e '2019-06-09 00:00' 81 | ``` 82 | ![chart example 1](https://i.imgur.com/FM73DDz.png) 83 | 84 | --- 85 | 86 | 87 | 88 | # TODO 89 | cache/save/check cache/refresh latest data for trading history / candles 90 | 91 | -------------------------------------------------------------------------------- /requirements.pip: -------------------------------------------------------------------------------- 1 | bitmex 2 | coloredlogs 3 | datetime 4 | pandas 5 | pprint 6 | matplotlib 7 | argparse 8 | requests 9 | --------------------------------------------------------------------------------