├── HistDataDownloaderGUI.py ├── driftvolEstimator.py └── testingDriftAndVar.py /HistDataDownloaderGUI.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import ibapi 4 | import tkinter as tk 5 | from tkinter import ttk 6 | from tkinter import messagebox 7 | 8 | from ibapi.wrapper import EWrapper 9 | from ibapi.client import EClient 10 | 11 | from threading import Thread 12 | 13 | from time import sleep, strftime, time, localtime 14 | 15 | class msgHandler(object): 16 | def handleHistoricalData(self, reqId: int, bar: ibapi.common.BarData): 17 | pass 18 | 19 | def handleHistoricalDataEnd(self, reqId:int, start:str, end:str): 20 | pass 21 | 22 | class IBWrapper(EWrapper): 23 | def __init__(self): 24 | EWrapper.__init__(self) 25 | self.exporter = None 26 | self.contractDetailsObtained = False 27 | 28 | def init_withExporter(self, exporter): 29 | EWrapper.__init__(self) 30 | self.exporter = exporter 31 | self.contractDetailsObtained = False 32 | 33 | def historicalData(self, reqId: int, bar: ibapi.common.BarData): 34 | """ returns the requested historical data bars 35 | 36 | reqId - the request's identifier 37 | date - the bar's date and time (either as a yyyymmss hh:mm:ssformatted 38 | string or as system time according to the request) 39 | open - the bar's open point 40 | high - the bar's high point 41 | low - the bar's low point 42 | close - the bar's closing point 43 | volume - the bar's traded volume if available 44 | count - the number of trades during the bar's timespan (only available 45 | for TRADES). 46 | WAP - the bar's Weighted Average Price 47 | hasGaps -indicates if the data has gaps or not. """ 48 | 49 | if self.exporter == None: 50 | self.logAnswer(ibapi.utils.current_fn_name(), vars()) 51 | return 52 | else: 53 | self.exporter.handleHistoricalData(reqId, bar) 54 | 55 | def historicalDataEnd(self, reqId:int, start:str, end:str): 56 | if self.exporter == None: 57 | self.logAnswer(ibapi.utils.current_fn_name(), vars()) 58 | return 59 | else: 60 | self.exporter.handleHistoricalDataEnd(reqId, start, end) 61 | 62 | def contractDetails(self, reqId:int, contractDetails:ibapi.contract.ContractDetails): 63 | self.resolved_contract = contractDetails.summary 64 | 65 | def contractDetailsEnd(self, reqId:int): 66 | self.contractDetailsObtained = True 67 | 68 | class IBClient(EClient): 69 | 70 | def __init__(self, wrapper): 71 | EClient.__init__(self, wrapper) 72 | 73 | class tApp(IBWrapper, IBClient): 74 | 75 | def __init__(self, IPAddress, PortId, ClientId, msgHandler): 76 | IBWrapper.init_withExporter(self, msgHandler) 77 | IBClient.__init__(self, wrapper = self) 78 | 79 | self.connect(IPAddress, PortId, ClientId) 80 | 81 | thread = Thread(target = self.run, name = "MainThread") 82 | thread.start() 83 | 84 | setattr(self, "_thread", thread) 85 | 86 | class Application(tk.Frame, msgHandler): 87 | 88 | def __init__(self, master): 89 | ttk.Frame.__init__(self, master) 90 | msgHandler.__init__(self) 91 | 92 | self.port = 7496 93 | self.clientID = 25 94 | self.grid() 95 | self.create_widgets() 96 | self.isConnected = False 97 | 98 | def create_widgets(self): 99 | 100 | now = strftime('%Y%m%d %H:%M:%S', localtime(int(time()))) 101 | myfont = ('Arial', 12) 102 | 103 | self.btnConnect = tk.ttk.Button(self, text = 'Connect', command = self.connect_to_tws) 104 | self.btnConnect.grid(row=1, column=1, sticky=tk.W) 105 | 106 | self.btnGetData = ttk.Button(self, text = 'GetData', command = self.getHistData) 107 | self.btnGetData.grid(row=1, column=2, sticky=tk.W) 108 | 109 | self.label_datetime = tk.Label(root, font=myfont, text = 'End Datetime').grid(row=3, column=0) 110 | self.label_duration = tk.Label(root, font=myfont, text = 'Duration').grid(row=3, column=1) 111 | self.label_barsize = tk.Label(root, font=myfont, text = 'BarSize').grid(row=3, column=2) 112 | 113 | varDateTimeEnd.set(now) 114 | 115 | self.cbDateTimeEnd = tk.Entry(root, font=myfont, textvariable = varDateTimeEnd).grid(row=4, column=0) 116 | self.cbDuration = ttk.Combobox(root, font = myfont, textvariable = varDuration) 117 | self.cbDuration['values'] = ('1 Y', '1 M', '6 M', '1 D', '7 D') 118 | self.cbDuration.grid(row=4, column=1) 119 | 120 | self.cbBarSize = ttk.Combobox(root, font=myfont, textvariable = varBarSize) 121 | self.cbBarSize['values'] = ('1 day', '1 min', '2 mins', '5 mins') 122 | self.cbBarSize.grid(row=4, column=2) 123 | 124 | 125 | 126 | self.listbox1 = tk.Listbox(root, font=("",12), width=75, height=30) 127 | self.listbox1.grid(row=6, column=0, columnspan=5, padx=5, pady=5, sticky='w') 128 | 129 | self.msgBox = tk.Listbox(root, font=("",12), width=75, height=10) 130 | self.msgBox.grid(row=7, column=0, columnspan=5, padx=5, pady=5, sticky='w') 131 | 132 | def connect_to_tws(self): 133 | 134 | if self.isConnected: 135 | self.tws_client.disconnect() 136 | self.btnConnect.config(text = 'Connect') 137 | self.msgBox.insert(tk.END, "Disconnected From IB") 138 | self.isConnected = False 139 | else: 140 | self.tws_client = tApp('LocalHost', self.port, self.clientID, self) 141 | timePassed = 0 142 | while not(self.tws_client.isConnected() ): 143 | sleep(0.1) 144 | timePassed += 0.1 145 | if (timePassed >5): 146 | self.msgBox.insert(tk.END, "waited more than 5 secs to establish connection to TWS") 147 | 148 | self.isConnected = True 149 | self.msgBox.insert(tk.END, "Successfully connected to IB") 150 | self.btnConnect.config(text = "Disconnect") 151 | 152 | def getHistData(self): 153 | if not(self.isConnected): 154 | self.msgBox.insert(tk.END, "Not Connected to IB yet") 155 | return 156 | 157 | self.listbox1.delete(0, tk.END) 158 | 159 | self.contract = ibapi.contract.Contract() 160 | self.contract.symbol = "COIL" 161 | self.contract.secType = "FUT" 162 | self.contract.exchange = "IPE" 163 | self.contract.currency = "USD" 164 | self.contract.lastTradeDateOrContractMonth = "201801" 165 | 166 | self.tws_client.reqContractDetails(reqId = 2, contract=self.contract) 167 | self.tws_client.contractDetailsObtained = False 168 | 169 | timePassed = 0 170 | while not(self.tws_client.contractDetailsObtained): 171 | sleep(0.1) 172 | timePassed += 0.1 173 | if (timePassed>10): 174 | self.msgBox.insert(tk.END, "Waited more than 10 secs for contract details request") 175 | 176 | self.msgBox.insert(tk.END, "Successfully obtained contract details") 177 | aContract = self.tws_client.resolved_contract 178 | aContract.includeExpired = True 179 | 180 | now = varDateTimeEnd.get() 181 | duration = varDuration.get() 182 | barsize = varBarSize.get() 183 | 184 | self.tws_client.reqHistoricalData(reqId = 1, 185 | contract = aContract, 186 | endDateTime = now, 187 | durationStr = duration, 188 | barSizeSetting = barsize, 189 | whatToShow = 'TRADES', 190 | useRTH = 1, 191 | formatDate = 1, 192 | keepUpToDate = False, 193 | chartOptions = []) 194 | 195 | def disconnect(self): 196 | if self.isConnected: 197 | self.tws_client.disconnect() 198 | self.isConnected = False 199 | 200 | def handleHistoricalData(self, reqId: int, bar: ibapi.common.BarData): 201 | str_open = str(bar.open) 202 | str_high = str(bar.high) 203 | str_low = str(bar.low) 204 | str_close = str(bar.close) 205 | str_volume = str(bar.volume) 206 | 207 | histData = bar.date + "," + str_open + "," + str_high + "," + str_low + "," + str_close + "," + str_volume 208 | self.listbox1.insert(tk.END, histData) 209 | 210 | def handleHistoricalDataEnd(self, reqId:int, start:str, end:str): 211 | self.msgBox.insert(tk.END, "Finished downloading historical data") 212 | 213 | root = tk.Tk() 214 | 215 | def on_closing(): 216 | if messagebox.askokcancel("Quit", "Do you want to quit?"): 217 | app.disconnect() 218 | root.destroy() 219 | 220 | root.title("Historical Data from IB Python API") 221 | root.geometry("800x1200") 222 | root.attributes("-topmost", True) 223 | root.protocol("WM_DELETE_WINDOW", on_closing) 224 | 225 | varDateTimeEnd = tk.StringVar() 226 | varDuration = tk.StringVar(root, value = "1 M") 227 | varBarSize = tk.StringVar(root, value = "1 day") 228 | 229 | app = Application(root) 230 | 231 | root.mainloop() -------------------------------------------------------------------------------- /driftvolEstimator.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import numpy as np 4 | import math 5 | #from scipy import stats 6 | #from scipy.interpolate import griddata 7 | #import matplotlib.pyplot as plt 8 | #from mpl_toolkits.mplot3d import Axes3D 9 | from scipy.interpolate import RegularGridInterpolator 10 | from scipy import optimize 11 | import pickle 12 | import os 13 | import pandas as pd 14 | import datetime as dt 15 | import time 16 | #import statsmodels.tsa.stattools as tsa 17 | 18 | class densityEstimator: 19 | def __init__(self): 20 | self.interpolators = [] 21 | self.ratioInterpolators = [] 22 | self.drifts = np.zeros(41) 23 | i = 0 24 | for aDrift in np.linspace(0.0,2,41 ): 25 | # print('drift = ' + '{:1.2f}'.format(aDrift)) 26 | # filename ='DensityInterpolators/DensityInterpolatorDrift'+'{:1.0f}'.format(aDrift*100)+'.obj' 27 | filename ='DensityInterpolators/fastKDEInterpolator'+'{:1.0f}'.format(aDrift*100)+'.obj' 28 | # print(filename) 29 | afile = open(filename, 'rb') 30 | aInterp = pickle.load(afile) 31 | self.interpolators.append(aInterp) 32 | self.drifts[i] = aDrift 33 | i += 1 34 | afile.close() 35 | filename = 'DensityInterpolators/cwRatioDensityInterpolatorDrift'+'{:1.0f}'.format(aDrift*100)+'.obj' 36 | bfile = open(filename, 'rb') 37 | aInterp = pickle.load(bfile) 38 | self.ratioInterpolators.append(aInterp) 39 | self.mw = [] 40 | self.mc = [] 41 | self.weights = [] 42 | 43 | def getDensity(self, aW: float, aC: float, drift: float): 44 | mu = drift 45 | myClose = aC 46 | if (drift < 0): 47 | mu = -drift 48 | myClose = -aC 49 | 50 | if aW<0: 51 | return 0.0 52 | if myClose>aW: 53 | return 0.0 54 | 55 | idx = (np.abs(self.drifts-mu)).argmin() 56 | 57 | if mu == self.drifts[idx]: 58 | return self.interpolators[idx](np.array([aW, myClose]).T) 59 | elif mu > self.drifts[idx]: 60 | if idx == self.drifts.size-1: 61 | return self.interpolators[idx](np.array([aW, myClose]).T) 62 | else: 63 | weight = (mu-self.drifts[idx])/(self.drifts[idx+1] - self.drifts[idx]) 64 | return self.interpolators[idx](np.array([aW, myClose]).T)*(1.0-weight)+self.interpolators[idx+1](np.array([aW, myClose]).T)*weight 65 | else: 66 | weight = (self.drifts[idx]-mu)/(self.drifts[idx] - self.drifts[idx-1]) 67 | return self.interpolators[idx](np.array([aW, myClose]).T)*(1.0-weight)+self.interpolators[idx-1](np.array([aW, myClose]).T)*weight 68 | 69 | 70 | def getDensityRawInput(self, w_in : float, c_in: float, drift: float, vol: float): 71 | newDrift = drift/vol 72 | newW = w_in/vol 73 | newC = c_in/vol 74 | # aValue = vol 75 | # print('{:1.5f}'.format(w_in) + ' ' + '{:1.5f}'.format(c_in)) 76 | # print( '{:1.5f}'.format(drift) ) 77 | # print( str(aValue) ) 78 | result = self.getDensity(newW, newC, newDrift) 79 | if result <= 1e-8: 80 | result = 1e-8 81 | return result 82 | 83 | def setWC(self, w_in:list, c_in:list, weights:list = None): 84 | self.mw = w_in 85 | self.mc = c_in 86 | if not(weights): 87 | self.weights = [1.0]*len(self.mw) 88 | else: 89 | self.weights = weights 90 | 91 | 92 | 93 | def evaluateSumLogDensity(self, driftvol): 94 | if not len(self.mw): #check if the list is empty 95 | return None; 96 | 97 | result = 0.0 98 | for aw,ac,weight in zip(self.mw, self.mc, self.weights): 99 | # print(aw) 100 | # print(ac) 101 | # print('W = ''{:1.5f}'.format(aw) +" C = " + '{:1.5f}'.format(ac) + " drift = "+ '{:1.5f}'.format(driftvol[0]) + ' vol = ' + '{:1.5f}'.format(driftvol[1])) 102 | result += math.log(self.getDensityRawInput(aw, ac, driftvol[0], driftvol[1]))*weight 103 | 104 | 105 | return -result #return negative log, for minimizing 106 | 107 | def evaluateSumLogDensity2(self, drift: float, vol: float): 108 | if not len(self.mw): #check if the list is empty 109 | return None; 110 | 111 | result = 0.0 112 | for aw,ac in zip(self.mw, self.mc): 113 | # print(aw) 114 | # print(ac) 115 | # print('W = ''{:1.5f}'.format(aw) +" C = " + '{:1.5f}'.format(ac) + " drift = "+ '{:1.5f}'.format(driftvol[0]) + ' vol = ' + '{:1.5f}'.format(driftvol[1])) 116 | result += math.log(self.getDensityRawInput(aw, ac, drift, vol)) 117 | 118 | 119 | return -result #return negative log, for minimizing 120 | 121 | def evaluateSumLogDensity3(self, vol: float, driftvolratio: float): 122 | return self.evaluateSumLogDensity2(vol*driftvolratio, vol) 123 | 124 | def evaluateSumLogDensityNoDrift(self, vol: float): 125 | if not len(self.mw): #check if the list is empty 126 | return None; 127 | 128 | result = 0.0 129 | for aw,ac in zip(self.mw, self.mc): 130 | # print(aw) 131 | # print(ac) 132 | # print('W = ''{:1.5f}'.format(aw) +" C = " + '{:1.5f}'.format(ac) + " drift = "+ '{:1.5f}'.format(driftvol[0]) + ' vol = ' + '{:1.5f}'.format(driftvol[1])) 133 | result += math.log(self.getDensityRawInput(aw, ac, 0.0, vol)) 134 | 135 | 136 | return -result #return negative log, for minimizing 137 | 138 | def getLikelyDriftAndVol(self, met = 'SLSQP'): 139 | # bnds = ((None,None), (1e-5, None)) 140 | avgW = np.average(self.mw) 141 | avgC = np.average(self.mc) 142 | 143 | guessVol = avgW/1.5 144 | # guessDrift = avgC/guessVol 145 | 146 | # results = optimize.minimize(self.evaluateSumLogDensity, [guessDrift, guessVol], method=met, bounds = bnds) 147 | results = optimize.brute(self.evaluateSumLogDensity, ((-1,1),(0.0001,0.5)), Ns = 50) 148 | 149 | return results 150 | 151 | def getLikelyVolNoDrift(self): 152 | 153 | avgW = np.average(self.mw) 154 | avgC = np.average(self.mc) 155 | 156 | guessVol = avgW/1.5 157 | guessDrift = avgC/guessVol 158 | 159 | # vol = optimize.fmin(self.evaluateSumLogDensityNoDrift, guessVol, xtol = 1e-5, disp = 0) 160 | vol = [0.00055] 161 | drift = optimize.fmin(self.evaluateSumLogDensity2, guessDrift, xtol = 1e-5, disp = 0, args=(vol[0],)) 162 | results = [drift[0],vol[0]] 163 | return results 164 | 165 | def getLikelyRatioDrift(self, driftGuess = -100.0): #actually returning drift/vol ratio, normalized drift 166 | 167 | if driftGuess >-190.0: 168 | avgW = np.average(self.mw) 169 | avgC = np.average(self.mc) 170 | 171 | guessVol = avgW/1.5 172 | guessDrift = avgC/guessVol 173 | else: 174 | guessDrift = driftGuess 175 | 176 | # drift = optimize.fmin(self.evaluateSumLogDensityRatio, guessDrift, xtol = 1e-4, disp = 0) 177 | drift = optimize.fminbound(self.evaluateSumLogDensityRatio, -2.0, 2.0, xtol = 1e-4, disp = 0) 178 | return drift 179 | 180 | def getLikelyRatioDrift2(self): #actually returning drift/vol ratio, normalized drift 181 | ###### 182 | # returning drift value at only the anchor values, i.e 0, 0.05, 0.1, etc. 183 | ###### 184 | idx = 0 185 | i=0 186 | minvalue = self.evaluateSumLogDensityRatio(0.0) 187 | ispositive = True 188 | avalue = 0 189 | for dr in self.drifts: 190 | avalue = self.evaluateSumLogDensityRatio(dr) 191 | if avalueahigh: 336 | ahigh = highs[i] 337 | if lows[i] lotunit: 341 | aw.pop(0) 342 | ac.pop(0) 343 | scale = 1.0/math.sqrt(1.0*totlots/lotunit) 344 | myw = math.log(ahigh/alow)*scale 345 | myc = math.log(aclose/aopen)*scale 346 | aw.append(myw) 347 | ac.append(myc) 348 | count += 1 349 | 350 | if count>=estnum: 351 | myDen.setWC(aw,ac) 352 | # results = myDen.getLikelyDriftAndVol() 353 | # aline = ts[i].strftime("%Y-%m-%d %H:%M:%S") + ',' + '{:1.5f}'.format(results[0]) + ',' + '{:1.5f}'.format(results[1])+ ',' + str(totlots) + '\n' 354 | # results = myDen.getLikelyVolNoDrift() 355 | if (len(drifts)<1): 356 | iniDrift = -100 357 | else: 358 | iniDrift = drifts[-1] 359 | results = myDen.getLikelyRatioDrift(iniDrift) 360 | vol = myDen.getLikelyVolDriftGivenRatio(results) 361 | 362 | drifts.append(results) 363 | aline = ts[i].strftime("%Y-%m-%d %H:%M:%S") + ',' + '{:1.5f}'.format(results)+ ',' + '{:1.5f}'.format(vol)+ ',' + '{:1.5f}'.format(aopen) +',' + '{:1.5f}'.format(ahigh) +',' + '{:1.5f}'.format(alow) +',' + '{:1.5f}'.format(aclose) +',' + str(totlots) + '\n' 364 | output.write(aline) 365 | totlots = 0.0 366 | ahigh = 0.0 367 | alow = 1e5 368 | aclose = 0.0 369 | aopen = -1.0 370 | if (i%100 == 0): 371 | print(str(i)+' out of '+str(totrows)) 372 | print(time.time()-start_time) 373 | 374 | output.close() -------------------------------------------------------------------------------- /testingDriftAndVar.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import numpy as np 4 | import math 5 | from scipy import stats 6 | #from scipy.interpolate import griddata 7 | import matplotlib.pyplot as plt 8 | #from mpl_toolkits.mplot3d import Axes3D 9 | from scipy.interpolate import RegularGridInterpolator 10 | from scipy.interpolate import interp1d 11 | import pickle 12 | #import os 13 | from fastkde import fastKDE 14 | import pylab as PP 15 | 16 | import time 17 | 18 | def getDensityInterpolator(drift, numSim = 1000000, nSteps = 100): 19 | 20 | numSteps = nSteps 21 | 22 | timeStep = 1.0/numSteps 23 | 24 | numSimulations = numSim 25 | 26 | hlc = np.zeros((numSimulations, 3), dtype = float) 27 | 28 | drift = drift 29 | 30 | minDensity = 1.0/numSimulations/10 31 | 32 | for re in hlc: 33 | arr = np.array(np.random.normal(0,math.sqrt(timeStep),numSteps)) 34 | arr = arr+drift*timeStep 35 | myHLC = np.cumsum(arr) 36 | re[0] = np.max(myHLC) 37 | re[1] = np.min(myHLC) 38 | re[2] = myHLC[-1] 39 | if re[0]<0.0: 40 | re[0] = 0 41 | if re[1]>0.0: 42 | re[1] = 0 43 | 44 | #hmax = np.max(hlc[:,0]) #max high 45 | #lmin = np.min(hlc[:,1]) #min low 46 | # cmin = np.min(hlc[:,2]) #min close 47 | # cmax = np.max(hlc[:,2]) #max close 48 | # 49 | # 50 | w = hlc[:,0]-hlc[:,1] 51 | c = hlc[:,2] 52 | # wmax = np.max(w) 53 | # W, C = np.mgrid[0:wmax:128j, cmin:cmax:128j] 54 | 55 | 56 | # positions = np.vstack([W.ravel(), C.ravel()]) 57 | # values = np.vstack([w,c]) 58 | # kernel = stats.gaussian_kde(values) 59 | # di = kernel(np.vstack([W.flatten(), C.flatten()])) 60 | 61 | # fig = plt.figure() 62 | # ax = fig.add_subplot(111) 63 | # ax.pcolormesh(W, C, di.reshape(W.shape)) 64 | #plt.show() 65 | 66 | wgrid = np.linspace(0.0, 6, 101) 67 | cgrid = np.linspace(-6, 6, 201) 68 | 69 | counts, edges = np.histogramdd([w,c], bins=(wgrid, cgrid)) 70 | myden = counts/np.sum(counts) 71 | myden[myden0.0: 122 | re[1] = 0 123 | 124 | # 125 | w = hlc[:,0]-hlc[:,1] 126 | c = hlc[:,2] 127 | 128 | akernel = fastKDE.fastKDE([w, c], \ 129 | beVerbose=False, \ 130 | doSaveMarginals = False, \ 131 | numPoints=129) 132 | 133 | # wmax = np.max(w) 134 | # W, C = np.mgrid[0:wmax:128j, cmin:cmax:128j] 135 | 136 | 137 | # positions = np.vstack([W.ravel(), C.ravel()]) 138 | # values = np.vstack([w,c]) 139 | # kernel = stats.gaussian_kde(values) 140 | # di = kernel(np.vstack([W.flatten(), C.flatten()])) 141 | 142 | # fig = plt.figure() 143 | # ax = fig.add_subplot(111) 144 | # ax.pcolormesh(W, C, di.reshape(W.shape)) 145 | #plt.show() 146 | 147 | return akernel 148 | 149 | 150 | def getDensityInterpolatorFastkdeWCRatio(drift, numSim = 2**22+1, nSteps = 50): 151 | 152 | numSteps = nSteps 153 | 154 | timeStep = 1.0/numSteps 155 | 156 | numSimulations = numSim 157 | 158 | hlc = np.zeros((numSimulations, 3), dtype = float) 159 | 160 | drift = drift 161 | 162 | minDensity = 1.0/numSimulations/10 163 | 164 | for re in hlc: 165 | arr = np.array(np.random.normal(0,math.sqrt(timeStep),numSteps)) 166 | arr = arr+drift*timeStep 167 | myHLC = np.cumsum(arr) 168 | re[0] = np.max(myHLC) 169 | re[1] = np.min(myHLC) 170 | re[2] = myHLC[-1] 171 | if re[0]<0.0: 172 | re[0] = 0 173 | if re[1]>0.0: 174 | re[1] = 0 175 | 176 | # 177 | w = hlc[:,0]-hlc[:,1] 178 | c = hlc[:,2] 179 | ratios = c/w 180 | 181 | # akernel = fastKDE.fastKDE(ratios, \ 182 | # beVerbose=False, \ 183 | # doSaveMarginals = False, \ 184 | # numPoints=513) 185 | akernel = stats.gaussian_kde(ratios) 186 | 187 | # wmax = np.max(w) 188 | # W, C = np.mgrid[0:wmax:128j, cmin:cmax:128j] 189 | 190 | 191 | # positions = np.vstack([W.ravel(), C.ravel()]) 192 | # values = np.vstack([w,c]) 193 | # kernel = stats.gaussian_kde(values) 194 | # di = kernel(np.vstack([W.flatten(), C.flatten()])) 195 | 196 | # fig = plt.figure() 197 | # ax = fig.add_subplot(111) 198 | # ax.pcolormesh(W, C, di.reshape(W.shape)) 199 | #plt.show() 200 | 201 | return akernel 202 | 203 | 204 | 205 | if __name__ == "__main__": 206 | 207 | aDrift = 0.0 208 | numSteps = 50 209 | numSim = 2**21+1 210 | 211 | # mw, mc = np.mgrid[0.06:5.94:100j, -5.94:5.94:200j] 212 | # pts = np.vstack([mw.ravel(), mc.ravel()]) 213 | 214 | #testing load interpolators 215 | # for aDrift in np.linspace(0.0,2,41 ): 216 | # print('drift = ' + '{:1.2f}'.format(aDrift)) 217 | # filename ='DensityInterpolators/DensityInterpolatorDrift'+'{:1.0f}'.format(aDrift*100)+'.obj' 218 | # aInterp = pickle.load(open(filename, 'rb')) 219 | # afile.close() 220 | # print(filename) 221 | # re = aInterp(pts.T) 222 | # re = np.reshape(re, (mw.shape[0], mw.shape[1])) 223 | # fig3 = plt.figure() 224 | # cx = fig3.add_subplot(111) 225 | # cx.pcolormesh(mw,mc,re) 226 | 227 | # generating density interpolators 228 | # for aDrift in np.linspace(0.0,2,41 ): 229 | # print('drift = ' + '{:1.2f}'.format(aDrift)) 230 | # aInterp = getDensityInterpolator(aDrift, numSim, numSteps) 231 | # filename ='DensityInterpolators/DensityInterpolatorDrift'+'{:1.0f}'.format(aDrift*100)+'.obj' 232 | # afile = open(filename, 'wb') 233 | # pickle.dump(aInterp, afile) 234 | # afile.close() 235 | # print(filename) 236 | 237 | 238 | 239 | # generating fastKDE objects and save 240 | # for aDrift in np.linspace(0.0,2,41 ): 241 | # start_time = time.time() 242 | # akernel = getDensityInterpolatorFastkde(aDrift, numSim, numSteps) 243 | # v1,v2 = akernel.axes 244 | # 245 | # print('drift = ' + '{:1.2f}'.format(aDrift)) 246 | # filename ='DensityInterpolators/fastKDEKernelDrift'+'{:1.0f}'.format(aDrift*100)+'.obj' 247 | # afile = open(filename, 'wb') 248 | # pickle.dump(akernel, afile) 249 | # afile.close() 250 | # print(filename) 251 | # PP.contour(v1,v2,akernel.pdf) 252 | # PP.show() 253 | # print("--- %s seconds ---" % (time.time() - start_time)) 254 | # x2,y2=np.meshgrid(v1,v2) 255 | # fig = plt.figure() 256 | # ax = fig.add_subplot(111) 257 | # ax.pcolormesh(x2, y2, akernel.pdf.reshape(x2.shape)) 258 | 259 | # generating fastKDE interpolators and save 260 | 261 | # v3,v4 = np.mgrid[0:5:501j, -5:5:1001j ] 262 | # aDrift = 0.25 263 | # start_time = time.time() 264 | ## pts = np.vstack([v3.flatten(), v4.flatten()]) 265 | # 266 | # for aDrift in np.linspace(0.0,2,41 ): 267 | # start_time = time.time() 268 | # 269 | # print('drift = ' + '{:1.2f}'.format(aDrift)) 270 | # filename ='DensityInterpolators/fastKDEKernelDrift'+'{:1.0f}'.format(aDrift*100)+'.obj' 271 | # afile = open(filename, 'rb') 272 | # akernel = pickle.load(afile) 273 | # afile.close() 274 | # pdfs = akernel.pdf 275 | # v1,v2 = akernel.axes 276 | # pdfs[pdfs<1e-7]=1e-8 277 | ## aInterp = RegularGridInterpolator((v1, v2), pdfs, method = 'linear', bounds_error = False, fill_value = 1e-8) 278 | # PP.contour(v1,v2,akernel.pdf) 279 | # PP.show() 280 | 281 | # x1,x2 = np.meshgrid(v1,v2) 282 | # pts = np.vstack((x1.flatten(), x2.flatten()) 283 | # newPdfs = aInterp(pts.T) 284 | # fig = plt.figure() 285 | # ax = fig.add_subplot(111) 286 | # 287 | # ax.pcolormesh(x1, x2, newPdfs.reshape(x2.shape).T) 288 | # ax.pcolormesh(v3, v4, newPdfs.reshape(v3.shape)) 289 | 290 | # print(filename+" closed") 291 | # filename ='DensityInterpolators/fastKDEInterpolator'+'{:1.0f}'.format(aDrift*100)+'.obj' 292 | # afile = open(filename, 'wb') 293 | # pickle.dump(aInterp, afile) 294 | # afile.close() 295 | # print('Interpolator saved to '+filename) 296 | # print("--- %s seconds ---" % (time.time() - start_time)) 297 | 298 | #testing fastKDE interpolators 299 | # 300 | # v1,v2 = np.mgrid[0:5:501j, -5:5:1001j ] 301 | # aDrift = 0.25 302 | # start_time = time.time() 303 | 304 | # filename ='DensityInterpolators/fastKDEKernelDrift'+'{:1.0f}'.format(aDrift*100)+'.obj' 305 | # afile = open(filename, 'rb') 306 | # akernel = pickle.load(afile) 307 | # afile.close() 308 | 309 | # print('drift = ' + '{:1.2f}'.format(aDrift)) 310 | # filename ='DensityInterpolators/fastKDEInterpolator'+'{:1.0f}'.format(aDrift*100)+'.obj' 311 | # afile = open(filename, 'rb') 312 | # aInterp = pickle.load(afile) 313 | # afile.close() 314 | # 315 | # pts = np.vstack([v1.ravel(), v2.ravel()]) 316 | # pdfs = aInterp(pts.T) 317 | # 318 | # fig = plt.figure() 319 | # ax = fig.add_subplot(111) 320 | # ax.pcolormesh(v1, v2, pdfs.reshape(v1.shape)) 321 | # 322 | # print("--- %s seconds ---" % (time.time() - start_time)) 323 | 324 | # generating wcratio density interpolators 325 | x=np.mgrid[-1:1:513j] 326 | start_time = time.time() 327 | # for aDrift in np.linspace(0.0,2,41 ): 328 | # akernel = getDensityInterpolatorFastkdeWCRatio(aDrift) 329 | # filename ='DensityInterpolators/cwRatioKernelDrift'+'{:1.0f}'.format(aDrift*100)+'.obj' 330 | # afile = open(filename, 'wb') 331 | # pickle.dump(akernel, afile) 332 | # afile.close() 333 | # print('drift = ' + '{:1.2f}'.format(aDrift) +' done') 334 | # print('kernel done') 335 | # print('------ %s seconds-----' % (time.time()-start_time)) 336 | # pdfs = akernel(x) 337 | # 338 | # filename = 'DensityInterpolators/cwRatioDensityInterpolatorDrift'+'{:1.0f}'.format(aDrift*100)+'.obj' 339 | # aInterp = interp1d(x,pdfs) 340 | # bfile = open(filename, 'wb') 341 | # pickle.dump(aInterp, bfile) 342 | # bfile.close() 343 | # print('interpolator done') 344 | # print('------ %s seconds-----' % (time.time()-start_time)) 345 | 346 | for aDrift in np.linspace(1.,2,11 ): 347 | filename = 'DensityInterpolators/cwRatioDensityInterpolatorDrift'+'{:1.0f}'.format(aDrift*100)+'.obj' 348 | 349 | bfile = open(filename, 'rb') 350 | aInterp=pickle.load(bfile) 351 | bfile.close() 352 | 353 | pdfs = aInterp(x) 354 | plt.plot(x,pdfs) 355 | plt.show() 356 | print('interpolator done') 357 | print('------ %s seconds-----' % (time.time()-start_time)) 358 | --------------------------------------------------------------------------------