├── paper.pdf
├── README.md
├── unpack.py
├── clean_data.py
├── models.py
└── itch.py


/paper.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vnatesh/VWAP-Prediction/HEAD/paper.pdf


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # VWAP-Prediction
2 | This is a research project where we designed several algorithms for volume-weighted-average-price (VWAP) prediction. Our dataset consisted of millisecond-level limit-order books for multiple stocks. Random forest and logistic regression were used for VWAP direction classification (up or down), while PCA and random forest were used for feature selection. Least absolute shrinkage and selection operator (LASSO) regression was performed to predict real VWAP value. We also used a long short-term memory (LSTM) recurrent neural network to predict real VWAP value. See paper.pdf for more details.
3 | 


--------------------------------------------------------------------------------
/unpack.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import pandas as pd
 3 | import itch 
 4 | import h5py
 5 | import sys
 6 | import time as timer
 7 | 
 8 | 
 9 | DATE_list = ['20181105','20181106','20181107','20181108','20181109',
10 |              '20181112','20181113','20181114','20181115','20181116',
11 |              '20181119','20181120','20181121','20181123','20181126',
12 |              '20181127','20181128','20181129','20181130','20181203',
13 |              '20181204' ]
14 | # stock = 'AAPL'
15 | stock = 'GS'
16 | 
17 | fout = h5py.File('gs_tick_data.hdf5', 'a')                           
18 | LEVELS = 10
19 | 
20 | for DATE in DATE_list:
21 |     print(DATE)
22 |     start = timer.time()
23 | 
24 |     df = pd.read_csv("~/Downloads/"+DATE+"_"+stock+".csv")                            
25 |     df = df.drop('MPID', axis = 1)
26 |     df = df.drop('X', axis = 1)                                                                              
27 |     df_v = df.values
28 | 
29 |     orderpool = itch.Orderpool() 
30 |     book = itch.Book(LEVELS)
31 |     messagedata = []
32 |     bookdata = []
33 | 
34 |     for i in range(len(df_v)):
35 |         line  = df_v[i]
36 |         message_type = line[3]
37 |         message = itch.get_message(line,message_type)
38 |         
39 |         # complete message...
40 |         if message_type in ('E', 'C', 'F', 'D'):
41 |             orderpool.complete_message(message)
42 |         
43 |         # update orderpool...
44 |         if message_type in ('B','S','E', 'C', 'F', 'D'):
45 |             orderpool.update(message)
46 | 
47 |         # update booklist...
48 |         if message_type in ('B','S','E', 'C', 'F', 'D'):
49 |             book.update(message)
50 | 
51 |         # update messagedata...
52 |         messagedata.append(message.values())
53 | 
54 |         # update bookdata...
55 |         # check if bookdata is all zero
56 |         book_v = book.values()
57 |         if np.any(book_v[1:]):
58 |             bookdata.append(book_v)
59 | 
60 |     # messagedata to HDF5...
61 |     messagedata = np.asarray(messagedata)
62 |     group = 'messages'
63 |     itch.writedata(messagedata,fout,group,stock,DATE)
64 | 
65 |     # bookdata to HDF5...
66 |     bookdata = np.asarray(bookdata)
67 |     group = 'orderbooks'
68 |     itch.writedata(bookdata,fout,group,stock,DATE)
69 | 
70 |     stop = timer.time()
71 | 
72 |     # OUTPUT #
73 |     print('Elapsed time:', stop - start, 'sec')
74 | 
75 | fout.close()
76 | 


--------------------------------------------------------------------------------
/clean_data.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import pandas as pd
  3 | import h5py
  4 | import matplotlib.pyplot as plt
  5 | 
  6 | def load_messages(path, name, date):
  7 |     data = h5py.File(path, 'r')
  8 |     messages = data['/messages/' + name + '/' + date]
  9 |     mdata = messages[:, :]
 10 |     t, n = mdata.shape
 11 |     data.close()
 12 |     mcolumns = ['msec',
 13 |                 'type',
 14 |                 'buysell',
 15 |                 'price',
 16 |                 'shares',
 17 |                 'refno']
 18 |     mout = pd.DataFrame(mdata, index=np.arange(0, t), columns=mcolumns)
 19 | #     mout["time"]= pd.to_datetime(mout["msec"],unit='ms',origin=pd.Timestamp(date)) 
 20 | #     mout = mout[['time','msec', 'type', 'buysell', 'price', 'shares', 'refno']]    
 21 |     return mout
 22 | 
 23 | 
 24 | def load_books(path, name, date):
 25 |     data = h5py.File(path, 'r')
 26 |     orderbooks = data['/orderbooks/' + name + '/' + date]
 27 |     mdata = orderbooks[:, :]
 28 |     t, n = mdata.shape
 29 |     data.close()
 30 |     # columns names
 31 |     time = ['msec']
 32 |     bid_price = ['bp'+str(i) for i in range(1,11)]
 33 |     ask_price = ['ap'+str(i) for i in range(1,11)]
 34 |     bid_volumn = ['bv'+str(i) for i in range(1,11)]
 35 |     ask_volumn = ['av'+str(i) for i in range(1,11)]
 36 |     mcolumns = time+bid_price+ask_price+bid_volumn+ask_volumn
 37 |     mout = pd.DataFrame(mdata, index=np.arange(0, t),columns=mcolumns)
 38 |     return mout
 39 | 
 40 | def vwap_series(df, tinterval):
 41 |     df['sec'] = df['msec']/1000
 42 |     vwap_list = []
 43 |     df_v = df.values
 44 |     time = 34200
 45 |     temp = []
 46 |     for i in range(len(df_v)):
 47 |         if df_v[i][6]>= time and df_v[i][6]< time+tinterval:
 48 |             temp.append([df_v[i][3],df_v[i][4]])
 49 |         if df_v[i][6]>= time+tinterval:
 50 |             time = time+tinterval
 51 |             vol_time_price = [x[0]*x[1] for x in temp]
 52 |             if sum([x[1] for x in temp]) != 0:
 53 |                 vwap_list.append(sum(vol_time_price)/sum([x[1] for x in temp]))
 54 |                 temp = []
 55 |             else:
 56 |                 vwap_list.append(np.nan)
 57 |                 temp = []
 58 |         if i == len(df_v)-1:
 59 |             # multiply volume by price for each row in the 10s interval
 60 |             vol_time_price = [x[0]*x[1] for x in temp]
 61 |             if sum([x[1] for x in temp]) != 0:
 62 |                 # sum all the vol*p and divide by total volume to get vwap
 63 |                 vwap_list.append(sum(vol_time_price)/sum([x[1] for x in temp]))
 64 |             else:
 65 |                 vwap_list.append(np.nan)
 66 |     return vwap_list 
 67 | 
 68 | 
 69 | 
 70 | 
 71 | 
 72 | DATE_list = ['20181105','20181106','20181107','20181108','20181109',
 73 |              '20181112','20181113','20181114','20181115','20181116',
 74 |              '20181119','20181120','20181121','20181126',
 75 |              '20181127','20181128','20181129','20181130','20181203',
 76 |              '20181204' ]
 77 | 
 78 | df_mult_date = pd.DataFrame()
 79 | 
 80 | for DATE in DATE_list:
 81 | 
 82 |     # Goldman Sachs message data
 83 |     # df = load_messages('gs_tick_data.hdf5', 'GS', DATE)
 84 |     # Apple message data
 85 |     df = load_messages('/Volumes/easystore/FML_project/aapl_tick_data.hdf5', 'AAPL', DATE)
 86 |     df = df[(df['msec'] >= 34200000) & (df['msec'] <= 57600000)]
 87 |     ex = df[df['type'].isin([2,4,6,7])]
 88 |     ex = ex.reset_index(drop=True)
 89 |     ex['price'] = ex['price']/10000
 90 | 
 91 |     # computing 10 second vwap series
 92 |     vwap = vwap_series(ex,10)
 93 |     sec = list(range(34210,57601,10))
 94 |     vwap_df = pd.DataFrame()
 95 |     vwap_df['vwap'] = vwap
 96 |     vwap_df['sec'] = sec
 97 |     vwap_df['msec'] = vwap_df['sec']*1000
 98 |     vwap_df["time"]= pd.to_datetime(vwap_df["msec"],unit='ms',origin=pd.Timestamp(20181204)) 
 99 |     vwap_df = vwap_df.dropna()
100 |     # l.plot(x='time', y='vwap')
101 |     # plt.show()
102 | 
103 |     # Goldman Sachs Tick data
104 |     # book = load_books('gs_tick_data.hdf5', 'GS', DATE)
105 |     # Apple tick data
106 |     book = load_books('/Volumes/easystore/FML_project/aapl_tick_data.hdf5','AAPL',DATE)
107 |     book = book[(book['msec'] >= 34200000) & (book['msec'] <= 57600000)]
108 |     book = book.reset_index(drop=True)
109 |     book = book[['msec', 'bp1', 'bp2', 'bp3', 'bp4', 'bp5', 'ap1', 'ap2', 'ap3', 'ap4', 'ap5', 'bv1', 'bv2', 'bv3', 'bv4', 'bv5', 'av1', 'av2', 'av3', 'av4', 'av5']]
110 |     book['origion'] = 1
111 | 
112 |     msec = [x*1000 for x in list(range(34200,57600,10))]
113 |     mcolumns = ['msec', 'bp1', 'bp2', 'bp3', 'bp4', 'bp5', 'ap1', 'ap2', 'ap3', 'ap4', 'ap5', 'bv1', 'bv2', 'bv3', 'bv4', 'bv5', 'av1', 'av2', 'av3', 'av4', 'av5','origion']
114 |     a = np.empty((len(msec),22,))
115 |     a[:] = np.nan
116 |     insert_book = pd.DataFrame(a,index=np.arange(0, len(msec)),columns=mcolumns)
117 |     insert_book['msec'] = msec
118 |     insert_book['origion'] = 0
119 | 
120 |     # merge
121 |     frames = [book, insert_book]
122 |     y = pd.concat(frames,ignore_index=True)
123 |     y = y.sort_values(by=['msec'])
124 |     y = y.fillna(method='ffill')
125 |     # pull out
126 |     new_book = y[y['origion']==0]
127 |     new_book = new_book.dropna()
128 |     new_book = new_book.drop(columns=['origion'])
129 | 
130 |     d = vwap_df.join(new_book.set_index('msec'), on='msec')
131 |     d = d.dropna()
132 |     d = d.drop(columns=['time','sec'])
133 | 
134 |     cols = ['bp1', 'bp2', 'bp3', 'bp4', 'bp5', 'ap1', 'ap2', 'ap3', 'ap4', 'ap5', 'bv1', 'bv2', 'bv3', 'bv4', 'bv5', 'av1', 'av2', 'av3', 'av4', 'av5']
135 |     for col in cols:
136 |         d['delta_'+col] = d[col].diff(1)
137 | 
138 |     d['mean_volumn_diff'] = (d['bv1']+d['bv2']+d['bv3']+d['bv4']+d['bv5'])/5 - (d['av1']+d['av2']+d['av3']+d['av4']+d['av5'])/5
139 |     d['spread'] = d['ap1'] - d['bp1']
140 |     d['vol_unb1'] = (d['bv1'] - d['av1'])/d['bv1']
141 |     d['vol_unb2'] = (d['bv2'] - d['av2'])/d['bv2']
142 |     d['vol_unb3'] = (d['bv3'] - d['av3'])/d['bv3']
143 |     d['vol_unb4'] = (d['bv4'] - d['av4'])/d['bv4']
144 |     d['vol_unb5'] = (d['bv5'] - d['av5'])/d['bv5']
145 | 
146 |     d_v = d.values
147 |     mom_b = [np.nan,np.nan,np.nan,np.nan,np.nan]
148 |     volat_b = [np.nan,np.nan,np.nan,np.nan,np.nan]
149 |     mom_a = [np.nan,np.nan,np.nan,np.nan,np.nan]
150 |     volat_a = [np.nan,np.nan,np.nan,np.nan,np.nan]
151 | 
152 |     # why volataility for last 5 
153 |     for i in range(5,len(d_v)):
154 |         bp_past5 = np.asarray([d_v[i-1][2]/10000,d_v[i-2][2]/10000,d_v[-3][2]/10000,d_v[-4][2]/10000,d_v[i-5][2]/10000])
155 |         ap_past5 = np.asarray([d_v[i-1][7]/10000,d_v[i-2][7]/10000,d_v[-3][7]/10000,d_v[-4][7]/10000,d_v[i-5][7]/10000])
156 |         mom_b.append((d_v[i][2]-d_v[i-5][2])/d_v[i-5][2])
157 |         volat_b.append(bp_past5.std())
158 |         mom_a.append((d_v[i][7]-d_v[i-5][7])/d_v[i-5][7])
159 |         volat_a.append(ap_past5.std())
160 |     d['mom_bp1'] = mom_b
161 |     d['mom_ap1'] = mom_a
162 |     d['vola_bp1'] = volat_b
163 |     d['vola_ap1'] = volat_a
164 | 
165 |     label1 = []
166 |     label2 = []
167 |     for i in range(len(d_v)-1):
168 |         if d_v[i+1][0]>d_v[i][0]:
169 |             label1.append(1)
170 |         if d_v[i+1][0]<d_v[i][0]:
171 |             label1.append(-1)
172 |         label2.append(d_v[i+1][0])
173 | 
174 |     label1.append(np.nan)
175 |     label2.append(np.nan)
176 | 
177 |     d['vwap_d'] = label1
178 |     d['vwap_v'] = label2
179 |     d = d.dropna()
180 |     d = d.reset_index(drop = True)
181 | 
182 |     frames = [df_mult_date, d]
183 |     df_mult_date = pd.concat(frames,ignore_index=True)
184 | 
185 | # AAPL limit order features data
186 | df_mult_date.to_csv('labelled_data_10s_AAPL', index=False)
187 |     
188 | 


--------------------------------------------------------------------------------
/models.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import pandas as pd
  3 | import h5py
  4 | import matplotlib.pyplot as plt
  5 | from sklearn.metrics import accuracy_score, mean_squared_error
  6 | from scipy import stats
  7 | from sklearn.ensemble import RandomForestClassifier
  8 | from sklearn.linear_model import LogisticRegression
  9 | from sklearn import linear_model
 10 | from sklearn.metrics import r2_score
 11 | from math import sqrt
 12 | from numpy import concatenate
 13 | from matplotlib import pyplot
 14 | from pandas import read_csv
 15 | from pandas import DataFrame
 16 | from pandas import concat
 17 | from sklearn.preprocessing import MinMaxScaler
 18 | from sklearn.preprocessing import LabelEncoder
 19 | from sklearn.metrics import mean_squared_error
 20 | from keras.models import Sequential
 21 | from keras.layers import Dense
 22 | from keras.layers import LSTM
 23 | 
 24 | 
 25 | 
 26 | df = pd.read_csv('/Users/vikasnatesh/Downloads/labeled_data_10s.csv')
 27 | 
 28 | 
 29 | 
 30 | X = df.loc[:, ~df.columns.isin(['vwap_d','vwap_v'])].values
 31 | y = df['vwap_d'].values
 32 | 
 33 | # Train Test split (80% train, 20% test)
 34 | X_train = X[:int(0.8*len(y))]
 35 | X_test = X[int(0.8*len(y)):]
 36 | y_train = y[:int(0.8*len(y))]
 37 | y_test = y[int(0.8*len(y)):]
 38 | 
 39 | 
 40 | def group_prob(x):
 41 |     for i in range(len(quantile)-1):
 42 |         if x>=quantile[i] and x<quantile[i+1]:
 43 |             return i+1
 44 |         if x==quantile[i+1]:
 45 |             return 10
 46 | 
 47 | #########   LogisticRegression
 48 | # clf = LogisticRegression(random_state=0, solver='lbfgs',multi_class='multinomial')
 49 | 
 50 | ####### Random Forest
 51 | 
 52 | clf = RandomForestClassifier(n_estimators=20, max_depth=3,random_state=0)
 53 | clf.fit(X_train, y_train)
 54 | y_pred = clf.predict(X_test)
 55 | print(accuracy_score(y_test, y_pred))
 56 | 
 57 | 
 58 | pred_prob = clf.predict_proba(X_test)
 59 | prob_down = [x[0] for x in pred_prob]
 60 | prob_df = pd.DataFrame()
 61 | prob_df['label'] = y_test
 62 | prob_df['prob_down'] = prob_down
 63 | 
 64 | global quantile
 65 | quantile = np.percentile(prob_down,list(list(range(0,101,10))))
 66 | 
 67 | prob_df['group'] = prob_df['prob_down'].apply(group_prob)
 68 | 
 69 | pd = []
 70 | td = []
 71 | for i in range(1,11):
 72 |     group_df = prob_df[prob_df['group']==i]
 73 |     true_down_prob = stats.itemfreq(group_df['label'])[0][1]/group_df.shape[0]
 74 |     print('group:',i,'predict_prob:',group_df['prob_down'].median(),'true_prob:',true_down_prob)
 75 |     pd.append(group_df['prob_down'].median())
 76 |     td.append(true_down_prob)
 77 | 
 78 | print('mse',mean_squared_error(pd,td))
 79 | 
 80 | 
 81 | 
 82 | ############ Lasso model ###############
 83 | 
 84 | 
 85 | from sklearn import linear_model
 86 | from sklearn.metrics import r2_score
 87 | 
 88 | df = pd.read_csv('/Users/vikasnatesh/Downloads/labeled_data_10s.csv')
 89 | # Train Test split (80% train, 20% test)
 90 | 
 91 | X = df.loc[:, ~df.columns.isin(['vwap_d','vwap_v'])].values
 92 | 
 93 | y = df['vwap_v'].values
 94 | y_train = y[:int(0.8*len(y))]
 95 | y_test = y[int(0.8*len(y)):]
 96 | 
 97 | X_train = X[:int(0.8*len(y))]
 98 | X_test = X[int(0.8*len(y)):]
 99 | 
100 | clf = linear_model.Lasso(alpha=1)
101 | clf.fit(X_train, y_train)
102 | y_pred = clf.predict(X_test)
103 | 
104 | print('mse',mean_squared_error(y_test, y_pred))
105 | print('R^2 score', r2_score(y_test, y_pred))
106 | 
107 | pyplot.plot(y_test, label='actual vwap')
108 | pyplot.plot(y_pred, label='predicted vwap')
109 | pyplot.xlabel('time (10s)')
110 | pyplot.ylabel('normalized vwap')
111 | pyplot.title('AAPL 10s VWAP')
112 | pyplot.legend()
113 | pyplot.show()
114 | 
115 | 
116 | 
117 | 
118 | ############ LSTM model ###############
119 | 
120 | # convert series to supervised learning
121 | def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):
122 |     n_vars = 1 if type(data) is list else data.shape[1]
123 |     df = DataFrame(data)
124 |     cols, names = list(), list()
125 |     # input sequence (t-n, ... t-1)
126 |     for i in range(n_in, 0, -1):
127 |         cols.append(df.shift(i))
128 |         names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]
129 |     # forecast sequence (t, t+1, ... t+n)
130 |     for i in range(0, n_out):
131 |         cols.append(df.shift(-i))
132 |         if i == 0:
133 |             names += [('var%d(t)' % (j+1)) for j in range(n_vars)]
134 |         else:
135 |             names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]
136 |     # put it all together
137 |     agg = concat(cols, axis=1)
138 |     agg.columns = names
139 |     # drop rows with NaN values
140 |     if dropnan:
141 |         agg.dropna(inplace=True)
142 |     return agg
143 | 
144 | # load dataset
145 | dataset = read_csv('/Users/vikasnatesh/Downloads/labeled_data_10s.csv', header=0, index_col=1)
146 | 
147 | 
148 | # values = dataset.values
149 | # # specify columns to plot
150 | # groups = [0, 1, 2, 3, 5, 6, 7,8,9,10]
151 | # i = 1
152 | # # plot each column
153 | # pyplot.figure()
154 | # for group in groups:
155 | #     pyplot.subplot(len(groups), 1, i)
156 | #     pyplot.plot(values[:, group])
157 | #     pyplot.title(dataset.columns[group], y=0.5, loc='right')
158 | #     i += 1
159 | # pyplot.show()
160 | 
161 | 
162 | values = dataset.loc[:, ~dataset.columns.isin(['vwap_d','vwap_v'])].values
163 | 
164 | # values = dataset.values
165 | # integer encode direction
166 | encoder = LabelEncoder()
167 | values[:,4] = encoder.fit_transform(values[:,4])
168 | # ensure all data is float
169 | values = values.astype('float32')
170 | # normalize features
171 | scaler = MinMaxScaler(feature_range=(0, 1))
172 | scaled = scaler.fit_transform(values)
173 | # frame as supervised learning
174 | reframed = series_to_supervised(scaled, 1, 1)
175 | # reframed = series_to_supervised(values, 1, 1)
176 | 
177 | 
178 | values = reframed.values
179 | # Train Test split (80% train, 20% test)
180 | train = values[:int(0.8*len(values))]
181 | test = values[int(0.8*len(values)):]
182 | 
183 | X_train, y_train = train[:, :-1], train[:, -1]
184 | X_test, y_test = test[:, :-1], test[:, -1]
185 | 
186 | # y = df['vwap_v'].values
187 | # y_train = y[:int(0.8*len(y))]
188 | # y_test = y[int(0.8*len(y)):]
189 | 
190 | 
191 | # reshape input to be 3D [samples, timesteps, features]
192 | X_train = X_train.reshape((X_train.shape[0], 1, X_train.shape[1]))
193 | X_test = X_test.reshape((X_test.shape[0], 1, X_test.shape[1]))
194 | print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)
195 | 
196 | 
197 | model = Sequential()
198 | model.add(LSTM(50, input_shape=(X_train.shape[1], X_train.shape[2])))
199 | model.add(Dense(1))
200 | model.compile(loss='mae', optimizer='adam')
201 | # fit network
202 | history = model.fit(X_train, y_train, epochs=40, batch_size=73, validation_data=(X_test, y_test), verbose=2, shuffle=False)
203 | # plot history
204 | pyplot.plot(history.history['loss'], label='train')
205 | pyplot.plot(history.history['val_loss'], label='test')
206 | pyplot.xlabel('epoch')
207 | pyplot.ylabel('error')
208 | pyplot.title('AAPL 10s VWAP')
209 | pyplot.legend()
210 | pyplot.show()
211 | 
212 | # make a prediction
213 | yhat = model.predict(X_test)
214 | X_test = X_test.reshape((X_test.shape[0], X_test.shape[2]))
215 | # invert scaling for forecast
216 | inv_yhat = concatenate((yhat, X_test[:, 1:]), axis=1)
217 | inv_yhat = scaler.inverse_transform(inv_yhat)
218 | inv_yhat = inv_yhat[:,0]
219 | # invert scaling for actual
220 | y_test = y_test.reshape((len(y_test), 1))
221 | inv_y = concatenate((y_test, X_test[:, 1:]), axis=1)
222 | inv_y = scaler.inverse_transform(inv_y)
223 | inv_y = inv_y[:,0]
224 | # calculate RMSE
225 | rmse = sqrt(mean_squared_error(inv_y, inv_yhat))
226 | print('Test RMSE: %.3f' % rmse)
227 | print('R^2 score', r2_score(inv_y, inv_yhat))
228 | 
229 | 
230 | 
231 | pyplot.plot(y_test, label='actual vwap')
232 | pyplot.plot(yhat, label='predicted vwap')
233 | pyplot.xlabel('time (10s)')
234 | pyplot.ylabel('normalized vwap')
235 | pyplot.title('AAPL 10s VWAP')
236 | pyplot.legend()
237 | pyplot.show()
238 | 
239 | 
240 | 
241 | 
242 | 
243 | clf = linear_model.Lasso(alpha=1)
244 | clf.fit(X_train, y_train)
245 | y_pred = clf.predict(X_test)
246 | 
247 | print('mse',mean_squared_error(y_test, yhat))
248 | print('R^2 score', r2_score(y_test, yhat))
249 | 
250 | # Random Forest
251 | # from sklearn.ensemble import RandomForestClassifier
252 | # clf = RandomForestClassifier(n_estimators=20, max_depth=3,random_state=0)
253 | # clf.fit(X_train, y_train)
254 | # y_pred = clf.predict(X_test)
255 | # print(accuracy_score(y_test, y_pred))
256 | 
257 | # # SVM
258 | # from sklearn.svm import LinearSVC
259 | # clf = LinearSVC(random_state=0, tol=1e-5)
260 | # clf.fit(X_train, y_train)
261 | # y_pred = clf.predict(X_test)
262 | # print(accuracy_score(y_test, y_pred))
263 | 
264 | 
265 | # # from scipy import stats
266 | # # print(stats.itemfreq(y_test))


--------------------------------------------------------------------------------
/itch.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import struct
  3 | import pandas as pd
  4 | import h5py
  5 | 
  6 | # CLASSES #
  7 | 
  8 | class Message:
  9 |     def __init__(self, msec=-1, type = '.',name = '.', buysell = '.',price = -1, shares = 0,refno=-1):
 10 |         self.msec = msec
 11 |         self.type = type
 12 |         self.name = name
 13 |         self.buysell = buysell
 14 |         self.price = price
 15 |         self.shares = shares
 16 |         self.refno = refno
 17 | 
 18 |     def values(self):
 19 |         values = [int(self.msec)]
 20 |         
 21 |         if self.type in ('B', 'S'):  # adds
 22 |             values.append(1)
 23 |         elif self.type == 'E':  # partial execute
 24 |             values.append(2)
 25 |         elif self.type == 'C':  # partial cancel
 26 |             values.append(3)
 27 |         elif self.type == 'F':  # execute outstanding in full
 28 |             values.append(4)
 29 |         elif self.type == 'D':  # delete outstanding in full
 30 |             values.append(5)
 31 |         elif self.type == 'X':  # bulk volume for the cross event
 32 |             values.append(6)
 33 |         elif self.type == 'T':  # Execute non-displayed order
 34 |             values.append(7)
 35 |         else:
 36 |             values.append(-1)  # other (ignored)
 37 | 
 38 |         values.append(int(self.buysell))
 39 |         values.append(int(self.price))
 40 |         values.append(int(self.shares))
 41 |         values.append(int(self.refno))
 42 |         # values.append(int(self.newrefno))
 43 | 
 44 |         return np.array(values)
 45 | 
 46 | class Order:
 47 |     def __init__(self, name='.', buysell='.', price='.', shares='.'):
 48 |         self.name = name
 49 |         self.buysell = buysell
 50 |         self.price = price
 51 |         self.shares = shares
 52 | 
 53 | class Orderpool:
 54 |     def __init__(self):
 55 |         self.orders = {}
 56 | 
 57 |     # Changes message by REFERENCE.
 58 |     def complete_message(self, message):
 59 |         if message.refno in self.orders.keys():
 60 |             ref_order = self.orders[message.refno]
 61 |             # partial execute
 62 |             if message.type == 'E':
 63 |                 message.buysell = ref_order.buysell
 64 |                 message.price = ref_order.price
 65 |             # partial cancel
 66 |             if message.type == 'C':
 67 |                 message.buysell = ref_order.buysell
 68 |                 message.price = ref_order.price
 69 |             # execute outstanding in full
 70 |             if message.type == 'F':
 71 |                 message.buysell = ref_order.buysell
 72 |                 message.price = ref_order.price
 73 |                 message.shares = ref_order.shares
 74 |             # delete outstanding in full
 75 |             if message.type == 'D':
 76 |                 message.buysell = ref_order.buysell
 77 |                 message.price = ref_order.price
 78 |                 message.shares = ref_order.shares
 79 | 
 80 |     def update(self, message):
 81 |         if message.type in ('B', 'S'):
 82 |             self.add_order(message)
 83 |         elif message.type in ('E', 'C', 'F', 'D'):
 84 |             self.update_order(message)
 85 | 
 86 |     def add_order(self, message):
 87 |         order = Order()
 88 |         order.name = message.name
 89 |         order.price = message.price
 90 |         order.shares = message.shares
 91 |         if message.type == 'B':
 92 |             order.buysell = 1
 93 |         elif message.type == 'S':
 94 |             order.buysell = -1
 95 |         self.orders[message.refno] = order
 96 | 
 97 |     def update_order(self, message):
 98 |         if message.refno in self.orders.keys():
 99 |             if message.type == 'E':  # partial execute
100 |                 self.orders[message.refno].shares -= message.shares
101 |             elif message.type == 'C':  # partial cancel
102 |                 self.orders[message.refno].shares -= message.shares
103 |             elif message.type == 'F':  # execute outstanding in full
104 |                 self.orders.pop(message.refno)
105 |             elif message.type == 'D':  # delete outstanding in full
106 |                 self.orders.pop(message.refno)
107 | 
108 | class Book:
109 |     def __init__(self, levels):
110 |         self.bids = {}
111 |         self.asks = {}
112 |         self.levels = levels
113 |         self.msec = -1
114 | 
115 |     def update(self, message):
116 |         self.msec = message.msec
117 | 
118 |         if message.buysell == 1:
119 |             if message.price in self.bids.keys():
120 |                 if message.type == 'B':
121 |                     self.bids[message.price] += message.shares
122 |                 elif message.type in ('E', 'C', 'F','D'):
123 |                     self.bids[message.price] -= message.shares
124 | 
125 |                     if self.bids[message.price] < 0:
126 |                         print('Warning!!! depth of LOB become negative bid', message.refno)
127 | 
128 |                     if self.bids[message.price] == 0:
129 |                         self.bids.pop(message.price)
130 |             else:
131 |                 if message.type == 'B':
132 |                     self.bids[message.price] = message.shares
133 | 
134 |         elif message.buysell == -1:
135 |             if message.price in self.asks.keys():
136 |                 if message.type == 'S':
137 |                     self.asks[message.price] += message.shares
138 |                 elif message.type in ('E', 'C', 'F','D'):
139 |                     self.asks[message.price] -= message.shares
140 | 
141 |                     if self.asks[message.price] < 0:
142 |                         print('Warning!!! depth of LOB become negative ask', message.refno)
143 |                         
144 |                     if self.asks[message.price] == 0:
145 |                         self.asks.pop(message.price)
146 |             else:
147 |                 if message.type == 'S':
148 |                     self.asks[message.price] = message.shares
149 | 
150 |     def values(self):
151 |         """Convert book to numpy array."""
152 |         values = [int(self.msec)]
153 |         sorted_bids = sorted(self.bids.keys(), reverse=True)
154 |         sorted_asks = sorted(self.asks.keys())
155 |         for i in range(0, self.levels):  # bid price
156 |             if i < len(self.bids):
157 |                 values.append(sorted_bids[i])
158 |             else:
159 |                 values.append(0)
160 |         for i in range(0, self.levels):  # ask price
161 |             if i < len(self.asks):
162 |                 values.append(sorted_asks[i])
163 |             else:
164 |                 values.append(0)
165 |         for i in range(0, self.levels):  # bid depth
166 |             if i < len(self.bids):
167 |                 values.append(self.bids[sorted_bids[i]])
168 |             else:
169 |                 values.append(0)
170 |         for i in range(0, self.levels):  # ask depth
171 |             if i < len(self.asks):
172 |                 values.append(self.asks[sorted_asks[i]])
173 |             else:
174 |                 values.append(0)
175 |         return np.array(values)
176 | 
177 | 
178 | # METHODS #
179 | 
180 | def get_message(line, message_type):
181 |     message = Message()
182 |     message.type = message_type
183 |     if message_type == 'B':
184 |         message.msec = line[0]
185 |         message.name = line[1]
186 |         message.refno = line[2]
187 |         message.shares = line[4]
188 |         message.price = line[5]
189 |         message.buysell = 1
190 |     if message_type == 'S':
191 |         message.msec = line[0]
192 |         message.name = line[1]
193 |         message.refno = line[2]
194 |         message.shares = line[4]
195 |         message.price = line[5]
196 |         message.buysell = -1
197 |     if message_type in ['E','C']:
198 |         message.msec = line[0]
199 |         message.name = line[1]
200 |         message.refno = line[2]
201 |         message.shares = line[4]
202 |     if message_type in ['F','D']:
203 |         message.msec = line[0]
204 |         message.name = line[1]
205 |         message.refno = line[2]
206 |     if message_type in ['X','T']:
207 |         message.msec = line[0]
208 |         message.name = line[1]
209 |         message.refno = line[2]
210 |         message.shares = line[4]
211 |         message.price = line[5]
212 |         message.buysell = 0
213 |     return message
214 | 
215 | def writedata(data, file, group, name, date):
216 |     n, m = data.shape
217 |     grp = file.require_group(group)
218 |     name_grp = grp.require_group(name)
219 |         
220 |     date_name_grp = name_grp.require_dataset(date,
221 |                                              shape=(n, m),
222 |                                              maxshape=(None, None),
223 |                                              dtype='i')
224 |     date_name_grp[:, :] = data


--------------------------------------------------------------------------------