grabber.py
17 | Generated with deepchart
21 |Clone the repo:
23 |
24 | git clone https://github.com/PiSimo/BitcoinForecast.git
25 |
26 | python3 network.py -train dataset_path -iterations number_of_training_iterations
30 | To finetune the new model with an old one just add -finetune base_model_path to the line above.
At the end of the training you will have an updated model.h5 with the new weights and you will see a plot with the test results.
32 |python3 network.py -run dataset_path -model model_path
35 | The dataset is also required when you run, to perform normalization.
36 |To visualize a plot with the real and predicted results enter Crtl-C and type no ,the program will create chart.png with the results.
37 | 38 |
43 |
44 |
--------------------------------------------------------------------------------
/data.db:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PiSimo/BitcoinForecast/158be42130e2a7f84ceb5829c7b17edbcf41bcb9/data.db
--------------------------------------------------------------------------------
/grabber.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python3
2 |
3 | ##
4 | ## run the code for about 2/3 days
5 | ##
6 |
7 | import requests
8 | import time
9 |
10 | f_name = input("dataset name:")
11 | f = open(f_name,"a")
12 | keys = ["price_usd","24h_volume_usd","market_cap_usd","available_supply","total_supply","percent_change_1h","percent_change_24h","percent_change_7d"]
13 | vals = [0]*len(keys)
14 |
15 | while True:
16 | data = requests.get("https://api.coinmarketcap.com/v1/ticker/bitcoin/").json()[0]
17 | bstamp = requests.get("https://www.bitstamp.net/api/v2/ticker/btcusd/").json()
18 | bkc = requests.get("https://blockchain.info/ticker").json()
19 |
20 | for d in data.keys():
21 | if d in keys:
22 | indx = keys.index(d)
23 | vals[indx] = data[d]
24 | for val in vals:
25 | f.write(val+",")
26 |
27 | f.write("{},{},".format(bstamp["volume"],bstamp["vwap"]))
28 | f.write("{},{},{}".format(bkc["USD"]["sell"],bkc["USD"]["buy"],bkc["USD"]["15m"]))
29 | f.write("\n")
30 | f.flush()
31 | time.sleep(9*60)
32 |
--------------------------------------------------------------------------------
/model.h5:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PiSimo/BitcoinForecast/158be42130e2a7f84ceb5829c7b17edbcf41bcb9/model.h5
--------------------------------------------------------------------------------
/network.py:
--------------------------------------------------------------------------------
1 | import util
2 | import predict
3 | import time
4 | import argparse
5 | import numpy as np
6 | import matplotlib.pyplot as plt
7 | from sys import argv,exit
8 | from keras.models import Sequential
9 | from keras.layers import Dense,Dropout,GRU,Reshape
10 | from keras.layers.normalization import BatchNormalization
11 | import sqlite3
12 | conn = sqlite3.connect('data.db')
13 |
14 | file_name = 'dataset.csv'
15 | net = None
16 | wait_time = 530
17 |
18 | def buildNet(w_init="glorot_uniform",act="tanh"):
19 | global net
20 | print("Building net..",end="")
21 | net = Sequential()
22 | net.add(Dense(12,kernel_initializer=w_init,input_dim=12,activation='linear'))
23 | net.add(Reshape((1,12)))
24 | net.add(BatchNormalization())
25 | net.add(GRU(40,kernel_initializer=w_init,activation=act,return_sequences=True))
26 | net.add(Dropout(0.4))
27 | net.add(GRU(70,kernel_initializer=w_init,activation=act,return_sequences=True))
28 | net.add(Dropout(0.3))
29 | net.add(GRU(70,kernel_initializer=w_init,activation=act,return_sequences=True))
30 | net.add(Dropout(0.4))
31 | net.add(GRU(40,kernel_initializer=w_init,activation=act,return_sequences=False))
32 | net.add(Dropout(0.4))
33 | net.add(Dense(1,kernel_initializer=w_init,activation='linear'))
34 | net.compile(optimizer='nadam',loss='mse')
35 | print("done!")
36 |
37 | def chart(real,predicted,show=True):
38 | plt.plot(real,color='g')
39 | plt.plot(predicted,color='r')
40 | plt.ylabel('BTC/USD')
41 | plt.xlabel("9Minutes")
42 | plt.savefig("chart.png")
43 | if show:plt.show()
44 |
45 | def predictFuture(m1,m2,old_pred,writeToFile=False):
46 | actual,latest_p = util.getCurrentData(label=True)
47 | actual = np.array(util.reduceCurrent(actual)).reshape(1,12)
48 | pred = util.augmentValue(net.predict(actual)[0],m1,m2)
49 | pred = float(int(pred[0]*100)/100)
50 | cex = util.getCEXData()
51 | slope,nrmse = predict.getslope(False)
52 | if writeToFile:
53 | f = open("results","a")
54 | f.write("[{}] Actual:{}$ Last Prediction:{}$ Next 9m:{}$".format(time.strftime("%H:%M:%S"),latest_p,old_pred,pred))
55 | f.close()
56 |
57 | c = conn.cursor()
58 | c.execute("INSERT INTO predict(actual,last,target,cex_ask,slope,nrmse) VALUES (?,?,?,?,?,?)",(latest_p,old_pred,pred,cex["ask"],slope,nrmse))
59 | conn.commit()
60 | print("[{}] Actual:{}$ Last Prediction:{}$ Next 9m:{}$ Slope:{}$ NRMSE:{}$\n".format(time.strftime("%H:%M:%S"),latest_p,old_pred,pred,slope,nrmse))
61 | return latest_p,pred
62 |
63 | if __name__ == '__main__':
64 | parser = argparse.ArgumentParser(description="Forecast btc price with deep learning.")
65 | parser.add_argument('-train',type=str,help="-train dataset.csv path")
66 | parser.add_argument('-run',type=str,help="-run dataset.csv path")
67 | parser.add_argument('-model',type=str,help='-model model\'s path')
68 | parser.add_argument('-iterations',type=int,help='-iteration number of epoches')
69 | parser.add_argument('-finetune',type=str,help='-finetune base-model path')
70 | args = parser.parse_args()
71 | print(args)
72 |
73 |
74 | #Assembling Net:
75 | buildNet()
76 | #data loading:
77 | file_name = args.run if args.run is not None else args.train
78 | print("Loading data...",end="")
79 | d = open(file_name,'r')
80 | data,labels = util.loadData(d)
81 | data = util.reduceMatRows(data)
82 | labels,m1,m2 =util.reduceVector(labels,getVal=True)
83 | print("{} chunk loaded!\n".format(len(labels)),end="")
84 |
85 | if args.run is not None:
86 | #Loading weights
87 | w_name = args.model
88 | net.load_weights(w_name)
89 | print("Starting main loop...")
90 | hip = 0
91 | reals,preds = [],[]
92 |
93 | for i in range(len(data)-40,len(data)):
94 | x = np.array(data[i]).reshape(1,12)
95 | predicted = util.augmentValue(net.predict(x)[0],m1,m2)[0]
96 | real = util.augmentValue(labels[i],m1,m2)
97 | preds.append(predicted)
98 | reals.append(real)
99 |
100 | while True:
101 | try:
102 | real,hip = predictFuture(m1,m2,hip,writeToFile=True)
103 | reals.append(real)
104 | preds.append(hip)
105 | time.sleep(wait_time)
106 | except KeyboardInterrupt:
107 | ### PLOTTING
108 | chart(reals,preds,show=False)
109 | print("Chart saved!")
110 | s = input("Type yes to close the program: ")
111 | if s.lower() == "yes":break
112 | print("Resuming...")
113 |
114 | print("Closing..")
115 |
116 | elif args.train is not None:
117 | if args.finetune is not None:
118 | model_name = args.finetune
119 | net.load_weights(model_name)
120 | print("Basic model loaded!")
121 | epochs = args.iterations
122 | #Training dnn
123 | print("training...")
124 | el = len(data)-10 #Last ten elements are for testing
125 | net.fit(np.array(data[:el]),np.array(labels[:el]),epochs=epochs,batch_size=300)
126 | print("trained!\nSaving...",end="")
127 | net.save_weights("model.h5")
128 | print("saved!")
129 |
130 | ### Predict all over the dataset to build the chart
131 | reals,preds = [],[]
132 | for i in range(len(data)-40,len(data)):
133 | x = np.array(data[i]).reshape(1,12)
134 | predicted = util.augmentValue(net.predict(x)[0],m1,m2)[0]
135 | real = util.augmentValue(labels[i],m1,m2)
136 | preds.append(predicted)
137 | reals.append(real)
138 |
139 | ### Predict Price the next 9m price (magic)
140 | real,hip = predictFuture(m1,m2,0)
141 | reals.append(real)
142 | preds.append(hip)
143 |
144 | ### PLOTTING
145 | chart(reals,preds)
146 |
147 | else :
148 | print("Wrong argument")
149 |
--------------------------------------------------------------------------------
/predict.py:
--------------------------------------------------------------------------------
1 |
2 | import sqlite3
3 | import pandas as pd
4 | import matplotlib.pyplot as plt
5 | import datetime
6 | import numpy as np
7 |
8 | def getslope(plot=False):
9 | con = sqlite3.connect('data.db')
10 | data = pd.read_sql_query("SELECT rowid,* from predict", con, index_col="created")
11 | print(data.tail(2))
12 |
13 | start = (datetime.datetime.utcnow() - datetime.timedelta(hours=1)).strftime("%Y-%m-%d %H:%M:%S")
14 | stop = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
15 | #print(start)
16 | #print(stop)
17 |
18 | selected = data.loc[(start < data.index) & (data.index < stop),'cex_ask']
19 | #print(selected)
20 | #selected.plot()
21 | #plt.show()
22 |
23 | try:
24 | coefficients, residuals, _, _, _ = np.polyfit(range(len(selected.index)),selected,1,full=True)
25 | mse = residuals[0]/(len(selected.index))
26 | nrmse = np.sqrt(mse)/(selected.max() - selected.min())
27 | slope = coefficients[0]
28 | if plot:
29 | print ('Slope: '+ str(slope))
30 | print('NRMSE: ' + str(nrmse))
31 | plt.plot(selected)
32 | plt.plot([coefficients[0]*x + coefficients[1] for x in range(len(selected))])
33 | plt.show()
34 | except:
35 | slope = 0
36 | nrmse = 0
37 | return slope,nrmse
38 |
39 |
40 | #getslope(True)
41 |
--------------------------------------------------------------------------------
/util.py:
--------------------------------------------------------------------------------
1 | import requests
2 |
3 | maxs =[]
4 | mins =[]
5 |
6 | def loadData(f_name):
7 | data = f_name.read().split("\n")
8 | data = data[:len(data)-1]
9 | label = []
10 | for i in range(len(data)):
11 | data[i] = data[i].split(",")
12 | data[i] = [float(x) for x in data[i]]
13 | label.append(data[i][len(data[i])-1])
14 | data[i] = data[i][0:len(data[i])-1]
15 | return data[:-2],label[2:] #Removing first two and last two so each X[i] tries to predict Y[i+2] (i've used i+2 and not to i+1 to force it to predict the future (O) )
16 |
17 | def reduceVector(vec,getVal=False):
18 | vect = []
19 | mx,mn = max(vec),min(vec)
20 | mx = mx+mn
21 | mn = mn-((mx-mn)*0.4)
22 | for x in vec:
23 | vect.append((x-mn)/(mx-mn))
24 | if not getVal:return vect
25 | else:return vect,mx,mn
26 |
27 | def reduceValue(x,mx,mn):
28 | return (x-mn)/(mx-mn)
29 |
30 | def augmentValue(x,mx,mn):
31 | return (mx-mn)*x+mn
32 |
33 | def reduceMatRows(data):
34 | l = len(data[0])
35 | for i in range(l):
36 | v = []
37 | for t in range(len(data)):
38 | v.append(data[t][i])
39 | v,mx,mn = reduceVector(v,getVal=True)
40 | maxs.append(mx)
41 | mins.append(mn)
42 | for t in range(len(data)):
43 | data[t][i] = v[t]
44 |
45 | return data
46 | def reduceCurrent(data):
47 | for i in range(len(data)):
48 | data[i] = reduceValue(data[i],maxs[i],mins[i])
49 | return data
50 |
51 | def getCurrentData(label=False):
52 |
53 | keys = ["price_usd","24h_volume_usd","market_cap_usd","available_supply","total_supply","percent_change_1h","percent_change_24h","percent_change_7d"]
54 | vect = []
55 | data = requests.get("https://api.coinmarketcap.com/v1/ticker/bitcoin/").json()[0]
56 | bstamp = requests.get("https://www.bitstamp.net/api/v2/ticker/btcusd/").json()
57 | bkc = requests.get("https://blockchain.info/ticker").json()
58 | '''
59 | for i in data.keys():
60 | if i in keys:
61 | vect.append(float(data[i]))
62 | '''
63 | for k in keys:
64 | for d in data.keys():
65 | if k == d:
66 | vect.append(float(data[d]))
67 |
68 | vect.append(float(bstamp["volume"]))
69 | vect.append(float(bstamp["vwap"]))
70 | vect.append(float(bkc["USD"]["sell"]))
71 | vect.append(float(bkc["USD"]["buy"]))
72 |
73 | #print("blockchain.info ",float(bkc["USD"]["15m"]))
74 | if label:
75 | return vect,float(bkc["USD"]["15m"])
76 | return vect
77 |
78 | def getCEXData():
79 | data = requests.get("https://cex.io/api/ticker/BTC/USD").json()
80 | #print("CEX ",data)
81 | return data
--------------------------------------------------------------------------------