├── README.md ├── data.db ├── dataset.csv ├── grabber.py ├── model.h5 ├── network.py ├── predict.py └── util.py /README.md: -------------------------------------------------------------------------------- 1 | # BitcoinForecast 2 | 3 | Predict bitcoin value for the next 9minutes, with Recurrental Neural Network GRU. 4 |
5 |

Requirements:

6 |
7 | 8 | 14 |
15 |

Data

16 | You can download data through the grabber.py 17 |
18 |

Model:

19 |
20 |

Generated with deepchart

21 |

Instructions

22 |

Clone the repo:

23 | 24 | git clone https://github.com/PiSimo/BitcoinForecast.git 25 | 26 |
27 |
28 | Training on new data:
29 | python3 network.py -train dataset_path -iterations number_of_training_iterations 30 |

To finetune the new model with an old one just add -finetune base_model_path to the line above.

31 |

At the end of the training you will have an updated model.h5 with the new weights and you will see a plot with the test results.

32 |
33 | Running:
34 | python3 network.py -run dataset_path -model model_path 35 |

The dataset is also required when you run, to perform normalization.

36 |

To visualize a plot with the real and predicted results enter Crtl-C and type no ,the program will create chart.png with the results.

37 | 38 |
39 | Working example with this model: 40 |
41 | (Red:Predicted,Green:Real values) 42 | 43 | 44 | -------------------------------------------------------------------------------- /data.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PiSimo/BitcoinForecast/158be42130e2a7f84ceb5829c7b17edbcf41bcb9/data.db -------------------------------------------------------------------------------- /grabber.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | ## 4 | ## run the code for about 2/3 days 5 | ## 6 | 7 | import requests 8 | import time 9 | 10 | f_name = input("dataset name:") 11 | f = open(f_name,"a") 12 | keys = ["price_usd","24h_volume_usd","market_cap_usd","available_supply","total_supply","percent_change_1h","percent_change_24h","percent_change_7d"] 13 | vals = [0]*len(keys) 14 | 15 | while True: 16 | data = requests.get("https://api.coinmarketcap.com/v1/ticker/bitcoin/").json()[0] 17 | bstamp = requests.get("https://www.bitstamp.net/api/v2/ticker/btcusd/").json() 18 | bkc = requests.get("https://blockchain.info/ticker").json() 19 | 20 | for d in data.keys(): 21 | if d in keys: 22 | indx = keys.index(d) 23 | vals[indx] = data[d] 24 | for val in vals: 25 | f.write(val+",") 26 | 27 | f.write("{},{},".format(bstamp["volume"],bstamp["vwap"])) 28 | f.write("{},{},{}".format(bkc["USD"]["sell"],bkc["USD"]["buy"],bkc["USD"]["15m"])) 29 | f.write("\n") 30 | f.flush() 31 | time.sleep(9*60) 32 | -------------------------------------------------------------------------------- /model.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PiSimo/BitcoinForecast/158be42130e2a7f84ceb5829c7b17edbcf41bcb9/model.h5 -------------------------------------------------------------------------------- /network.py: -------------------------------------------------------------------------------- 1 | import util 2 | import predict 3 | import time 4 | import argparse 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | from sys import argv,exit 8 | from keras.models import Sequential 9 | from keras.layers import Dense,Dropout,GRU,Reshape 10 | from keras.layers.normalization import BatchNormalization 11 | import sqlite3 12 | conn = sqlite3.connect('data.db') 13 | 14 | file_name = 'dataset.csv' 15 | net = None 16 | wait_time = 530 17 | 18 | def buildNet(w_init="glorot_uniform",act="tanh"): 19 | global net 20 | print("Building net..",end="") 21 | net = Sequential() 22 | net.add(Dense(12,kernel_initializer=w_init,input_dim=12,activation='linear')) 23 | net.add(Reshape((1,12))) 24 | net.add(BatchNormalization()) 25 | net.add(GRU(40,kernel_initializer=w_init,activation=act,return_sequences=True)) 26 | net.add(Dropout(0.4)) 27 | net.add(GRU(70,kernel_initializer=w_init,activation=act,return_sequences=True)) 28 | net.add(Dropout(0.3)) 29 | net.add(GRU(70,kernel_initializer=w_init,activation=act,return_sequences=True)) 30 | net.add(Dropout(0.4)) 31 | net.add(GRU(40,kernel_initializer=w_init,activation=act,return_sequences=False)) 32 | net.add(Dropout(0.4)) 33 | net.add(Dense(1,kernel_initializer=w_init,activation='linear')) 34 | net.compile(optimizer='nadam',loss='mse') 35 | print("done!") 36 | 37 | def chart(real,predicted,show=True): 38 | plt.plot(real,color='g') 39 | plt.plot(predicted,color='r') 40 | plt.ylabel('BTC/USD') 41 | plt.xlabel("9Minutes") 42 | plt.savefig("chart.png") 43 | if show:plt.show() 44 | 45 | def predictFuture(m1,m2,old_pred,writeToFile=False): 46 | actual,latest_p = util.getCurrentData(label=True) 47 | actual = np.array(util.reduceCurrent(actual)).reshape(1,12) 48 | pred = util.augmentValue(net.predict(actual)[0],m1,m2) 49 | pred = float(int(pred[0]*100)/100) 50 | cex = util.getCEXData() 51 | slope,nrmse = predict.getslope(False) 52 | if writeToFile: 53 | f = open("results","a") 54 | f.write("[{}] Actual:{}$ Last Prediction:{}$ Next 9m:{}$".format(time.strftime("%H:%M:%S"),latest_p,old_pred,pred)) 55 | f.close() 56 | 57 | c = conn.cursor() 58 | c.execute("INSERT INTO predict(actual,last,target,cex_ask,slope,nrmse) VALUES (?,?,?,?,?,?)",(latest_p,old_pred,pred,cex["ask"],slope,nrmse)) 59 | conn.commit() 60 | print("[{}] Actual:{}$ Last Prediction:{}$ Next 9m:{}$ Slope:{}$ NRMSE:{}$\n".format(time.strftime("%H:%M:%S"),latest_p,old_pred,pred,slope,nrmse)) 61 | return latest_p,pred 62 | 63 | if __name__ == '__main__': 64 | parser = argparse.ArgumentParser(description="Forecast btc price with deep learning.") 65 | parser.add_argument('-train',type=str,help="-train dataset.csv path") 66 | parser.add_argument('-run',type=str,help="-run dataset.csv path") 67 | parser.add_argument('-model',type=str,help='-model model\'s path') 68 | parser.add_argument('-iterations',type=int,help='-iteration number of epoches') 69 | parser.add_argument('-finetune',type=str,help='-finetune base-model path') 70 | args = parser.parse_args() 71 | print(args) 72 | 73 | 74 | #Assembling Net: 75 | buildNet() 76 | #data loading: 77 | file_name = args.run if args.run is not None else args.train 78 | print("Loading data...",end="") 79 | d = open(file_name,'r') 80 | data,labels = util.loadData(d) 81 | data = util.reduceMatRows(data) 82 | labels,m1,m2 =util.reduceVector(labels,getVal=True) 83 | print("{} chunk loaded!\n".format(len(labels)),end="") 84 | 85 | if args.run is not None: 86 | #Loading weights 87 | w_name = args.model 88 | net.load_weights(w_name) 89 | print("Starting main loop...") 90 | hip = 0 91 | reals,preds = [],[] 92 | 93 | for i in range(len(data)-40,len(data)): 94 | x = np.array(data[i]).reshape(1,12) 95 | predicted = util.augmentValue(net.predict(x)[0],m1,m2)[0] 96 | real = util.augmentValue(labels[i],m1,m2) 97 | preds.append(predicted) 98 | reals.append(real) 99 | 100 | while True: 101 | try: 102 | real,hip = predictFuture(m1,m2,hip,writeToFile=True) 103 | reals.append(real) 104 | preds.append(hip) 105 | time.sleep(wait_time) 106 | except KeyboardInterrupt: 107 | ### PLOTTING 108 | chart(reals,preds,show=False) 109 | print("Chart saved!") 110 | s = input("Type yes to close the program: ") 111 | if s.lower() == "yes":break 112 | print("Resuming...") 113 | 114 | print("Closing..") 115 | 116 | elif args.train is not None: 117 | if args.finetune is not None: 118 | model_name = args.finetune 119 | net.load_weights(model_name) 120 | print("Basic model loaded!") 121 | epochs = args.iterations 122 | #Training dnn 123 | print("training...") 124 | el = len(data)-10 #Last ten elements are for testing 125 | net.fit(np.array(data[:el]),np.array(labels[:el]),epochs=epochs,batch_size=300) 126 | print("trained!\nSaving...",end="") 127 | net.save_weights("model.h5") 128 | print("saved!") 129 | 130 | ### Predict all over the dataset to build the chart 131 | reals,preds = [],[] 132 | for i in range(len(data)-40,len(data)): 133 | x = np.array(data[i]).reshape(1,12) 134 | predicted = util.augmentValue(net.predict(x)[0],m1,m2)[0] 135 | real = util.augmentValue(labels[i],m1,m2) 136 | preds.append(predicted) 137 | reals.append(real) 138 | 139 | ### Predict Price the next 9m price (magic) 140 | real,hip = predictFuture(m1,m2,0) 141 | reals.append(real) 142 | preds.append(hip) 143 | 144 | ### PLOTTING 145 | chart(reals,preds) 146 | 147 | else : 148 | print("Wrong argument") 149 | -------------------------------------------------------------------------------- /predict.py: -------------------------------------------------------------------------------- 1 | 2 | import sqlite3 3 | import pandas as pd 4 | import matplotlib.pyplot as plt 5 | import datetime 6 | import numpy as np 7 | 8 | def getslope(plot=False): 9 | con = sqlite3.connect('data.db') 10 | data = pd.read_sql_query("SELECT rowid,* from predict", con, index_col="created") 11 | print(data.tail(2)) 12 | 13 | start = (datetime.datetime.utcnow() - datetime.timedelta(hours=1)).strftime("%Y-%m-%d %H:%M:%S") 14 | stop = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") 15 | #print(start) 16 | #print(stop) 17 | 18 | selected = data.loc[(start < data.index) & (data.index < stop),'cex_ask'] 19 | #print(selected) 20 | #selected.plot() 21 | #plt.show() 22 | 23 | try: 24 | coefficients, residuals, _, _, _ = np.polyfit(range(len(selected.index)),selected,1,full=True) 25 | mse = residuals[0]/(len(selected.index)) 26 | nrmse = np.sqrt(mse)/(selected.max() - selected.min()) 27 | slope = coefficients[0] 28 | if plot: 29 | print ('Slope: '+ str(slope)) 30 | print('NRMSE: ' + str(nrmse)) 31 | plt.plot(selected) 32 | plt.plot([coefficients[0]*x + coefficients[1] for x in range(len(selected))]) 33 | plt.show() 34 | except: 35 | slope = 0 36 | nrmse = 0 37 | return slope,nrmse 38 | 39 | 40 | #getslope(True) 41 | -------------------------------------------------------------------------------- /util.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | maxs =[] 4 | mins =[] 5 | 6 | def loadData(f_name): 7 | data = f_name.read().split("\n") 8 | data = data[:len(data)-1] 9 | label = [] 10 | for i in range(len(data)): 11 | data[i] = data[i].split(",") 12 | data[i] = [float(x) for x in data[i]] 13 | label.append(data[i][len(data[i])-1]) 14 | data[i] = data[i][0:len(data[i])-1] 15 | return data[:-2],label[2:] #Removing first two and last two so each X[i] tries to predict Y[i+2] (i've used i+2 and not to i+1 to force it to predict the future (O) ) 16 | 17 | def reduceVector(vec,getVal=False): 18 | vect = [] 19 | mx,mn = max(vec),min(vec) 20 | mx = mx+mn 21 | mn = mn-((mx-mn)*0.4) 22 | for x in vec: 23 | vect.append((x-mn)/(mx-mn)) 24 | if not getVal:return vect 25 | else:return vect,mx,mn 26 | 27 | def reduceValue(x,mx,mn): 28 | return (x-mn)/(mx-mn) 29 | 30 | def augmentValue(x,mx,mn): 31 | return (mx-mn)*x+mn 32 | 33 | def reduceMatRows(data): 34 | l = len(data[0]) 35 | for i in range(l): 36 | v = [] 37 | for t in range(len(data)): 38 | v.append(data[t][i]) 39 | v,mx,mn = reduceVector(v,getVal=True) 40 | maxs.append(mx) 41 | mins.append(mn) 42 | for t in range(len(data)): 43 | data[t][i] = v[t] 44 | 45 | return data 46 | def reduceCurrent(data): 47 | for i in range(len(data)): 48 | data[i] = reduceValue(data[i],maxs[i],mins[i]) 49 | return data 50 | 51 | def getCurrentData(label=False): 52 | 53 | keys = ["price_usd","24h_volume_usd","market_cap_usd","available_supply","total_supply","percent_change_1h","percent_change_24h","percent_change_7d"] 54 | vect = [] 55 | data = requests.get("https://api.coinmarketcap.com/v1/ticker/bitcoin/").json()[0] 56 | bstamp = requests.get("https://www.bitstamp.net/api/v2/ticker/btcusd/").json() 57 | bkc = requests.get("https://blockchain.info/ticker").json() 58 | ''' 59 | for i in data.keys(): 60 | if i in keys: 61 | vect.append(float(data[i])) 62 | ''' 63 | for k in keys: 64 | for d in data.keys(): 65 | if k == d: 66 | vect.append(float(data[d])) 67 | 68 | vect.append(float(bstamp["volume"])) 69 | vect.append(float(bstamp["vwap"])) 70 | vect.append(float(bkc["USD"]["sell"])) 71 | vect.append(float(bkc["USD"]["buy"])) 72 | 73 | #print("blockchain.info ",float(bkc["USD"]["15m"])) 74 | if label: 75 | return vect,float(bkc["USD"]["15m"]) 76 | return vect 77 | 78 | def getCEXData(): 79 | data = requests.get("https://cex.io/api/ticker/BTC/USD").json() 80 | #print("CEX ",data) 81 | return data --------------------------------------------------------------------------------